scribe-cms 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1207 -204
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1208 -205
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +969 -166
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +970 -168
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +136 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +136 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +18 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import remarkParse from 'remark-parse';
|
|
|
10
10
|
import { unified } from 'unified';
|
|
11
11
|
import { createJiti } from 'jiti';
|
|
12
12
|
import { createHash } from 'crypto';
|
|
13
|
-
import { GoogleGenAI } from '@google/genai';
|
|
13
|
+
import { GoogleGenAI, ThinkingLevel, ApiError } from '@google/genai';
|
|
14
14
|
import dotenv from 'dotenv';
|
|
15
15
|
import { serve } from '@hono/node-server';
|
|
16
16
|
import { Hono } from 'hono';
|
|
@@ -59,6 +59,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
function readSchemaVersion(db) {
|
|
62
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
63
|
+
if (!metaTable) return 0;
|
|
62
64
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
63
65
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
64
66
|
}
|
|
@@ -83,7 +85,7 @@ var SCHEMA_VERSION, MIGRATIONS;
|
|
|
83
85
|
var init_sqlite = __esm({
|
|
84
86
|
"src/storage/sqlite.ts"() {
|
|
85
87
|
init_esm_shims();
|
|
86
|
-
SCHEMA_VERSION =
|
|
88
|
+
SCHEMA_VERSION = 5;
|
|
87
89
|
MIGRATIONS = [
|
|
88
90
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
89
91
|
key TEXT PRIMARY KEY,
|
|
@@ -131,7 +133,30 @@ var init_sqlite = __esm({
|
|
|
131
133
|
UNIQUE (content_type, en_slug, en_hash)
|
|
132
134
|
)`,
|
|
133
135
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
134
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
136
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
137
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
138
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
139
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
140
|
+
model TEXT NOT NULL,
|
|
141
|
+
display_model TEXT NOT NULL,
|
|
142
|
+
created_at TEXT NOT NULL,
|
|
143
|
+
state TEXT NOT NULL,
|
|
144
|
+
completed_at TEXT
|
|
145
|
+
)`,
|
|
146
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
147
|
+
job_id INTEGER NOT NULL,
|
|
148
|
+
request_index INTEGER NOT NULL,
|
|
149
|
+
content_type TEXT NOT NULL,
|
|
150
|
+
en_slug TEXT NOT NULL,
|
|
151
|
+
locale TEXT NOT NULL,
|
|
152
|
+
en_hash TEXT NOT NULL,
|
|
153
|
+
snapshot_id INTEGER NOT NULL,
|
|
154
|
+
status TEXT NOT NULL,
|
|
155
|
+
error TEXT,
|
|
156
|
+
PRIMARY KEY (job_id, request_index)
|
|
157
|
+
)`,
|
|
158
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
159
|
+
ON translation_batch_items(status)`
|
|
135
160
|
];
|
|
136
161
|
}
|
|
137
162
|
});
|
|
@@ -173,26 +198,6 @@ function getRelationTarget(schema) {
|
|
|
173
198
|
}
|
|
174
199
|
return null;
|
|
175
200
|
}
|
|
176
|
-
function unwrapSchema(schema) {
|
|
177
|
-
let current = schema;
|
|
178
|
-
for (let i = 0; i < 8; i++) {
|
|
179
|
-
const anySchema = current;
|
|
180
|
-
if (typeof anySchema.unwrap === "function") {
|
|
181
|
-
current = anySchema.unwrap();
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
if (typeof anySchema.removeDefault === "function") {
|
|
185
|
-
current = anySchema.removeDefault();
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
if (anySchema._def?.innerType) {
|
|
189
|
-
current = anySchema._def.innerType;
|
|
190
|
-
continue;
|
|
191
|
-
}
|
|
192
|
-
break;
|
|
193
|
-
}
|
|
194
|
-
return current;
|
|
195
|
-
}
|
|
196
201
|
function peelOptionalWrappers(schema) {
|
|
197
202
|
let current = schema;
|
|
198
203
|
for (let i = 0; i < 8; i++) {
|
|
@@ -338,6 +343,7 @@ function resolveConfig(input, baseDir) {
|
|
|
338
343
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
339
344
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
340
345
|
}
|
|
346
|
+
const localeFallbacks = raw.localeFallbacks === false ? {} : deriveLocaleFallbacks(raw.locales, defaultLocale);
|
|
341
347
|
const projectRoot = path4.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
342
348
|
const contentRoot = path4.resolve(projectRoot, raw.contentDir ?? "content");
|
|
343
349
|
const storePath = path4.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -367,6 +373,7 @@ function resolveConfig(input, baseDir) {
|
|
|
367
373
|
defaultLocale,
|
|
368
374
|
localeRouting,
|
|
369
375
|
localePresets: raw.localePresets,
|
|
376
|
+
localeFallbacks,
|
|
370
377
|
translate: raw.translate,
|
|
371
378
|
types
|
|
372
379
|
};
|
|
@@ -376,12 +383,37 @@ function resolveConfig(input, baseDir) {
|
|
|
376
383
|
});
|
|
377
384
|
return config;
|
|
378
385
|
}
|
|
386
|
+
function deriveLocaleFallbacks(locales, defaultLocale) {
|
|
387
|
+
const byLowercase = new Map(locales.map((l) => [l.toLowerCase(), l]));
|
|
388
|
+
const fallbacks = {};
|
|
389
|
+
for (const locale of locales) {
|
|
390
|
+
const subtags = locale.split("-");
|
|
391
|
+
if (subtags.length < 2) continue;
|
|
392
|
+
const chain = [];
|
|
393
|
+
for (let end = subtags.length - 1; end >= 1; end--) {
|
|
394
|
+
const prefix = byLowercase.get(subtags.slice(0, end).join("-").toLowerCase());
|
|
395
|
+
if (prefix && prefix !== locale && prefix !== defaultLocale) {
|
|
396
|
+
chain.push(prefix);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (chain.length > 0) {
|
|
400
|
+
fallbacks[locale] = chain;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return fallbacks;
|
|
404
|
+
}
|
|
379
405
|
|
|
380
406
|
// src/create-project.ts
|
|
381
407
|
init_esm_shims();
|
|
382
408
|
|
|
383
409
|
// src/core/introspect-schema.ts
|
|
384
410
|
init_esm_shims();
|
|
411
|
+
function getArrayElement(schema) {
|
|
412
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
413
|
+
return schema.element;
|
|
414
|
+
}
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
385
417
|
function introspectSchema(schema, prefix = []) {
|
|
386
418
|
const relation = getRelationTarget(schema);
|
|
387
419
|
if (relation && prefix.length > 0) {
|
|
@@ -395,7 +427,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
395
427
|
}
|
|
396
428
|
];
|
|
397
429
|
}
|
|
398
|
-
|
|
430
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
431
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
432
|
+
}
|
|
433
|
+
const base = peelOptionalWrappers(schema);
|
|
399
434
|
if (base instanceof Object && "shape" in base) {
|
|
400
435
|
const shape = base.shape;
|
|
401
436
|
const fields = [];
|
|
@@ -404,9 +439,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
404
439
|
}
|
|
405
440
|
return fields;
|
|
406
441
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
const elementBase =
|
|
442
|
+
const element = getArrayElement(base);
|
|
443
|
+
if (element) {
|
|
444
|
+
const elementBase = peelOptionalWrappers(element);
|
|
410
445
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
411
446
|
const shape = elementBase.shape;
|
|
412
447
|
const fields = [];
|
|
@@ -418,15 +453,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
418
453
|
}
|
|
419
454
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
420
455
|
}
|
|
421
|
-
function
|
|
422
|
-
const
|
|
456
|
+
function buildPathTrie(paths) {
|
|
457
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
423
458
|
for (const path16 of paths) {
|
|
424
|
-
|
|
459
|
+
let node = root;
|
|
460
|
+
for (const segment of path16) {
|
|
461
|
+
let child = node.children.get(segment);
|
|
462
|
+
if (!child) {
|
|
463
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
464
|
+
node.children.set(segment, child);
|
|
465
|
+
}
|
|
466
|
+
node = child;
|
|
467
|
+
}
|
|
468
|
+
node.leaf = true;
|
|
469
|
+
}
|
|
470
|
+
return root;
|
|
471
|
+
}
|
|
472
|
+
function extractNode(data, trie) {
|
|
473
|
+
if (trie.leaf) return data;
|
|
474
|
+
const star = trie.children.get("*");
|
|
475
|
+
if (star) {
|
|
476
|
+
if (!Array.isArray(data)) return void 0;
|
|
477
|
+
return data.map(
|
|
478
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
479
|
+
);
|
|
480
|
+
}
|
|
481
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
482
|
+
const record = data;
|
|
483
|
+
const out = {};
|
|
484
|
+
let any = false;
|
|
485
|
+
for (const [key, child] of trie.children) {
|
|
486
|
+
const value = extractNode(record[key], child);
|
|
425
487
|
if (value !== void 0) {
|
|
426
|
-
|
|
488
|
+
out[key] = value;
|
|
489
|
+
any = true;
|
|
427
490
|
}
|
|
428
491
|
}
|
|
429
|
-
return out;
|
|
492
|
+
return any ? out : void 0;
|
|
493
|
+
}
|
|
494
|
+
function extractByPaths(data, paths) {
|
|
495
|
+
if (paths.length === 0) return {};
|
|
496
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
497
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
430
498
|
}
|
|
431
499
|
function pickTranslatable(data, schema) {
|
|
432
500
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -441,32 +509,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
441
509
|
const translatable = pickTranslatable(localeData, schema);
|
|
442
510
|
return deepMerge(structural, translatable);
|
|
443
511
|
}
|
|
444
|
-
function getAtPath(obj, path16) {
|
|
445
|
-
let current = obj;
|
|
446
|
-
for (const segment of path16) {
|
|
447
|
-
if (segment === "*") {
|
|
448
|
-
if (!Array.isArray(current)) return void 0;
|
|
449
|
-
return current.map(
|
|
450
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path16.slice(path16.indexOf("*") + 1)) : void 0
|
|
451
|
-
);
|
|
452
|
-
}
|
|
453
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
454
|
-
current = current[segment];
|
|
455
|
-
}
|
|
456
|
-
return current;
|
|
457
|
-
}
|
|
458
|
-
function setAtPath(obj, path16, value) {
|
|
459
|
-
if (path16.length === 0) return;
|
|
460
|
-
if (path16.length === 1) {
|
|
461
|
-
obj[path16[0]] = value;
|
|
462
|
-
return;
|
|
463
|
-
}
|
|
464
|
-
const [head, ...rest] = path16;
|
|
465
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
466
|
-
obj[head] = {};
|
|
467
|
-
}
|
|
468
|
-
setAtPath(obj[head], rest, value);
|
|
469
|
-
}
|
|
470
512
|
function deepMerge(base, overlay) {
|
|
471
513
|
const out = { ...base };
|
|
472
514
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -757,7 +799,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
757
799
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
758
800
|
const attrStart = i;
|
|
759
801
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
760
|
-
body.slice(attrStart, i);
|
|
802
|
+
const attrName = body.slice(attrStart, i);
|
|
803
|
+
if (!attrName) break;
|
|
761
804
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
762
805
|
if (body[i] !== "=") {
|
|
763
806
|
tagOut += body.slice(attrStart, i);
|
|
@@ -766,10 +809,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
766
809
|
tagOut += body.slice(attrStart, i);
|
|
767
810
|
tagOut += "=";
|
|
768
811
|
i += 1;
|
|
812
|
+
const wsStart = i;
|
|
769
813
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
814
|
+
tagOut += body.slice(wsStart, i);
|
|
770
815
|
const quote = body[i];
|
|
771
816
|
if (quote !== '"' && quote !== "'") {
|
|
772
|
-
|
|
817
|
+
break;
|
|
773
818
|
}
|
|
774
819
|
if (quote === "'") {
|
|
775
820
|
const valStart2 = i + 1;
|
|
@@ -1091,7 +1136,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1091
1136
|
|
|
1092
1137
|
// src/i18n/resolve-document.ts
|
|
1093
1138
|
init_esm_shims();
|
|
1094
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1139
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1095
1140
|
const urlBuilder = createUrlBuilder({
|
|
1096
1141
|
locales: [defaultLocale, locale],
|
|
1097
1142
|
defaultLocale,
|
|
@@ -1105,15 +1150,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1105
1150
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1106
1151
|
const found = docIdx.bySlug.get(slug);
|
|
1107
1152
|
if (!found) continue;
|
|
1108
|
-
const
|
|
1109
|
-
|
|
1153
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1154
|
+
let target;
|
|
1155
|
+
let targetLocale = locale;
|
|
1156
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1157
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1158
|
+
if (cand) {
|
|
1159
|
+
target = cand;
|
|
1160
|
+
targetLocale = candidateLocale;
|
|
1161
|
+
break;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
if (target) {
|
|
1165
|
+
if (target.slug === slug) {
|
|
1166
|
+
return { document: target, actualLocale: targetLocale };
|
|
1167
|
+
}
|
|
1110
1168
|
if (!type.path) {
|
|
1111
1169
|
return { document: null, actualLocale: locale };
|
|
1112
1170
|
}
|
|
1113
1171
|
return {
|
|
1114
1172
|
document: null,
|
|
1115
1173
|
actualLocale: locale,
|
|
1116
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1174
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1117
1175
|
};
|
|
1118
1176
|
}
|
|
1119
1177
|
if (docLocale === defaultLocale) {
|
|
@@ -1144,13 +1202,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1144
1202
|
}
|
|
1145
1203
|
return { document: null, actualLocale: locale };
|
|
1146
1204
|
}
|
|
1147
|
-
function getSlugForLocale(document, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1148
|
-
if (sourceLocale === targetLocale) return document.slug;
|
|
1149
|
-
const englishSlug = sourceLocale === defaultLocale ? document.slug : document.enSlug;
|
|
1150
|
-
if (!englishSlug) return null;
|
|
1151
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1152
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1153
|
-
}
|
|
1154
1205
|
|
|
1155
1206
|
// src/create-project.ts
|
|
1156
1207
|
function comparatorFor(orderBy) {
|
|
@@ -1203,7 +1254,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1203
1254
|
config.defaultLocale,
|
|
1204
1255
|
load(),
|
|
1205
1256
|
type,
|
|
1206
|
-
config.localeRouting
|
|
1257
|
+
config.localeRouting,
|
|
1258
|
+
config.localeFallbacks[locale] ?? []
|
|
1207
1259
|
);
|
|
1208
1260
|
if (result.document && type.path) {
|
|
1209
1261
|
return {
|
|
@@ -1225,8 +1277,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1225
1277
|
const params = [];
|
|
1226
1278
|
for (const locale of options.locales ?? config.locales) {
|
|
1227
1279
|
const localeIdx = all.get(locale);
|
|
1280
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1228
1281
|
for (const doc of enIdx.bySlug.values()) {
|
|
1229
|
-
|
|
1282
|
+
let slug;
|
|
1283
|
+
if (locale === config.defaultLocale) {
|
|
1284
|
+
slug = doc.slug;
|
|
1285
|
+
} else {
|
|
1286
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1287
|
+
}
|
|
1230
1288
|
params.push({ locale, slug });
|
|
1231
1289
|
}
|
|
1232
1290
|
}
|
|
@@ -1688,7 +1746,7 @@ init_sqlite();
|
|
|
1688
1746
|
|
|
1689
1747
|
// src/validate/validate-relations.ts
|
|
1690
1748
|
init_esm_shims();
|
|
1691
|
-
function
|
|
1749
|
+
function getAtPath(obj, fieldPath) {
|
|
1692
1750
|
let current = obj;
|
|
1693
1751
|
for (const segment of fieldPath) {
|
|
1694
1752
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -1733,7 +1791,7 @@ function validateRelations(project) {
|
|
|
1733
1791
|
});
|
|
1734
1792
|
continue;
|
|
1735
1793
|
}
|
|
1736
|
-
const value =
|
|
1794
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
1737
1795
|
if (value === void 0 || value === null || value === "") continue;
|
|
1738
1796
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
1739
1797
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2117,6 +2175,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2117
2175
|
// src/translate/page-translator.ts
|
|
2118
2176
|
init_esm_shims();
|
|
2119
2177
|
|
|
2178
|
+
// src/storage/batch-jobs.ts
|
|
2179
|
+
init_esm_shims();
|
|
2180
|
+
function insertBatchJob(db, input) {
|
|
2181
|
+
const info = db.prepare(
|
|
2182
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2183
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2184
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2185
|
+
return Number(info.lastInsertRowid);
|
|
2186
|
+
}
|
|
2187
|
+
function insertBatchItems(db, jobId, items) {
|
|
2188
|
+
const stmt = db.prepare(
|
|
2189
|
+
`INSERT INTO translation_batch_items (
|
|
2190
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2191
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2192
|
+
);
|
|
2193
|
+
for (const item of items) {
|
|
2194
|
+
stmt.run(
|
|
2195
|
+
jobId,
|
|
2196
|
+
item.requestIndex,
|
|
2197
|
+
item.contentType,
|
|
2198
|
+
item.enSlug,
|
|
2199
|
+
item.locale,
|
|
2200
|
+
item.enHash,
|
|
2201
|
+
item.snapshotId
|
|
2202
|
+
);
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
function listPendingBatchJobs(db) {
|
|
2206
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2207
|
+
}
|
|
2208
|
+
function listBatchItems(db, jobId) {
|
|
2209
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2210
|
+
}
|
|
2211
|
+
function listPendingBatchItems(db) {
|
|
2212
|
+
return db.prepare(
|
|
2213
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2214
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2215
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2216
|
+
ORDER BY i.job_id, i.request_index`
|
|
2217
|
+
).all();
|
|
2218
|
+
}
|
|
2219
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2220
|
+
const info = db.prepare(
|
|
2221
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2222
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2223
|
+
).run(state, completedAt, jobId);
|
|
2224
|
+
return info.changes > 0;
|
|
2225
|
+
}
|
|
2226
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2227
|
+
db.prepare(
|
|
2228
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2229
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2230
|
+
}
|
|
2231
|
+
|
|
2232
|
+
// src/translate/page-translator.ts
|
|
2233
|
+
init_sqlite();
|
|
2234
|
+
|
|
2235
|
+
// src/translate/batch-worklist.ts
|
|
2236
|
+
init_esm_shims();
|
|
2237
|
+
|
|
2120
2238
|
// src/history/record-snapshot.ts
|
|
2121
2239
|
init_esm_shims();
|
|
2122
2240
|
init_sqlite();
|
|
@@ -2134,9 +2252,111 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2134
2252
|
return id;
|
|
2135
2253
|
}
|
|
2136
2254
|
|
|
2137
|
-
// src/translate/
|
|
2255
|
+
// src/translate/batch-worklist.ts
|
|
2138
2256
|
init_sqlite();
|
|
2139
2257
|
|
|
2258
|
+
// src/translate/gemini-batch.ts
|
|
2259
|
+
init_esm_shims();
|
|
2260
|
+
|
|
2261
|
+
// src/translate/retry.ts
|
|
2262
|
+
init_esm_shims();
|
|
2263
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2264
|
+
function statusFromError(error) {
|
|
2265
|
+
if (error instanceof ApiError && typeof error.status === "number") {
|
|
2266
|
+
return error.status;
|
|
2267
|
+
}
|
|
2268
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2269
|
+
const status = error.status;
|
|
2270
|
+
if (typeof status === "number") return status;
|
|
2271
|
+
}
|
|
2272
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2273
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2274
|
+
if (match) return Number(match[1]);
|
|
2275
|
+
return void 0;
|
|
2276
|
+
}
|
|
2277
|
+
function isNetworkError(error) {
|
|
2278
|
+
const err = error;
|
|
2279
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2280
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2281
|
+
return true;
|
|
2282
|
+
}
|
|
2283
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2284
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2285
|
+
}
|
|
2286
|
+
function isTransientError(error) {
|
|
2287
|
+
const status = statusFromError(error);
|
|
2288
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2289
|
+
return isNetworkError(error);
|
|
2290
|
+
}
|
|
2291
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2292
|
+
async function withRetry(fn, options = {}) {
|
|
2293
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2294
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2295
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2296
|
+
const random = options.random ?? Math.random;
|
|
2297
|
+
let lastError;
|
|
2298
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2299
|
+
try {
|
|
2300
|
+
return await fn();
|
|
2301
|
+
} catch (error) {
|
|
2302
|
+
lastError = error;
|
|
2303
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2304
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2305
|
+
const delay = backoff + random() * backoff;
|
|
2306
|
+
await sleep2(delay);
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
throw lastError;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
// src/translate/gemini-batch.ts
|
|
2313
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2314
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2315
|
+
"SUCCEEDED",
|
|
2316
|
+
"FAILED",
|
|
2317
|
+
"CANCELLED",
|
|
2318
|
+
"EXPIRED",
|
|
2319
|
+
"PARTIALLY_SUCCEEDED"
|
|
2320
|
+
]);
|
|
2321
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2322
|
+
function normalizeBatchState(state) {
|
|
2323
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2324
|
+
}
|
|
2325
|
+
function isTerminalBatchState(state) {
|
|
2326
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2327
|
+
}
|
|
2328
|
+
function isSuccessfulBatchState(state) {
|
|
2329
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2330
|
+
}
|
|
2331
|
+
function makeClient(apiKey) {
|
|
2332
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2333
|
+
if (!key) {
|
|
2334
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2335
|
+
}
|
|
2336
|
+
return new GoogleGenAI({ apiKey: key });
|
|
2337
|
+
}
|
|
2338
|
+
function textFromBatchResponse(response) {
|
|
2339
|
+
if (typeof response.text === "string") return response.text;
|
|
2340
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2341
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2342
|
+
}
|
|
2343
|
+
async function createGeminiBatchJob(input) {
|
|
2344
|
+
const ai = makeClient(input.apiKey);
|
|
2345
|
+
const job = await withRetry(
|
|
2346
|
+
() => ai.batches.create({
|
|
2347
|
+
model: input.model,
|
|
2348
|
+
src: input.requests,
|
|
2349
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2350
|
+
})
|
|
2351
|
+
);
|
|
2352
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2353
|
+
return job;
|
|
2354
|
+
}
|
|
2355
|
+
async function getGeminiBatchJob(input) {
|
|
2356
|
+
const ai = makeClient(input.apiKey);
|
|
2357
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2140
2360
|
// src/translate/gemini-client.ts
|
|
2141
2361
|
init_esm_shims();
|
|
2142
2362
|
|
|
@@ -2161,6 +2381,12 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2161
2381
|
if (alias) return alias[0];
|
|
2162
2382
|
return name;
|
|
2163
2383
|
}
|
|
2384
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2385
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2386
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: ThinkingLevel.LOW };
|
|
2387
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2388
|
+
return void 0;
|
|
2389
|
+
}
|
|
2164
2390
|
|
|
2165
2391
|
// src/translate/gemini-client.ts
|
|
2166
2392
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
@@ -2174,11 +2400,36 @@ function extractJson(text) {
|
|
|
2174
2400
|
return trimmed;
|
|
2175
2401
|
}
|
|
2176
2402
|
function parseGeminiResponse(text) {
|
|
2403
|
+
let parsed;
|
|
2177
2404
|
try {
|
|
2178
|
-
|
|
2405
|
+
parsed = JSON.parse(text.trim());
|
|
2179
2406
|
} catch {
|
|
2180
|
-
|
|
2407
|
+
parsed = JSON.parse(extractJson(text));
|
|
2408
|
+
}
|
|
2409
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2410
|
+
parsed.frontmatter = {};
|
|
2181
2411
|
}
|
|
2412
|
+
return parsed;
|
|
2413
|
+
}
|
|
2414
|
+
function buildGeminiRequestConfig(input) {
|
|
2415
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2416
|
+
return {
|
|
2417
|
+
responseMimeType: "application/json",
|
|
2418
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2419
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2420
|
+
};
|
|
2421
|
+
}
|
|
2422
|
+
function usageFromResponse(response) {
|
|
2423
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2424
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2425
|
+
return {
|
|
2426
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2427
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2428
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2429
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2430
|
+
thoughtsTokens,
|
|
2431
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2432
|
+
};
|
|
2182
2433
|
}
|
|
2183
2434
|
async function translatePageWithGemini(input) {
|
|
2184
2435
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2190,28 +2441,28 @@ async function translatePageWithGemini(input) {
|
|
|
2190
2441
|
);
|
|
2191
2442
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2192
2443
|
const ai = new GoogleGenAI({ apiKey });
|
|
2193
|
-
const response = await
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2444
|
+
const response = await withRetry(
|
|
2445
|
+
() => ai.models.generateContent({
|
|
2446
|
+
model: apiModel,
|
|
2447
|
+
contents: input.prompt,
|
|
2448
|
+
config: buildGeminiRequestConfig({
|
|
2449
|
+
apiModelId: apiModel,
|
|
2450
|
+
responseSchema: input.responseSchema
|
|
2451
|
+
})
|
|
2452
|
+
})
|
|
2453
|
+
);
|
|
2201
2454
|
const raw = response.text ?? "";
|
|
2202
2455
|
const parsed = parseGeminiResponse(raw);
|
|
2203
2456
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2204
2457
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2205
2458
|
}
|
|
2206
|
-
|
|
2207
|
-
const usage = {
|
|
2208
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2209
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2210
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2211
|
-
};
|
|
2212
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2459
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2213
2460
|
}
|
|
2214
2461
|
|
|
2462
|
+
// src/translate/translate-core.ts
|
|
2463
|
+
init_esm_shims();
|
|
2464
|
+
init_sqlite();
|
|
2465
|
+
|
|
2215
2466
|
// src/translate/gemini-pricing.ts
|
|
2216
2467
|
init_esm_shims();
|
|
2217
2468
|
var CONTEXT_TIER_TOKENS = 2e5;
|
|
@@ -2230,12 +2481,14 @@ function resolveModelPricing(model) {
|
|
|
2230
2481
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2231
2482
|
return match?.[1];
|
|
2232
2483
|
}
|
|
2233
|
-
|
|
2484
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2485
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2234
2486
|
const pricing = resolveModelPricing(model);
|
|
2235
2487
|
if (!pricing) return void 0;
|
|
2236
2488
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2237
2489
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2238
|
-
|
|
2490
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2491
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2239
2492
|
}
|
|
2240
2493
|
function formatTokenCount(tokens) {
|
|
2241
2494
|
if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
|
|
@@ -2263,6 +2516,7 @@ var LOCALE_NAMES = {
|
|
|
2263
2516
|
"zh-CN": "Simplified Chinese",
|
|
2264
2517
|
"zh-TW": "Traditional Chinese",
|
|
2265
2518
|
pt: "Portuguese",
|
|
2519
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2266
2520
|
ja: "Japanese",
|
|
2267
2521
|
ru: "Russian",
|
|
2268
2522
|
it: "Italian",
|
|
@@ -2270,29 +2524,29 @@ var LOCALE_NAMES = {
|
|
|
2270
2524
|
};
|
|
2271
2525
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2272
2526
|
return [
|
|
2273
|
-
`Localize the content
|
|
2527
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2274
2528
|
"Do not translate word-for-word.",
|
|
2275
2529
|
"Preserve the source tone and brand voice.",
|
|
2276
2530
|
"Write as if a native speaker authored it for the target market."
|
|
2277
2531
|
].join(" ");
|
|
2278
2532
|
}
|
|
2533
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2534
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2535
|
+
if (!hasFrontmatter) {
|
|
2536
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2537
|
+
}
|
|
2538
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2539
|
+
}
|
|
2279
2540
|
function buildPageTranslationPrompt(input) {
|
|
2280
2541
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2281
|
-
const
|
|
2282
|
-
const
|
|
2283
|
-
|
|
2542
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2543
|
+
const prefix = [
|
|
2544
|
+
TASK_FRAMING,
|
|
2284
2545
|
"",
|
|
2285
2546
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2286
2547
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2287
2548
|
"## Rules",
|
|
2288
2549
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2289
|
-
...input.slugStrategy === "localized" ? [
|
|
2290
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2291
|
-
] : [],
|
|
2292
|
-
"",
|
|
2293
|
-
"## Output format",
|
|
2294
|
-
"Return ONLY valid JSON with keys:",
|
|
2295
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2296
2550
|
"",
|
|
2297
2551
|
"## EN translatable frontmatter (JSON)",
|
|
2298
2552
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2300,7 +2554,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2300
2554
|
"## EN body (MDX)",
|
|
2301
2555
|
input.enBody
|
|
2302
2556
|
];
|
|
2303
|
-
|
|
2557
|
+
const suffix = [
|
|
2558
|
+
"",
|
|
2559
|
+
"## Target language",
|
|
2560
|
+
localizationPrompt,
|
|
2561
|
+
...input.slugStrategy === "localized" ? [
|
|
2562
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2563
|
+
] : [],
|
|
2564
|
+
"",
|
|
2565
|
+
"## Output format",
|
|
2566
|
+
"Return ONLY valid JSON with keys:",
|
|
2567
|
+
buildOutputFormatLine(
|
|
2568
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2569
|
+
input.slugStrategy
|
|
2570
|
+
)
|
|
2571
|
+
];
|
|
2572
|
+
return [...prefix, ...suffix].join("\n");
|
|
2304
2573
|
}
|
|
2305
2574
|
|
|
2306
2575
|
// src/translate/response-schema.ts
|
|
@@ -2351,9 +2620,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2351
2620
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2352
2621
|
try {
|
|
2353
2622
|
const translatable = buildTranslatableSubschema(schema);
|
|
2354
|
-
if (!translatable) return null;
|
|
2355
2623
|
const responseShape = {
|
|
2356
|
-
frontmatter: translatable,
|
|
2624
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2357
2625
|
body: z.string()
|
|
2358
2626
|
};
|
|
2359
2627
|
if (slugStrategy === "localized") {
|
|
@@ -2434,7 +2702,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2434
2702
|
return { ok: true, frontmatter };
|
|
2435
2703
|
}
|
|
2436
2704
|
|
|
2437
|
-
// src/translate/
|
|
2705
|
+
// src/translate/translate-core.ts
|
|
2706
|
+
function translationItemKey(item) {
|
|
2707
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2708
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2709
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2710
|
+
}
|
|
2438
2711
|
function summarizeResults(results, durationMs) {
|
|
2439
2712
|
return results.reduce(
|
|
2440
2713
|
(totals, result) => {
|
|
@@ -2470,17 +2743,6 @@ function formatTranslateError(error) {
|
|
|
2470
2743
|
}
|
|
2471
2744
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2472
2745
|
}
|
|
2473
|
-
async function runPool(items, concurrency, worker) {
|
|
2474
|
-
if (items.length === 0) return;
|
|
2475
|
-
let nextIndex = 0;
|
|
2476
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2477
|
-
while (nextIndex < items.length) {
|
|
2478
|
-
const index = nextIndex++;
|
|
2479
|
-
await worker(items[index], index);
|
|
2480
|
-
}
|
|
2481
|
-
});
|
|
2482
|
-
await Promise.all(runners);
|
|
2483
|
-
}
|
|
2484
2746
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2485
2747
|
const fm = enDoc.frontmatter;
|
|
2486
2748
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2489,73 +2751,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2489
2751
|
}
|
|
2490
2752
|
return enSlug;
|
|
2491
2753
|
}
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
const base = {
|
|
2754
|
+
function baseForItem(item) {
|
|
2755
|
+
return {
|
|
2495
2756
|
contentType: item.contentType,
|
|
2496
2757
|
enSlug: item.enSlug,
|
|
2497
2758
|
locale: item.locale
|
|
2498
2759
|
};
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2760
|
+
}
|
|
2761
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
2762
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
2763
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
2764
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
2765
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
2766
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
2767
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
2768
|
+
const db = openStore(config, "readonly");
|
|
2769
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
2770
|
+
db.close();
|
|
2771
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
2772
|
+
return {
|
|
2773
|
+
status: "done",
|
|
2774
|
+
result: {
|
|
2775
|
+
...baseForItem(item),
|
|
2512
2776
|
skipped: true,
|
|
2513
2777
|
reason: "fresh",
|
|
2514
2778
|
durationMs: Date.now() - startedAt
|
|
2515
|
-
}
|
|
2516
|
-
}
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2779
|
+
}
|
|
2780
|
+
};
|
|
2781
|
+
}
|
|
2782
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
2783
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
2784
|
+
const prompt = buildPageTranslationPrompt({
|
|
2785
|
+
resolved: resolvedTranslate,
|
|
2786
|
+
targetLocale: item.locale,
|
|
2787
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
2788
|
+
translatableFrontmatter: payload.frontmatter,
|
|
2789
|
+
enBody: payload.body,
|
|
2790
|
+
slugStrategy: type.slugStrategy
|
|
2791
|
+
});
|
|
2792
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
2793
|
+
return {
|
|
2794
|
+
status: "ready",
|
|
2795
|
+
prepared: {
|
|
2796
|
+
item,
|
|
2797
|
+
type,
|
|
2798
|
+
enDoc,
|
|
2799
|
+
payload,
|
|
2800
|
+
currentEnHash,
|
|
2801
|
+
existingSlug: existing?.slug,
|
|
2538
2802
|
model,
|
|
2803
|
+
prompt,
|
|
2539
2804
|
responseSchema: responseSchema ?? void 0
|
|
2540
|
-
}
|
|
2541
|
-
|
|
2805
|
+
}
|
|
2806
|
+
};
|
|
2807
|
+
}
|
|
2808
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
2809
|
+
const { item, type, enDoc, payload } = prepared;
|
|
2810
|
+
const base = baseForItem(item);
|
|
2811
|
+
try {
|
|
2812
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2542
2813
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2543
2814
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2544
2815
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2545
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
2816
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2546
2817
|
if (!validated.ok) {
|
|
2547
2818
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2548
2819
|
}
|
|
2549
2820
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2550
|
-
|
|
2821
|
+
output.parsed.body
|
|
2551
2822
|
);
|
|
2552
2823
|
const writeDb = openStore(config, "readwrite");
|
|
2553
|
-
const snapshotId = recordEnSnapshot(
|
|
2824
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2554
2825
|
config,
|
|
2555
2826
|
{
|
|
2556
2827
|
contentType: type.id,
|
|
2557
2828
|
enSlug: item.enSlug,
|
|
2558
|
-
enHash: currentEnHash,
|
|
2829
|
+
enHash: prepared.currentEnHash,
|
|
2559
2830
|
frontmatter: payload.frontmatter,
|
|
2560
2831
|
body: payload.body
|
|
2561
2832
|
},
|
|
@@ -2568,24 +2839,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2568
2839
|
slug,
|
|
2569
2840
|
frontmatter: validated.frontmatter,
|
|
2570
2841
|
body: translatedBody,
|
|
2571
|
-
enHash: currentEnHash,
|
|
2842
|
+
enHash: prepared.currentEnHash,
|
|
2572
2843
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2573
|
-
model:
|
|
2844
|
+
model: output.model,
|
|
2574
2845
|
snapshotId
|
|
2575
2846
|
});
|
|
2576
2847
|
writeDb.close();
|
|
2577
2848
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2578
|
-
normalizeGeminiDisplayName(
|
|
2579
|
-
|
|
2580
|
-
|
|
2849
|
+
normalizeGeminiDisplayName(output.model),
|
|
2850
|
+
output.usage.inputTokens,
|
|
2851
|
+
output.usage.outputTokens,
|
|
2852
|
+
options.costMode
|
|
2581
2853
|
);
|
|
2582
2854
|
return {
|
|
2583
2855
|
...base,
|
|
2584
2856
|
skipped: false,
|
|
2585
|
-
model:
|
|
2586
|
-
usage:
|
|
2857
|
+
model: output.model,
|
|
2858
|
+
usage: output.usage,
|
|
2587
2859
|
estimatedCostUsd,
|
|
2588
|
-
durationMs: Date.now() - startedAt,
|
|
2860
|
+
durationMs: Date.now() - options.startedAt,
|
|
2589
2861
|
slugAdjusted,
|
|
2590
2862
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2591
2863
|
};
|
|
@@ -2595,33 +2867,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2595
2867
|
skipped: false,
|
|
2596
2868
|
failed: true,
|
|
2597
2869
|
error: formatTranslateError(error),
|
|
2870
|
+
durationMs: Date.now() - options.startedAt
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
function displayModelFor(prepared) {
|
|
2875
|
+
return normalizeGeminiDisplayName(
|
|
2876
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
2877
|
+
);
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
// src/translate/batch-worklist.ts
|
|
2881
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
2882
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
2883
|
+
var INITIAL_POLL_MS = 5e3;
|
|
2884
|
+
var MAX_POLL_MS = 3e4;
|
|
2885
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
2886
|
+
function sleep(ms) {
|
|
2887
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2888
|
+
}
|
|
2889
|
+
function planBatchJobs(entries) {
|
|
2890
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
2891
|
+
for (const entry of entries) {
|
|
2892
|
+
const group = byModel.get(entry.apiModel);
|
|
2893
|
+
if (group) group.push(entry);
|
|
2894
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
2895
|
+
}
|
|
2896
|
+
const plans = [];
|
|
2897
|
+
for (const [apiModel, group] of byModel) {
|
|
2898
|
+
let current = [];
|
|
2899
|
+
let currentBytes = 0;
|
|
2900
|
+
for (const entry of group) {
|
|
2901
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
2902
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
2903
|
+
plans.push({ apiModel, entries: current });
|
|
2904
|
+
current = [];
|
|
2905
|
+
currentBytes = 0;
|
|
2906
|
+
}
|
|
2907
|
+
current.push(entry);
|
|
2908
|
+
currentBytes += size;
|
|
2909
|
+
}
|
|
2910
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
2911
|
+
}
|
|
2912
|
+
return plans;
|
|
2913
|
+
}
|
|
2914
|
+
function readPendingBatchWork(config) {
|
|
2915
|
+
const db = openStore(config, "readwrite");
|
|
2916
|
+
const jobs = listPendingBatchJobs(db);
|
|
2917
|
+
const pendingItems = listPendingBatchItems(db);
|
|
2918
|
+
db.close();
|
|
2919
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
2920
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
2921
|
+
}
|
|
2922
|
+
async function submitBatchJobPlan(config, input) {
|
|
2923
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
2924
|
+
const job = await createGeminiBatchJob({
|
|
2925
|
+
model: plan.apiModel,
|
|
2926
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
2927
|
+
contents: prepared.prompt,
|
|
2928
|
+
config: buildGeminiRequestConfig({
|
|
2929
|
+
apiModelId: plan.apiModel,
|
|
2930
|
+
responseSchema: prepared.responseSchema
|
|
2931
|
+
})
|
|
2932
|
+
})),
|
|
2933
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
2934
|
+
});
|
|
2935
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2936
|
+
const db = openStore(config, "readwrite");
|
|
2937
|
+
const jobId = insertBatchJob(db, {
|
|
2938
|
+
jobName: job.name,
|
|
2939
|
+
model: plan.apiModel,
|
|
2940
|
+
displayModel,
|
|
2941
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2942
|
+
createdAt
|
|
2943
|
+
});
|
|
2944
|
+
insertBatchItems(
|
|
2945
|
+
db,
|
|
2946
|
+
jobId,
|
|
2947
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
2948
|
+
requestIndex,
|
|
2949
|
+
contentType: prepared.item.contentType,
|
|
2950
|
+
enSlug: prepared.item.enSlug,
|
|
2951
|
+
locale: prepared.item.locale,
|
|
2952
|
+
enHash: prepared.currentEnHash,
|
|
2953
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
2954
|
+
// on the EN files still matching (or existing) on disk.
|
|
2955
|
+
snapshotId: recordEnSnapshot(
|
|
2956
|
+
config,
|
|
2957
|
+
{
|
|
2958
|
+
contentType: prepared.item.contentType,
|
|
2959
|
+
enSlug: prepared.item.enSlug,
|
|
2960
|
+
enHash: prepared.currentEnHash,
|
|
2961
|
+
frontmatter: prepared.payload.frontmatter,
|
|
2962
|
+
body: prepared.payload.body
|
|
2963
|
+
},
|
|
2964
|
+
db
|
|
2965
|
+
)
|
|
2966
|
+
}))
|
|
2967
|
+
);
|
|
2968
|
+
const row = {
|
|
2969
|
+
id: jobId,
|
|
2970
|
+
job_name: job.name,
|
|
2971
|
+
model: plan.apiModel,
|
|
2972
|
+
display_model: displayModel,
|
|
2973
|
+
created_at: createdAt,
|
|
2974
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2975
|
+
completed_at: null
|
|
2976
|
+
};
|
|
2977
|
+
db.close();
|
|
2978
|
+
input.onProgress?.({
|
|
2979
|
+
type: "batch-submitted",
|
|
2980
|
+
name: job.name,
|
|
2981
|
+
count: plan.entries.length,
|
|
2982
|
+
model: displayModel,
|
|
2983
|
+
jobIndex,
|
|
2984
|
+
jobCount,
|
|
2985
|
+
createdAt
|
|
2986
|
+
});
|
|
2987
|
+
return row;
|
|
2988
|
+
}
|
|
2989
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
2990
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
2991
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
2992
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
2993
|
+
if (!enDoc) {
|
|
2994
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
2995
|
+
if (!snapshot) {
|
|
2996
|
+
throw new Error(
|
|
2997
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
2998
|
+
);
|
|
2999
|
+
}
|
|
3000
|
+
enDoc = {
|
|
3001
|
+
slug: itemRow.en_slug,
|
|
3002
|
+
enSlug: itemRow.en_slug,
|
|
3003
|
+
locale: config.defaultLocale,
|
|
3004
|
+
noindex: false,
|
|
3005
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3006
|
+
content: snapshot.body
|
|
3007
|
+
};
|
|
3008
|
+
}
|
|
3009
|
+
const item = {
|
|
3010
|
+
contentType: itemRow.content_type,
|
|
3011
|
+
enSlug: itemRow.en_slug,
|
|
3012
|
+
locale: itemRow.locale,
|
|
3013
|
+
reason: "missing",
|
|
3014
|
+
currentEnHash: itemRow.en_hash
|
|
3015
|
+
};
|
|
3016
|
+
return {
|
|
3017
|
+
item,
|
|
3018
|
+
type,
|
|
3019
|
+
enDoc,
|
|
3020
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3021
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3022
|
+
currentEnHash: itemRow.en_hash,
|
|
3023
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3024
|
+
model: jobRow.display_model,
|
|
3025
|
+
prompt: "",
|
|
3026
|
+
responseSchema: void 0
|
|
3027
|
+
};
|
|
3028
|
+
}
|
|
3029
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3030
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3031
|
+
const db = openStore(config, "readwrite");
|
|
3032
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3033
|
+
db.close();
|
|
3034
|
+
return null;
|
|
3035
|
+
}
|
|
3036
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3037
|
+
const results = [];
|
|
3038
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3039
|
+
const result = {
|
|
3040
|
+
contentType: itemRow.content_type,
|
|
3041
|
+
enSlug: itemRow.en_slug,
|
|
3042
|
+
locale: itemRow.locale,
|
|
3043
|
+
skipped: false,
|
|
3044
|
+
failed: true,
|
|
3045
|
+
error: formatTranslateError(new Error(message)),
|
|
2598
3046
|
durationMs: Date.now() - startedAt
|
|
2599
3047
|
};
|
|
3048
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3049
|
+
results.push(result);
|
|
3050
|
+
onResult?.(result);
|
|
3051
|
+
};
|
|
3052
|
+
if (isSuccessfulBatchState(state)) {
|
|
3053
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3054
|
+
for (const itemRow of pendingItems) {
|
|
3055
|
+
const startedAt = Date.now();
|
|
3056
|
+
const inlined = responses[itemRow.request_index];
|
|
3057
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3058
|
+
failItem(
|
|
3059
|
+
itemRow,
|
|
3060
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3061
|
+
startedAt
|
|
3062
|
+
);
|
|
3063
|
+
continue;
|
|
3064
|
+
}
|
|
3065
|
+
let result;
|
|
3066
|
+
try {
|
|
3067
|
+
const response = inlined.response;
|
|
3068
|
+
const raw = textFromBatchResponse(response);
|
|
3069
|
+
const parsed = parseGeminiResponse(raw);
|
|
3070
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3071
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3072
|
+
}
|
|
3073
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3074
|
+
result = finalizeTranslation(
|
|
3075
|
+
config,
|
|
3076
|
+
prepared,
|
|
3077
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3078
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3079
|
+
);
|
|
3080
|
+
} catch (error) {
|
|
3081
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3082
|
+
continue;
|
|
3083
|
+
}
|
|
3084
|
+
updateBatchItemStatus(
|
|
3085
|
+
db,
|
|
3086
|
+
jobRow.id,
|
|
3087
|
+
itemRow.request_index,
|
|
3088
|
+
result.failed ? "failed" : "done",
|
|
3089
|
+
result.error
|
|
3090
|
+
);
|
|
3091
|
+
results.push(result);
|
|
3092
|
+
onResult?.(result);
|
|
3093
|
+
}
|
|
3094
|
+
} else {
|
|
3095
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3096
|
+
const startedAt = Date.now();
|
|
3097
|
+
for (const itemRow of pendingItems) {
|
|
3098
|
+
failItem(itemRow, message, startedAt);
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
db.close();
|
|
3102
|
+
return results;
|
|
3103
|
+
}
|
|
3104
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3105
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3106
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3107
|
+
const loopStartedAt = Date.now();
|
|
3108
|
+
let delay = INITIAL_POLL_MS;
|
|
3109
|
+
let firstRound = true;
|
|
3110
|
+
const elapsedFor = (row) => {
|
|
3111
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3112
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3113
|
+
};
|
|
3114
|
+
while (active.length > 0) {
|
|
3115
|
+
if (!firstRound) {
|
|
3116
|
+
await sleep(delay);
|
|
3117
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3118
|
+
}
|
|
3119
|
+
firstRound = false;
|
|
3120
|
+
for (const tracked of [...active]) {
|
|
3121
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3122
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3123
|
+
options.onProgress?.({
|
|
3124
|
+
type: "batch-polling",
|
|
3125
|
+
name: tracked.row.job_name,
|
|
3126
|
+
state,
|
|
3127
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3128
|
+
jobIndex: tracked.jobIndex,
|
|
3129
|
+
jobCount
|
|
3130
|
+
});
|
|
3131
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3132
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3133
|
+
const results = ingestBatchJob(
|
|
3134
|
+
config,
|
|
3135
|
+
tracked.row,
|
|
3136
|
+
batchJob,
|
|
3137
|
+
options.onResult
|
|
3138
|
+
);
|
|
3139
|
+
if (results === null) {
|
|
3140
|
+
options.onProgress?.({
|
|
3141
|
+
type: "batch-done",
|
|
3142
|
+
name: tracked.row.job_name,
|
|
3143
|
+
state,
|
|
3144
|
+
model: tracked.row.display_model,
|
|
3145
|
+
count: 0,
|
|
3146
|
+
jobIndex: tracked.jobIndex,
|
|
3147
|
+
jobCount,
|
|
3148
|
+
translated: 0,
|
|
3149
|
+
failed: 0,
|
|
3150
|
+
inputTokens: 0,
|
|
3151
|
+
outputTokens: 0,
|
|
3152
|
+
estimatedCostUsd: 0,
|
|
3153
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3154
|
+
alreadyIngested: true
|
|
3155
|
+
});
|
|
3156
|
+
continue;
|
|
3157
|
+
}
|
|
3158
|
+
options.onProgress?.({
|
|
3159
|
+
type: "batch-done",
|
|
3160
|
+
name: tracked.row.job_name,
|
|
3161
|
+
state,
|
|
3162
|
+
model: tracked.row.display_model,
|
|
3163
|
+
count: results.length,
|
|
3164
|
+
jobIndex: tracked.jobIndex,
|
|
3165
|
+
jobCount,
|
|
3166
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3167
|
+
failed: results.filter((r) => r.failed).length,
|
|
3168
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3169
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3170
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3171
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3172
|
+
});
|
|
3173
|
+
}
|
|
3174
|
+
}
|
|
2600
3175
|
}
|
|
2601
3176
|
}
|
|
3177
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3178
|
+
return {
|
|
3179
|
+
...baseForItem(item),
|
|
3180
|
+
skipped: false,
|
|
3181
|
+
failed: true,
|
|
3182
|
+
error: formatTranslateError(error),
|
|
3183
|
+
durationMs: Date.now() - startedAt
|
|
3184
|
+
};
|
|
3185
|
+
}
|
|
3186
|
+
|
|
3187
|
+
// src/translate/page-translator.ts
|
|
3188
|
+
async function runPool(items, concurrency, worker) {
|
|
3189
|
+
if (items.length === 0) return;
|
|
3190
|
+
let nextIndex = 0;
|
|
3191
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3192
|
+
while (nextIndex < items.length) {
|
|
3193
|
+
const index = nextIndex++;
|
|
3194
|
+
await worker(items[index], index);
|
|
3195
|
+
}
|
|
3196
|
+
});
|
|
3197
|
+
await Promise.all(runners);
|
|
3198
|
+
}
|
|
2602
3199
|
function labelForItem(item) {
|
|
2603
3200
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2604
3201
|
}
|
|
3202
|
+
async function translatePage(config, item, options = {}) {
|
|
3203
|
+
const startedAt = Date.now();
|
|
3204
|
+
let outcome;
|
|
3205
|
+
try {
|
|
3206
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3207
|
+
} catch (error) {
|
|
3208
|
+
return failedResultForItem(item, error, startedAt);
|
|
3209
|
+
}
|
|
3210
|
+
if (outcome.status === "done") return outcome.result;
|
|
3211
|
+
const prepared = outcome.prepared;
|
|
3212
|
+
if (options.dryRun) {
|
|
3213
|
+
return {
|
|
3214
|
+
...baseForItem(item),
|
|
3215
|
+
skipped: false,
|
|
3216
|
+
model: prepared.model,
|
|
3217
|
+
durationMs: Date.now() - startedAt
|
|
3218
|
+
};
|
|
3219
|
+
}
|
|
3220
|
+
try {
|
|
3221
|
+
const result = await translatePageWithGemini({
|
|
3222
|
+
prompt: prepared.prompt,
|
|
3223
|
+
model: prepared.model,
|
|
3224
|
+
responseSchema: prepared.responseSchema
|
|
3225
|
+
});
|
|
3226
|
+
return finalizeTranslation(
|
|
3227
|
+
config,
|
|
3228
|
+
prepared,
|
|
3229
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3230
|
+
{ costMode: "interactive", startedAt }
|
|
3231
|
+
);
|
|
3232
|
+
} catch (error) {
|
|
3233
|
+
return failedResultForItem(item, error, startedAt);
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
function createResultCollector(items, onProgress) {
|
|
3237
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3238
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3239
|
+
const slots = new Array(items.length);
|
|
3240
|
+
const extras = [];
|
|
3241
|
+
return {
|
|
3242
|
+
emit(result) {
|
|
3243
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3244
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3245
|
+
else extras.push(result);
|
|
3246
|
+
onProgress?.({ type: "item-done", result });
|
|
3247
|
+
},
|
|
3248
|
+
assemble() {
|
|
3249
|
+
return [
|
|
3250
|
+
...slots.filter((result) => result !== void 0),
|
|
3251
|
+
...extras
|
|
3252
|
+
];
|
|
3253
|
+
}
|
|
3254
|
+
};
|
|
3255
|
+
}
|
|
2605
3256
|
async function translateWorklist(config, items, options = {}) {
|
|
3257
|
+
const mode = options.mode ?? "batch";
|
|
2606
3258
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2607
3259
|
const startedAt = Date.now();
|
|
2608
|
-
|
|
2609
|
-
|
|
3260
|
+
if (options.dryRun) {
|
|
3261
|
+
options.onProgress?.({
|
|
3262
|
+
type: "start",
|
|
3263
|
+
total: items.length,
|
|
3264
|
+
concurrency,
|
|
3265
|
+
dryRun: true,
|
|
3266
|
+
model: options.model,
|
|
3267
|
+
mode
|
|
3268
|
+
});
|
|
3269
|
+
const results2 = items.map((item) => {
|
|
3270
|
+
const itemStartedAt = Date.now();
|
|
3271
|
+
try {
|
|
3272
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3273
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3274
|
+
...baseForItem(item),
|
|
3275
|
+
skipped: false,
|
|
3276
|
+
model: outcome.prepared.model,
|
|
3277
|
+
durationMs: Date.now() - itemStartedAt
|
|
3278
|
+
};
|
|
3279
|
+
} catch (error) {
|
|
3280
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3281
|
+
}
|
|
3282
|
+
});
|
|
3283
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3284
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3285
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3286
|
+
return results2;
|
|
3287
|
+
}
|
|
3288
|
+
const pending = readPendingBatchWork(config);
|
|
3289
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3290
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3291
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3292
|
+
).length;
|
|
2610
3293
|
options.onProgress?.({
|
|
2611
3294
|
type: "start",
|
|
2612
|
-
total: items.length,
|
|
3295
|
+
total: items.length + resumedExtraCount,
|
|
2613
3296
|
concurrency,
|
|
2614
|
-
dryRun:
|
|
2615
|
-
model: options.model
|
|
3297
|
+
dryRun: false,
|
|
3298
|
+
model: options.model,
|
|
3299
|
+
mode
|
|
2616
3300
|
});
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
3301
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3302
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3303
|
+
if (mode === "direct") {
|
|
3304
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3305
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3306
|
+
const active = /* @__PURE__ */ new Set();
|
|
3307
|
+
await Promise.all([
|
|
3308
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3309
|
+
const label = labelForItem(item);
|
|
3310
|
+
active.add(label);
|
|
3311
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3312
|
+
const result = await translatePage(config, item, options);
|
|
3313
|
+
active.delete(label);
|
|
3314
|
+
collector.emit(result);
|
|
3315
|
+
}),
|
|
3316
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3317
|
+
onProgress: options.onProgress,
|
|
3318
|
+
onResult: collector.emit
|
|
3319
|
+
}) : Promise.resolve()
|
|
3320
|
+
]);
|
|
3321
|
+
} else {
|
|
3322
|
+
const entries = [];
|
|
3323
|
+
for (const item of workItems) {
|
|
3324
|
+
const itemStartedAt = Date.now();
|
|
3325
|
+
try {
|
|
3326
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3327
|
+
if (outcome.status === "done") {
|
|
3328
|
+
collector.emit(outcome.result);
|
|
3329
|
+
} else {
|
|
3330
|
+
entries.push({
|
|
3331
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3332
|
+
prompt: outcome.prepared.prompt,
|
|
3333
|
+
prepared: outcome.prepared
|
|
3334
|
+
});
|
|
3335
|
+
}
|
|
3336
|
+
} catch (error) {
|
|
3337
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3338
|
+
}
|
|
3339
|
+
}
|
|
3340
|
+
const plans = planBatchJobs(entries);
|
|
3341
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3342
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3343
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3344
|
+
const submitted = await Promise.all(
|
|
3345
|
+
plans.map(async (plan, planIndex) => {
|
|
3346
|
+
try {
|
|
3347
|
+
return await submitBatchJobPlan(config, {
|
|
3348
|
+
plan,
|
|
3349
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3350
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3351
|
+
jobCount,
|
|
3352
|
+
onProgress: options.onProgress
|
|
3353
|
+
});
|
|
3354
|
+
} catch (error) {
|
|
3355
|
+
for (const entry of plan.entries) {
|
|
3356
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3357
|
+
}
|
|
3358
|
+
return void 0;
|
|
3359
|
+
}
|
|
3360
|
+
})
|
|
3361
|
+
);
|
|
3362
|
+
const jobsToPoll = [
|
|
3363
|
+
...pending.jobs,
|
|
3364
|
+
...submitted.filter((row) => row !== void 0)
|
|
3365
|
+
];
|
|
3366
|
+
if (jobsToPoll.length > 0) {
|
|
3367
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3368
|
+
jobCount,
|
|
3369
|
+
onProgress: options.onProgress,
|
|
3370
|
+
onResult: collector.emit
|
|
3371
|
+
});
|
|
3372
|
+
}
|
|
3373
|
+
}
|
|
3374
|
+
const results = collector.assemble();
|
|
3375
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3376
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3377
|
+
return results;
|
|
3378
|
+
}
|
|
3379
|
+
function countPendingItems(config, jobs) {
|
|
3380
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3381
|
+
const db = openStore(config, "readonly");
|
|
3382
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3383
|
+
for (const job of jobs) {
|
|
3384
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3385
|
+
}
|
|
3386
|
+
db.close();
|
|
3387
|
+
return counts;
|
|
3388
|
+
}
|
|
3389
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3390
|
+
jobs.forEach((job, jobIndex) => {
|
|
3391
|
+
onProgress?.({
|
|
3392
|
+
type: "batch-submitted",
|
|
3393
|
+
name: job.job_name,
|
|
3394
|
+
count: counts.get(job.id) ?? 0,
|
|
3395
|
+
model: job.display_model,
|
|
3396
|
+
jobIndex,
|
|
3397
|
+
jobCount,
|
|
3398
|
+
resumed: true,
|
|
3399
|
+
createdAt: job.created_at
|
|
3400
|
+
});
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3403
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3404
|
+
const startedAt = Date.now();
|
|
3405
|
+
const pending = readPendingBatchWork(config);
|
|
3406
|
+
if (pending.jobs.length === 0) return null;
|
|
3407
|
+
options.onProgress?.({
|
|
3408
|
+
type: "start",
|
|
3409
|
+
total: pending.pendingItems.length,
|
|
3410
|
+
concurrency: 1,
|
|
3411
|
+
dryRun: false,
|
|
3412
|
+
mode: "batch"
|
|
3413
|
+
});
|
|
3414
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3415
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3416
|
+
const results = [];
|
|
3417
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3418
|
+
onProgress: options.onProgress,
|
|
3419
|
+
onResult: (result) => {
|
|
3420
|
+
results.push(result);
|
|
3421
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3422
|
+
}
|
|
2625
3423
|
});
|
|
2626
3424
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2627
3425
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3239,35 +4037,43 @@ async function promptContentType(config) {
|
|
|
3239
4037
|
})
|
|
3240
4038
|
);
|
|
3241
4039
|
}
|
|
4040
|
+
var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
|
|
4041
|
+
var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
|
|
4042
|
+
async function promptManualLocales(config) {
|
|
4043
|
+
const locales = nonDefaultLocales(config);
|
|
4044
|
+
if (locales.length <= 1) return {};
|
|
4045
|
+
const picked = await runPrompt(
|
|
4046
|
+
checkbox({
|
|
4047
|
+
message: "Locales to translate",
|
|
4048
|
+
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
4049
|
+
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
4050
|
+
})
|
|
4051
|
+
);
|
|
4052
|
+
return picked.length === locales.length ? {} : { locale: picked };
|
|
4053
|
+
}
|
|
3242
4054
|
async function promptLocalePreset(config) {
|
|
3243
4055
|
const presets = Object.entries(config.localePresets ?? {}).filter(
|
|
3244
4056
|
(entry) => Array.isArray(entry[1])
|
|
3245
4057
|
);
|
|
3246
4058
|
if (presets.length === 0) {
|
|
3247
|
-
|
|
3248
|
-
if (locales.length <= 1) return {};
|
|
3249
|
-
const picked = await runPrompt(
|
|
3250
|
-
checkbox({
|
|
3251
|
-
message: "Locales to translate",
|
|
3252
|
-
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
3253
|
-
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
3254
|
-
})
|
|
3255
|
-
);
|
|
3256
|
-
return picked.length === locales.length ? {} : { locale: picked };
|
|
4059
|
+
return promptManualLocales(config);
|
|
3257
4060
|
}
|
|
3258
4061
|
const choice = await runPrompt(
|
|
3259
4062
|
select({
|
|
3260
4063
|
message: "Locale preset",
|
|
3261
4064
|
choices: [
|
|
3262
|
-
{ name: "All locales", value:
|
|
4065
|
+
{ name: "All locales", value: ALL_LOCALES },
|
|
3263
4066
|
...presets.map(([name, locales]) => ({
|
|
3264
4067
|
name: `${name} (${locales.join(", ")})`,
|
|
3265
4068
|
value: name
|
|
3266
|
-
}))
|
|
4069
|
+
})),
|
|
4070
|
+
{ name: "Choose locales manually\u2026", value: PICK_LOCALES }
|
|
3267
4071
|
]
|
|
3268
4072
|
})
|
|
3269
4073
|
);
|
|
3270
|
-
|
|
4074
|
+
if (choice === PICK_LOCALES) return promptManualLocales(config);
|
|
4075
|
+
if (choice === ALL_LOCALES) return {};
|
|
4076
|
+
return { preset: choice };
|
|
3271
4077
|
}
|
|
3272
4078
|
async function promptStrategy() {
|
|
3273
4079
|
return runPrompt(
|
|
@@ -3281,12 +4087,25 @@ async function promptStrategy() {
|
|
|
3281
4087
|
})
|
|
3282
4088
|
);
|
|
3283
4089
|
}
|
|
4090
|
+
async function promptMode() {
|
|
4091
|
+
return runPrompt(
|
|
4092
|
+
select({
|
|
4093
|
+
message: "Translation mode",
|
|
4094
|
+
choices: [
|
|
4095
|
+
{ name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
|
|
4096
|
+
{ name: "Direct \u2014 immediate results, full price", value: "direct" }
|
|
4097
|
+
],
|
|
4098
|
+
default: "batch"
|
|
4099
|
+
})
|
|
4100
|
+
);
|
|
4101
|
+
}
|
|
3284
4102
|
async function promptTranslateSelection(config, flags) {
|
|
3285
4103
|
const selection = {
|
|
3286
4104
|
contentType: flags.type,
|
|
3287
4105
|
preset: flags.preset,
|
|
3288
4106
|
locale: flags.locale,
|
|
3289
|
-
strategy: flags.strategy
|
|
4107
|
+
strategy: flags.strategy,
|
|
4108
|
+
mode: flags.mode
|
|
3290
4109
|
};
|
|
3291
4110
|
if (!isInteractive()) return selection;
|
|
3292
4111
|
if (!selection.contentType) {
|
|
@@ -3298,6 +4117,9 @@ async function promptTranslateSelection(config, flags) {
|
|
|
3298
4117
|
if (!selection.strategy) {
|
|
3299
4118
|
selection.strategy = await promptStrategy();
|
|
3300
4119
|
}
|
|
4120
|
+
if (!selection.mode) {
|
|
4121
|
+
selection.mode = await promptMode();
|
|
4122
|
+
}
|
|
3301
4123
|
return selection;
|
|
3302
4124
|
}
|
|
3303
4125
|
|
|
@@ -3319,6 +4141,34 @@ function progressBar(ratio, width = 28) {
|
|
|
3319
4141
|
function labelForResult(result) {
|
|
3320
4142
|
return `${result.contentType}/${result.enSlug}@${result.locale}`;
|
|
3321
4143
|
}
|
|
4144
|
+
function shortJobId(name) {
|
|
4145
|
+
const segments = name.split("/");
|
|
4146
|
+
return segments[segments.length - 1] || name;
|
|
4147
|
+
}
|
|
4148
|
+
function formatElapsed(elapsedMs) {
|
|
4149
|
+
return `${Math.round(elapsedMs / 1e3)}s`;
|
|
4150
|
+
}
|
|
4151
|
+
function batchDoneParts(event) {
|
|
4152
|
+
if (event.alreadyIngested) {
|
|
4153
|
+
return [
|
|
4154
|
+
`job ${shortJobId(event.name)}`,
|
|
4155
|
+
event.model ?? "?",
|
|
4156
|
+
"already ingested by another scribe run (results saved there)",
|
|
4157
|
+
formatElapsed(event.elapsedMs)
|
|
4158
|
+
].join(" \xB7 ");
|
|
4159
|
+
}
|
|
4160
|
+
const parts = [
|
|
4161
|
+
`job ${shortJobId(event.name)}`,
|
|
4162
|
+
event.model ?? "?",
|
|
4163
|
+
`${event.count} request${event.count === 1 ? "" : "s"}`,
|
|
4164
|
+
`${event.translated} translated`
|
|
4165
|
+
];
|
|
4166
|
+
if (event.failed > 0) parts.push(`${event.failed} failed`);
|
|
4167
|
+
parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
|
|
4168
|
+
parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
|
|
4169
|
+
parts.push(formatElapsed(event.elapsedMs));
|
|
4170
|
+
return parts.join(" \xB7 ");
|
|
4171
|
+
}
|
|
3322
4172
|
function statusForResult(result, dryRun) {
|
|
3323
4173
|
if (result.failed) return red("failed");
|
|
3324
4174
|
if (result.skipped) return dim("skipped");
|
|
@@ -3364,6 +4214,25 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3364
4214
|
if (!enabled) {
|
|
3365
4215
|
return {
|
|
3366
4216
|
onEvent(event) {
|
|
4217
|
+
if (event.type === "batch-submitted") {
|
|
4218
|
+
const verb = event.resumed ? "Resuming" : "Submitted";
|
|
4219
|
+
console.log(
|
|
4220
|
+
`${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
|
|
4221
|
+
);
|
|
4222
|
+
return;
|
|
4223
|
+
}
|
|
4224
|
+
if (event.type === "batch-polling") {
|
|
4225
|
+
console.log(
|
|
4226
|
+
`batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
|
|
4227
|
+
);
|
|
4228
|
+
return;
|
|
4229
|
+
}
|
|
4230
|
+
if (event.type === "batch-done") {
|
|
4231
|
+
const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
|
|
4232
|
+
if (event.failed > 0 && event.translated === 0) console.error(line);
|
|
4233
|
+
else console.log(line);
|
|
4234
|
+
return;
|
|
4235
|
+
}
|
|
3367
4236
|
if (event.type === "item-done") {
|
|
3368
4237
|
const label = labelForResult(event.result);
|
|
3369
4238
|
const status = statusForResult(event.result, dryRun);
|
|
@@ -3394,6 +4263,9 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3394
4263
|
let outputTokens = 0;
|
|
3395
4264
|
let estimatedCostUsd = 0;
|
|
3396
4265
|
let active = [];
|
|
4266
|
+
let mode;
|
|
4267
|
+
const batchJobs = /* @__PURE__ */ new Map();
|
|
4268
|
+
let batchElapsedMs = 0;
|
|
3397
4269
|
const recent = [];
|
|
3398
4270
|
let renderedLines = 0;
|
|
3399
4271
|
let cursorHidden = false;
|
|
@@ -3409,6 +4281,15 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3409
4281
|
cursorHidden = false;
|
|
3410
4282
|
}
|
|
3411
4283
|
}
|
|
4284
|
+
function printPersistent(lines) {
|
|
4285
|
+
if (renderedLines > 0) {
|
|
4286
|
+
process.stdout.write(`\x1B[${renderedLines}A`);
|
|
4287
|
+
}
|
|
4288
|
+
for (const line of lines) {
|
|
4289
|
+
process.stdout.write("\x1B[2K" + line + "\n");
|
|
4290
|
+
}
|
|
4291
|
+
renderedLines = 0;
|
|
4292
|
+
}
|
|
3412
4293
|
function render() {
|
|
3413
4294
|
hideCursor();
|
|
3414
4295
|
if (renderedLines > 0) {
|
|
@@ -3416,12 +4297,34 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3416
4297
|
}
|
|
3417
4298
|
const lines = [];
|
|
3418
4299
|
const title = dryRun ? "Dry run" : "Translating";
|
|
3419
|
-
|
|
4300
|
+
const isBatch = mode === "batch" || batchJobs.size > 0;
|
|
4301
|
+
lines.push(
|
|
4302
|
+
cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
|
|
4303
|
+
);
|
|
3420
4304
|
lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
|
|
3421
4305
|
lines.push(
|
|
3422
4306
|
dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
|
|
3423
4307
|
);
|
|
3424
|
-
if (
|
|
4308
|
+
if (batchJobs.size > 0 && done < total) {
|
|
4309
|
+
const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
|
|
4310
|
+
let running = 0;
|
|
4311
|
+
let finished = 0;
|
|
4312
|
+
let requests = 0;
|
|
4313
|
+
for (const job of batchJobs.values()) {
|
|
4314
|
+
if (terminal.test(job.state)) finished += 1;
|
|
4315
|
+
else running += 1;
|
|
4316
|
+
requests += job.count;
|
|
4317
|
+
}
|
|
4318
|
+
lines.push(
|
|
4319
|
+
dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
|
|
4320
|
+
);
|
|
4321
|
+
for (const [name, job] of batchJobs) {
|
|
4322
|
+
if (terminal.test(job.state)) continue;
|
|
4323
|
+
lines.push(
|
|
4324
|
+
dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
|
|
4325
|
+
);
|
|
4326
|
+
}
|
|
4327
|
+
} else if (active.length > 0) {
|
|
3425
4328
|
lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
|
|
3426
4329
|
} else if (done < total) {
|
|
3427
4330
|
lines.push(dim("Active ") + "starting\u2026");
|
|
@@ -3449,12 +4352,58 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3449
4352
|
total = event.total;
|
|
3450
4353
|
concurrency = event.concurrency;
|
|
3451
4354
|
model = event.model;
|
|
4355
|
+
mode = event.mode;
|
|
3452
4356
|
render();
|
|
3453
4357
|
break;
|
|
3454
4358
|
case "item-start":
|
|
3455
4359
|
active = event.active;
|
|
3456
4360
|
render();
|
|
3457
4361
|
break;
|
|
4362
|
+
case "batch-submitted": {
|
|
4363
|
+
const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
|
|
4364
|
+
const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
|
|
4365
|
+
batchJobs.set(event.name, {
|
|
4366
|
+
state: event.resumed ? "RESUMED" : "SUBMITTED",
|
|
4367
|
+
count: event.count,
|
|
4368
|
+
model: event.model,
|
|
4369
|
+
createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
|
|
4370
|
+
elapsedMs
|
|
4371
|
+
});
|
|
4372
|
+
const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
|
|
4373
|
+
const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
|
|
4374
|
+
printPersistent([
|
|
4375
|
+
dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
|
|
4376
|
+
]);
|
|
4377
|
+
render();
|
|
4378
|
+
break;
|
|
4379
|
+
}
|
|
4380
|
+
case "batch-polling": {
|
|
4381
|
+
const job = batchJobs.get(event.name);
|
|
4382
|
+
batchJobs.set(event.name, {
|
|
4383
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4384
|
+
count: job?.count ?? 0,
|
|
4385
|
+
model: job?.model,
|
|
4386
|
+
createdAtMs: job?.createdAtMs,
|
|
4387
|
+
elapsedMs: event.elapsedMs
|
|
4388
|
+
});
|
|
4389
|
+
batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
|
|
4390
|
+
render();
|
|
4391
|
+
break;
|
|
4392
|
+
}
|
|
4393
|
+
case "batch-done": {
|
|
4394
|
+
const job = batchJobs.get(event.name);
|
|
4395
|
+
batchJobs.set(event.name, {
|
|
4396
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4397
|
+
count: job?.count ?? event.count,
|
|
4398
|
+
model: job?.model ?? event.model,
|
|
4399
|
+
createdAtMs: job?.createdAtMs,
|
|
4400
|
+
elapsedMs: event.elapsedMs
|
|
4401
|
+
});
|
|
4402
|
+
const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
|
|
4403
|
+
printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
|
|
4404
|
+
render();
|
|
4405
|
+
break;
|
|
4406
|
+
}
|
|
3458
4407
|
case "item-done": {
|
|
3459
4408
|
done += 1;
|
|
3460
4409
|
active = active.filter((label) => label !== labelForResult(event.result));
|
|
@@ -3577,6 +4526,21 @@ function parseArgs(argv) {
|
|
|
3577
4526
|
i++;
|
|
3578
4527
|
continue;
|
|
3579
4528
|
}
|
|
4529
|
+
if (arg === "--batch") {
|
|
4530
|
+
options.batch = true;
|
|
4531
|
+
i++;
|
|
4532
|
+
continue;
|
|
4533
|
+
}
|
|
4534
|
+
if (arg === "--direct") {
|
|
4535
|
+
options.direct = true;
|
|
4536
|
+
i++;
|
|
4537
|
+
continue;
|
|
4538
|
+
}
|
|
4539
|
+
if (arg === "--resume") {
|
|
4540
|
+
options.resume = true;
|
|
4541
|
+
i++;
|
|
4542
|
+
continue;
|
|
4543
|
+
}
|
|
3580
4544
|
if (arg === "--strategy") {
|
|
3581
4545
|
const value = argv[++i];
|
|
3582
4546
|
if (value !== "all" && value !== "missing-only") {
|
|
@@ -3645,11 +4609,19 @@ Translate flags:
|
|
|
3645
4609
|
--locale <code>... Target locale(s); overrides --preset
|
|
3646
4610
|
--slug <en-slug> Single English document
|
|
3647
4611
|
--model <id> Gemini model override
|
|
3648
|
-
--
|
|
4612
|
+
--batch Force Gemini Batch API mode (50% token cost; default)
|
|
4613
|
+
--direct Force direct interactive calls (immediate, full price)
|
|
4614
|
+
--resume Only poll/ingest pending batch jobs; submit nothing new
|
|
4615
|
+
--concurrency <n> Parallel translations in --direct mode (default: 3)
|
|
3649
4616
|
--dry-run List work without writing
|
|
3650
4617
|
--force Re-translate even when hashes match
|
|
3651
4618
|
--strategy <mode> all (default) or missing-only
|
|
3652
4619
|
--no-progress Plain line logging instead of live progress
|
|
4620
|
+
|
|
4621
|
+
Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
|
|
4622
|
+
during polling is safe: the next run (or --resume) picks the jobs back up and
|
|
4623
|
+
ingests their results. In a TTY without --batch/--direct, a prompt asks which
|
|
4624
|
+
mode to use.
|
|
3653
4625
|
`);
|
|
3654
4626
|
return;
|
|
3655
4627
|
}
|
|
@@ -3695,11 +4667,29 @@ Translate flags:
|
|
|
3695
4667
|
break;
|
|
3696
4668
|
}
|
|
3697
4669
|
case "translate": {
|
|
4670
|
+
if (options.batch && options.direct) {
|
|
4671
|
+
throw new Error("--batch and --direct are mutually exclusive");
|
|
4672
|
+
}
|
|
4673
|
+
if (options.resume) {
|
|
4674
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4675
|
+
enabled: !options.noProgress,
|
|
4676
|
+
dryRun: false
|
|
4677
|
+
});
|
|
4678
|
+
const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4679
|
+
reporter2.finish();
|
|
4680
|
+
if (results2 === null) {
|
|
4681
|
+
console.log("No pending translation batch jobs.");
|
|
4682
|
+
break;
|
|
4683
|
+
}
|
|
4684
|
+
if (results2.some((result) => result.failed)) process.exitCode = 1;
|
|
4685
|
+
break;
|
|
4686
|
+
}
|
|
3698
4687
|
const selection = await promptTranslateSelection(config, {
|
|
3699
4688
|
type: options.type,
|
|
3700
4689
|
preset: options.preset,
|
|
3701
4690
|
locale: options.locale,
|
|
3702
|
-
strategy: options.strategy
|
|
4691
|
+
strategy: options.strategy,
|
|
4692
|
+
mode: options.batch ? "batch" : options.direct ? "direct" : void 0
|
|
3703
4693
|
});
|
|
3704
4694
|
const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
|
|
3705
4695
|
const worklist = buildWorklist(config, {
|
|
@@ -3709,6 +4699,18 @@ Translate flags:
|
|
|
3709
4699
|
strategy: selection.strategy ?? "all"
|
|
3710
4700
|
});
|
|
3711
4701
|
if (worklist.length === 0) {
|
|
4702
|
+
if (!options.dryRun) {
|
|
4703
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4704
|
+
enabled: !options.noProgress,
|
|
4705
|
+
dryRun: false
|
|
4706
|
+
});
|
|
4707
|
+
const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4708
|
+
reporter2.finish();
|
|
4709
|
+
if (resumed !== null) {
|
|
4710
|
+
if (resumed.some((result) => result.failed)) process.exitCode = 1;
|
|
4711
|
+
break;
|
|
4712
|
+
}
|
|
4713
|
+
}
|
|
3712
4714
|
console.log("Nothing to translate.");
|
|
3713
4715
|
break;
|
|
3714
4716
|
}
|
|
@@ -3721,6 +4723,7 @@ Translate flags:
|
|
|
3721
4723
|
dryRun: options.dryRun,
|
|
3722
4724
|
force: options.force,
|
|
3723
4725
|
concurrency: options.concurrency,
|
|
4726
|
+
mode: selection.mode,
|
|
3724
4727
|
onProgress: reporter.onEvent
|
|
3725
4728
|
});
|
|
3726
4729
|
reporter.finish();
|