scribe-cms 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1207 -204
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1208 -205
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +969 -166
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +970 -168
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +136 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +136 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +18 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.cjs
CHANGED
|
@@ -76,6 +76,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
function readSchemaVersion(db) {
|
|
79
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
80
|
+
if (!metaTable) return 0;
|
|
79
81
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
80
82
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
81
83
|
}
|
|
@@ -100,7 +102,7 @@ var SCHEMA_VERSION, MIGRATIONS;
|
|
|
100
102
|
var init_sqlite = __esm({
|
|
101
103
|
"src/storage/sqlite.ts"() {
|
|
102
104
|
init_cjs_shims();
|
|
103
|
-
SCHEMA_VERSION =
|
|
105
|
+
SCHEMA_VERSION = 5;
|
|
104
106
|
MIGRATIONS = [
|
|
105
107
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
106
108
|
key TEXT PRIMARY KEY,
|
|
@@ -148,7 +150,30 @@ var init_sqlite = __esm({
|
|
|
148
150
|
UNIQUE (content_type, en_slug, en_hash)
|
|
149
151
|
)`,
|
|
150
152
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
151
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
153
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
154
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
155
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
156
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
157
|
+
model TEXT NOT NULL,
|
|
158
|
+
display_model TEXT NOT NULL,
|
|
159
|
+
created_at TEXT NOT NULL,
|
|
160
|
+
state TEXT NOT NULL,
|
|
161
|
+
completed_at TEXT
|
|
162
|
+
)`,
|
|
163
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
164
|
+
job_id INTEGER NOT NULL,
|
|
165
|
+
request_index INTEGER NOT NULL,
|
|
166
|
+
content_type TEXT NOT NULL,
|
|
167
|
+
en_slug TEXT NOT NULL,
|
|
168
|
+
locale TEXT NOT NULL,
|
|
169
|
+
en_hash TEXT NOT NULL,
|
|
170
|
+
snapshot_id INTEGER NOT NULL,
|
|
171
|
+
status TEXT NOT NULL,
|
|
172
|
+
error TEXT,
|
|
173
|
+
PRIMARY KEY (job_id, request_index)
|
|
174
|
+
)`,
|
|
175
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
176
|
+
ON translation_batch_items(status)`
|
|
152
177
|
];
|
|
153
178
|
}
|
|
154
179
|
});
|
|
@@ -190,26 +215,6 @@ function getRelationTarget(schema) {
|
|
|
190
215
|
}
|
|
191
216
|
return null;
|
|
192
217
|
}
|
|
193
|
-
function unwrapSchema(schema) {
|
|
194
|
-
let current = schema;
|
|
195
|
-
for (let i = 0; i < 8; i++) {
|
|
196
|
-
const anySchema = current;
|
|
197
|
-
if (typeof anySchema.unwrap === "function") {
|
|
198
|
-
current = anySchema.unwrap();
|
|
199
|
-
continue;
|
|
200
|
-
}
|
|
201
|
-
if (typeof anySchema.removeDefault === "function") {
|
|
202
|
-
current = anySchema.removeDefault();
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
if (anySchema._def?.innerType) {
|
|
206
|
-
current = anySchema._def.innerType;
|
|
207
|
-
continue;
|
|
208
|
-
}
|
|
209
|
-
break;
|
|
210
|
-
}
|
|
211
|
-
return current;
|
|
212
|
-
}
|
|
213
218
|
function peelOptionalWrappers(schema) {
|
|
214
219
|
let current = schema;
|
|
215
220
|
for (let i = 0; i < 8; i++) {
|
|
@@ -355,6 +360,7 @@ function resolveConfig(input, baseDir) {
|
|
|
355
360
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
356
361
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
357
362
|
}
|
|
363
|
+
const localeFallbacks = raw.localeFallbacks === false ? {} : deriveLocaleFallbacks(raw.locales, defaultLocale);
|
|
358
364
|
const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
359
365
|
const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
|
|
360
366
|
const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -384,6 +390,7 @@ function resolveConfig(input, baseDir) {
|
|
|
384
390
|
defaultLocale,
|
|
385
391
|
localeRouting,
|
|
386
392
|
localePresets: raw.localePresets,
|
|
393
|
+
localeFallbacks,
|
|
387
394
|
translate: raw.translate,
|
|
388
395
|
types
|
|
389
396
|
};
|
|
@@ -393,12 +400,37 @@ function resolveConfig(input, baseDir) {
|
|
|
393
400
|
});
|
|
394
401
|
return config;
|
|
395
402
|
}
|
|
403
|
+
function deriveLocaleFallbacks(locales, defaultLocale) {
|
|
404
|
+
const byLowercase = new Map(locales.map((l) => [l.toLowerCase(), l]));
|
|
405
|
+
const fallbacks = {};
|
|
406
|
+
for (const locale of locales) {
|
|
407
|
+
const subtags = locale.split("-");
|
|
408
|
+
if (subtags.length < 2) continue;
|
|
409
|
+
const chain = [];
|
|
410
|
+
for (let end = subtags.length - 1; end >= 1; end--) {
|
|
411
|
+
const prefix = byLowercase.get(subtags.slice(0, end).join("-").toLowerCase());
|
|
412
|
+
if (prefix && prefix !== locale && prefix !== defaultLocale) {
|
|
413
|
+
chain.push(prefix);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (chain.length > 0) {
|
|
417
|
+
fallbacks[locale] = chain;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
return fallbacks;
|
|
421
|
+
}
|
|
396
422
|
|
|
397
423
|
// src/create-project.ts
|
|
398
424
|
init_cjs_shims();
|
|
399
425
|
|
|
400
426
|
// src/core/introspect-schema.ts
|
|
401
427
|
init_cjs_shims();
|
|
428
|
+
function getArrayElement(schema) {
|
|
429
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
430
|
+
return schema.element;
|
|
431
|
+
}
|
|
432
|
+
return null;
|
|
433
|
+
}
|
|
402
434
|
function introspectSchema(schema, prefix = []) {
|
|
403
435
|
const relation = getRelationTarget(schema);
|
|
404
436
|
if (relation && prefix.length > 0) {
|
|
@@ -412,7 +444,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
412
444
|
}
|
|
413
445
|
];
|
|
414
446
|
}
|
|
415
|
-
|
|
447
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
448
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
449
|
+
}
|
|
450
|
+
const base = peelOptionalWrappers(schema);
|
|
416
451
|
if (base instanceof Object && "shape" in base) {
|
|
417
452
|
const shape = base.shape;
|
|
418
453
|
const fields = [];
|
|
@@ -421,9 +456,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
421
456
|
}
|
|
422
457
|
return fields;
|
|
423
458
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
const elementBase =
|
|
459
|
+
const element = getArrayElement(base);
|
|
460
|
+
if (element) {
|
|
461
|
+
const elementBase = peelOptionalWrappers(element);
|
|
427
462
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
428
463
|
const shape = elementBase.shape;
|
|
429
464
|
const fields = [];
|
|
@@ -435,15 +470,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
435
470
|
}
|
|
436
471
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
437
472
|
}
|
|
438
|
-
function
|
|
439
|
-
const
|
|
473
|
+
function buildPathTrie(paths) {
|
|
474
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
440
475
|
for (const path15 of paths) {
|
|
441
|
-
|
|
476
|
+
let node = root;
|
|
477
|
+
for (const segment of path15) {
|
|
478
|
+
let child = node.children.get(segment);
|
|
479
|
+
if (!child) {
|
|
480
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
481
|
+
node.children.set(segment, child);
|
|
482
|
+
}
|
|
483
|
+
node = child;
|
|
484
|
+
}
|
|
485
|
+
node.leaf = true;
|
|
486
|
+
}
|
|
487
|
+
return root;
|
|
488
|
+
}
|
|
489
|
+
function extractNode(data, trie) {
|
|
490
|
+
if (trie.leaf) return data;
|
|
491
|
+
const star = trie.children.get("*");
|
|
492
|
+
if (star) {
|
|
493
|
+
if (!Array.isArray(data)) return void 0;
|
|
494
|
+
return data.map(
|
|
495
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
499
|
+
const record = data;
|
|
500
|
+
const out = {};
|
|
501
|
+
let any = false;
|
|
502
|
+
for (const [key, child] of trie.children) {
|
|
503
|
+
const value = extractNode(record[key], child);
|
|
442
504
|
if (value !== void 0) {
|
|
443
|
-
|
|
505
|
+
out[key] = value;
|
|
506
|
+
any = true;
|
|
444
507
|
}
|
|
445
508
|
}
|
|
446
|
-
return out;
|
|
509
|
+
return any ? out : void 0;
|
|
510
|
+
}
|
|
511
|
+
function extractByPaths(data, paths) {
|
|
512
|
+
if (paths.length === 0) return {};
|
|
513
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
514
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
447
515
|
}
|
|
448
516
|
function pickTranslatable(data, schema) {
|
|
449
517
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -458,32 +526,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
458
526
|
const translatable = pickTranslatable(localeData, schema);
|
|
459
527
|
return deepMerge(structural, translatable);
|
|
460
528
|
}
|
|
461
|
-
function getAtPath(obj, path15) {
|
|
462
|
-
let current = obj;
|
|
463
|
-
for (const segment of path15) {
|
|
464
|
-
if (segment === "*") {
|
|
465
|
-
if (!Array.isArray(current)) return void 0;
|
|
466
|
-
return current.map(
|
|
467
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path15.slice(path15.indexOf("*") + 1)) : void 0
|
|
468
|
-
);
|
|
469
|
-
}
|
|
470
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
471
|
-
current = current[segment];
|
|
472
|
-
}
|
|
473
|
-
return current;
|
|
474
|
-
}
|
|
475
|
-
function setAtPath(obj, path15, value) {
|
|
476
|
-
if (path15.length === 0) return;
|
|
477
|
-
if (path15.length === 1) {
|
|
478
|
-
obj[path15[0]] = value;
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
const [head, ...rest] = path15;
|
|
482
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
483
|
-
obj[head] = {};
|
|
484
|
-
}
|
|
485
|
-
setAtPath(obj[head], rest, value);
|
|
486
|
-
}
|
|
487
529
|
function deepMerge(base, overlay) {
|
|
488
530
|
const out = { ...base };
|
|
489
531
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -774,7 +816,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
774
816
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
775
817
|
const attrStart = i;
|
|
776
818
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
777
|
-
body.slice(attrStart, i);
|
|
819
|
+
const attrName = body.slice(attrStart, i);
|
|
820
|
+
if (!attrName) break;
|
|
778
821
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
779
822
|
if (body[i] !== "=") {
|
|
780
823
|
tagOut += body.slice(attrStart, i);
|
|
@@ -783,10 +826,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
783
826
|
tagOut += body.slice(attrStart, i);
|
|
784
827
|
tagOut += "=";
|
|
785
828
|
i += 1;
|
|
829
|
+
const wsStart = i;
|
|
786
830
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
831
|
+
tagOut += body.slice(wsStart, i);
|
|
787
832
|
const quote = body[i];
|
|
788
833
|
if (quote !== '"' && quote !== "'") {
|
|
789
|
-
|
|
834
|
+
break;
|
|
790
835
|
}
|
|
791
836
|
if (quote === "'") {
|
|
792
837
|
const valStart2 = i + 1;
|
|
@@ -1108,7 +1153,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1108
1153
|
|
|
1109
1154
|
// src/i18n/resolve-document.ts
|
|
1110
1155
|
init_cjs_shims();
|
|
1111
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1156
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1112
1157
|
const urlBuilder = createUrlBuilder({
|
|
1113
1158
|
locales: [defaultLocale, locale],
|
|
1114
1159
|
defaultLocale,
|
|
@@ -1122,15 +1167,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1122
1167
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1123
1168
|
const found = docIdx.bySlug.get(slug);
|
|
1124
1169
|
if (!found) continue;
|
|
1125
|
-
const
|
|
1126
|
-
|
|
1170
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1171
|
+
let target;
|
|
1172
|
+
let targetLocale = locale;
|
|
1173
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1174
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1175
|
+
if (cand) {
|
|
1176
|
+
target = cand;
|
|
1177
|
+
targetLocale = candidateLocale;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
if (target) {
|
|
1182
|
+
if (target.slug === slug) {
|
|
1183
|
+
return { document: target, actualLocale: targetLocale };
|
|
1184
|
+
}
|
|
1127
1185
|
if (!type.path) {
|
|
1128
1186
|
return { document: null, actualLocale: locale };
|
|
1129
1187
|
}
|
|
1130
1188
|
return {
|
|
1131
1189
|
document: null,
|
|
1132
1190
|
actualLocale: locale,
|
|
1133
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1191
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1134
1192
|
};
|
|
1135
1193
|
}
|
|
1136
1194
|
if (docLocale === defaultLocale) {
|
|
@@ -1161,13 +1219,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1161
1219
|
}
|
|
1162
1220
|
return { document: null, actualLocale: locale };
|
|
1163
1221
|
}
|
|
1164
|
-
function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1165
|
-
if (sourceLocale === targetLocale) return document2.slug;
|
|
1166
|
-
const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
|
|
1167
|
-
if (!englishSlug) return null;
|
|
1168
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1169
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1170
|
-
}
|
|
1171
1222
|
|
|
1172
1223
|
// src/create-project.ts
|
|
1173
1224
|
function comparatorFor(orderBy) {
|
|
@@ -1220,7 +1271,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1220
1271
|
config.defaultLocale,
|
|
1221
1272
|
load(),
|
|
1222
1273
|
type,
|
|
1223
|
-
config.localeRouting
|
|
1274
|
+
config.localeRouting,
|
|
1275
|
+
config.localeFallbacks[locale] ?? []
|
|
1224
1276
|
);
|
|
1225
1277
|
if (result.document && type.path) {
|
|
1226
1278
|
return {
|
|
@@ -1242,8 +1294,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1242
1294
|
const params = [];
|
|
1243
1295
|
for (const locale of options.locales ?? config.locales) {
|
|
1244
1296
|
const localeIdx = all.get(locale);
|
|
1297
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1245
1298
|
for (const doc of enIdx.bySlug.values()) {
|
|
1246
|
-
|
|
1299
|
+
let slug;
|
|
1300
|
+
if (locale === config.defaultLocale) {
|
|
1301
|
+
slug = doc.slug;
|
|
1302
|
+
} else {
|
|
1303
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1304
|
+
}
|
|
1247
1305
|
params.push({ locale, slug });
|
|
1248
1306
|
}
|
|
1249
1307
|
}
|
|
@@ -1705,7 +1763,7 @@ init_sqlite();
|
|
|
1705
1763
|
|
|
1706
1764
|
// src/validate/validate-relations.ts
|
|
1707
1765
|
init_cjs_shims();
|
|
1708
|
-
function
|
|
1766
|
+
function getAtPath(obj, fieldPath) {
|
|
1709
1767
|
let current = obj;
|
|
1710
1768
|
for (const segment of fieldPath) {
|
|
1711
1769
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -1750,7 +1808,7 @@ function validateRelations(project) {
|
|
|
1750
1808
|
});
|
|
1751
1809
|
continue;
|
|
1752
1810
|
}
|
|
1753
|
-
const value =
|
|
1811
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
1754
1812
|
if (value === void 0 || value === null || value === "") continue;
|
|
1755
1813
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
1756
1814
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2134,6 +2192,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2134
2192
|
// src/translate/page-translator.ts
|
|
2135
2193
|
init_cjs_shims();
|
|
2136
2194
|
|
|
2195
|
+
// src/storage/batch-jobs.ts
|
|
2196
|
+
init_cjs_shims();
|
|
2197
|
+
function insertBatchJob(db, input) {
|
|
2198
|
+
const info = db.prepare(
|
|
2199
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2200
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2201
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2202
|
+
return Number(info.lastInsertRowid);
|
|
2203
|
+
}
|
|
2204
|
+
function insertBatchItems(db, jobId, items) {
|
|
2205
|
+
const stmt = db.prepare(
|
|
2206
|
+
`INSERT INTO translation_batch_items (
|
|
2207
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2208
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2209
|
+
);
|
|
2210
|
+
for (const item of items) {
|
|
2211
|
+
stmt.run(
|
|
2212
|
+
jobId,
|
|
2213
|
+
item.requestIndex,
|
|
2214
|
+
item.contentType,
|
|
2215
|
+
item.enSlug,
|
|
2216
|
+
item.locale,
|
|
2217
|
+
item.enHash,
|
|
2218
|
+
item.snapshotId
|
|
2219
|
+
);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
function listPendingBatchJobs(db) {
|
|
2223
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2224
|
+
}
|
|
2225
|
+
function listBatchItems(db, jobId) {
|
|
2226
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2227
|
+
}
|
|
2228
|
+
function listPendingBatchItems(db) {
|
|
2229
|
+
return db.prepare(
|
|
2230
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2231
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2232
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2233
|
+
ORDER BY i.job_id, i.request_index`
|
|
2234
|
+
).all();
|
|
2235
|
+
}
|
|
2236
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2237
|
+
const info = db.prepare(
|
|
2238
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2239
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2240
|
+
).run(state, completedAt, jobId);
|
|
2241
|
+
return info.changes > 0;
|
|
2242
|
+
}
|
|
2243
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2244
|
+
db.prepare(
|
|
2245
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2246
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2247
|
+
}
|
|
2248
|
+
|
|
2249
|
+
// src/translate/page-translator.ts
|
|
2250
|
+
init_sqlite();
|
|
2251
|
+
|
|
2252
|
+
// src/translate/batch-worklist.ts
|
|
2253
|
+
init_cjs_shims();
|
|
2254
|
+
|
|
2137
2255
|
// src/history/record-snapshot.ts
|
|
2138
2256
|
init_cjs_shims();
|
|
2139
2257
|
init_sqlite();
|
|
@@ -2151,9 +2269,111 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2151
2269
|
return id;
|
|
2152
2270
|
}
|
|
2153
2271
|
|
|
2154
|
-
// src/translate/
|
|
2272
|
+
// src/translate/batch-worklist.ts
|
|
2155
2273
|
init_sqlite();
|
|
2156
2274
|
|
|
2275
|
+
// src/translate/gemini-batch.ts
|
|
2276
|
+
init_cjs_shims();
|
|
2277
|
+
|
|
2278
|
+
// src/translate/retry.ts
|
|
2279
|
+
init_cjs_shims();
|
|
2280
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2281
|
+
function statusFromError(error) {
|
|
2282
|
+
if (error instanceof genai.ApiError && typeof error.status === "number") {
|
|
2283
|
+
return error.status;
|
|
2284
|
+
}
|
|
2285
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2286
|
+
const status = error.status;
|
|
2287
|
+
if (typeof status === "number") return status;
|
|
2288
|
+
}
|
|
2289
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2290
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2291
|
+
if (match) return Number(match[1]);
|
|
2292
|
+
return void 0;
|
|
2293
|
+
}
|
|
2294
|
+
function isNetworkError(error) {
|
|
2295
|
+
const err = error;
|
|
2296
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2297
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2298
|
+
return true;
|
|
2299
|
+
}
|
|
2300
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2301
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2302
|
+
}
|
|
2303
|
+
function isTransientError(error) {
|
|
2304
|
+
const status = statusFromError(error);
|
|
2305
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2306
|
+
return isNetworkError(error);
|
|
2307
|
+
}
|
|
2308
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2309
|
+
async function withRetry(fn, options = {}) {
|
|
2310
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2311
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2312
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2313
|
+
const random = options.random ?? Math.random;
|
|
2314
|
+
let lastError;
|
|
2315
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2316
|
+
try {
|
|
2317
|
+
return await fn();
|
|
2318
|
+
} catch (error) {
|
|
2319
|
+
lastError = error;
|
|
2320
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2321
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2322
|
+
const delay = backoff + random() * backoff;
|
|
2323
|
+
await sleep2(delay);
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
throw lastError;
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
// src/translate/gemini-batch.ts
|
|
2330
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2331
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2332
|
+
"SUCCEEDED",
|
|
2333
|
+
"FAILED",
|
|
2334
|
+
"CANCELLED",
|
|
2335
|
+
"EXPIRED",
|
|
2336
|
+
"PARTIALLY_SUCCEEDED"
|
|
2337
|
+
]);
|
|
2338
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2339
|
+
function normalizeBatchState(state) {
|
|
2340
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2341
|
+
}
|
|
2342
|
+
function isTerminalBatchState(state) {
|
|
2343
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2344
|
+
}
|
|
2345
|
+
function isSuccessfulBatchState(state) {
|
|
2346
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2347
|
+
}
|
|
2348
|
+
function makeClient(apiKey) {
|
|
2349
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2350
|
+
if (!key) {
|
|
2351
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2352
|
+
}
|
|
2353
|
+
return new genai.GoogleGenAI({ apiKey: key });
|
|
2354
|
+
}
|
|
2355
|
+
function textFromBatchResponse(response) {
|
|
2356
|
+
if (typeof response.text === "string") return response.text;
|
|
2357
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2358
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2359
|
+
}
|
|
2360
|
+
async function createGeminiBatchJob(input) {
|
|
2361
|
+
const ai = makeClient(input.apiKey);
|
|
2362
|
+
const job = await withRetry(
|
|
2363
|
+
() => ai.batches.create({
|
|
2364
|
+
model: input.model,
|
|
2365
|
+
src: input.requests,
|
|
2366
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2367
|
+
})
|
|
2368
|
+
);
|
|
2369
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2370
|
+
return job;
|
|
2371
|
+
}
|
|
2372
|
+
async function getGeminiBatchJob(input) {
|
|
2373
|
+
const ai = makeClient(input.apiKey);
|
|
2374
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2157
2377
|
// src/translate/gemini-client.ts
|
|
2158
2378
|
init_cjs_shims();
|
|
2159
2379
|
|
|
@@ -2178,6 +2398,12 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2178
2398
|
if (alias) return alias[0];
|
|
2179
2399
|
return name;
|
|
2180
2400
|
}
|
|
2401
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2402
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2403
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
|
|
2404
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2405
|
+
return void 0;
|
|
2406
|
+
}
|
|
2181
2407
|
|
|
2182
2408
|
// src/translate/gemini-client.ts
|
|
2183
2409
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
@@ -2191,11 +2417,36 @@ function extractJson(text) {
|
|
|
2191
2417
|
return trimmed;
|
|
2192
2418
|
}
|
|
2193
2419
|
function parseGeminiResponse(text) {
|
|
2420
|
+
let parsed;
|
|
2194
2421
|
try {
|
|
2195
|
-
|
|
2422
|
+
parsed = JSON.parse(text.trim());
|
|
2196
2423
|
} catch {
|
|
2197
|
-
|
|
2424
|
+
parsed = JSON.parse(extractJson(text));
|
|
2425
|
+
}
|
|
2426
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2427
|
+
parsed.frontmatter = {};
|
|
2198
2428
|
}
|
|
2429
|
+
return parsed;
|
|
2430
|
+
}
|
|
2431
|
+
function buildGeminiRequestConfig(input) {
|
|
2432
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2433
|
+
return {
|
|
2434
|
+
responseMimeType: "application/json",
|
|
2435
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2436
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
function usageFromResponse(response) {
|
|
2440
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2441
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2442
|
+
return {
|
|
2443
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2444
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2445
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2446
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2447
|
+
thoughtsTokens,
|
|
2448
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2449
|
+
};
|
|
2199
2450
|
}
|
|
2200
2451
|
async function translatePageWithGemini(input) {
|
|
2201
2452
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2207,28 +2458,28 @@ async function translatePageWithGemini(input) {
|
|
|
2207
2458
|
);
|
|
2208
2459
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2209
2460
|
const ai = new genai.GoogleGenAI({ apiKey });
|
|
2210
|
-
const response = await
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2461
|
+
const response = await withRetry(
|
|
2462
|
+
() => ai.models.generateContent({
|
|
2463
|
+
model: apiModel,
|
|
2464
|
+
contents: input.prompt,
|
|
2465
|
+
config: buildGeminiRequestConfig({
|
|
2466
|
+
apiModelId: apiModel,
|
|
2467
|
+
responseSchema: input.responseSchema
|
|
2468
|
+
})
|
|
2469
|
+
})
|
|
2470
|
+
);
|
|
2218
2471
|
const raw = response.text ?? "";
|
|
2219
2472
|
const parsed = parseGeminiResponse(raw);
|
|
2220
2473
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2221
2474
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2222
2475
|
}
|
|
2223
|
-
|
|
2224
|
-
const usage = {
|
|
2225
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2226
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2227
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2228
|
-
};
|
|
2229
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2476
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2230
2477
|
}
|
|
2231
2478
|
|
|
2479
|
+
// src/translate/translate-core.ts
|
|
2480
|
+
init_cjs_shims();
|
|
2481
|
+
init_sqlite();
|
|
2482
|
+
|
|
2232
2483
|
// src/translate/gemini-pricing.ts
|
|
2233
2484
|
init_cjs_shims();
|
|
2234
2485
|
var CONTEXT_TIER_TOKENS = 2e5;
|
|
@@ -2247,12 +2498,14 @@ function resolveModelPricing(model) {
|
|
|
2247
2498
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2248
2499
|
return match?.[1];
|
|
2249
2500
|
}
|
|
2250
|
-
|
|
2501
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2502
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2251
2503
|
const pricing = resolveModelPricing(model);
|
|
2252
2504
|
if (!pricing) return void 0;
|
|
2253
2505
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2254
2506
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2255
|
-
|
|
2507
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2508
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2256
2509
|
}
|
|
2257
2510
|
function formatTokenCount(tokens) {
|
|
2258
2511
|
if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
|
|
@@ -2280,6 +2533,7 @@ var LOCALE_NAMES = {
|
|
|
2280
2533
|
"zh-CN": "Simplified Chinese",
|
|
2281
2534
|
"zh-TW": "Traditional Chinese",
|
|
2282
2535
|
pt: "Portuguese",
|
|
2536
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2283
2537
|
ja: "Japanese",
|
|
2284
2538
|
ru: "Russian",
|
|
2285
2539
|
it: "Italian",
|
|
@@ -2287,29 +2541,29 @@ var LOCALE_NAMES = {
|
|
|
2287
2541
|
};
|
|
2288
2542
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2289
2543
|
return [
|
|
2290
|
-
`Localize the content
|
|
2544
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2291
2545
|
"Do not translate word-for-word.",
|
|
2292
2546
|
"Preserve the source tone and brand voice.",
|
|
2293
2547
|
"Write as if a native speaker authored it for the target market."
|
|
2294
2548
|
].join(" ");
|
|
2295
2549
|
}
|
|
2550
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2551
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2552
|
+
if (!hasFrontmatter) {
|
|
2553
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2554
|
+
}
|
|
2555
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2556
|
+
}
|
|
2296
2557
|
function buildPageTranslationPrompt(input) {
|
|
2297
2558
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2298
|
-
const
|
|
2299
|
-
const
|
|
2300
|
-
|
|
2559
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2560
|
+
const prefix = [
|
|
2561
|
+
TASK_FRAMING,
|
|
2301
2562
|
"",
|
|
2302
2563
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2303
2564
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2304
2565
|
"## Rules",
|
|
2305
2566
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2306
|
-
...input.slugStrategy === "localized" ? [
|
|
2307
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2308
|
-
] : [],
|
|
2309
|
-
"",
|
|
2310
|
-
"## Output format",
|
|
2311
|
-
"Return ONLY valid JSON with keys:",
|
|
2312
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2313
2567
|
"",
|
|
2314
2568
|
"## EN translatable frontmatter (JSON)",
|
|
2315
2569
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2317,7 +2571,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2317
2571
|
"## EN body (MDX)",
|
|
2318
2572
|
input.enBody
|
|
2319
2573
|
];
|
|
2320
|
-
|
|
2574
|
+
const suffix = [
|
|
2575
|
+
"",
|
|
2576
|
+
"## Target language",
|
|
2577
|
+
localizationPrompt,
|
|
2578
|
+
...input.slugStrategy === "localized" ? [
|
|
2579
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2580
|
+
] : [],
|
|
2581
|
+
"",
|
|
2582
|
+
"## Output format",
|
|
2583
|
+
"Return ONLY valid JSON with keys:",
|
|
2584
|
+
buildOutputFormatLine(
|
|
2585
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2586
|
+
input.slugStrategy
|
|
2587
|
+
)
|
|
2588
|
+
];
|
|
2589
|
+
return [...prefix, ...suffix].join("\n");
|
|
2321
2590
|
}
|
|
2322
2591
|
|
|
2323
2592
|
// src/translate/response-schema.ts
|
|
@@ -2368,9 +2637,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2368
2637
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2369
2638
|
try {
|
|
2370
2639
|
const translatable = buildTranslatableSubschema(schema);
|
|
2371
|
-
if (!translatable) return null;
|
|
2372
2640
|
const responseShape = {
|
|
2373
|
-
frontmatter: translatable,
|
|
2641
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2374
2642
|
body: zod.z.string()
|
|
2375
2643
|
};
|
|
2376
2644
|
if (slugStrategy === "localized") {
|
|
@@ -2451,7 +2719,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2451
2719
|
return { ok: true, frontmatter };
|
|
2452
2720
|
}
|
|
2453
2721
|
|
|
2454
|
-
// src/translate/
|
|
2722
|
+
// src/translate/translate-core.ts
|
|
2723
|
+
function translationItemKey(item) {
|
|
2724
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2725
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2726
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2727
|
+
}
|
|
2455
2728
|
function summarizeResults(results, durationMs) {
|
|
2456
2729
|
return results.reduce(
|
|
2457
2730
|
(totals, result) => {
|
|
@@ -2487,17 +2760,6 @@ function formatTranslateError(error) {
|
|
|
2487
2760
|
}
|
|
2488
2761
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2489
2762
|
}
|
|
2490
|
-
async function runPool(items, concurrency, worker) {
|
|
2491
|
-
if (items.length === 0) return;
|
|
2492
|
-
let nextIndex = 0;
|
|
2493
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2494
|
-
while (nextIndex < items.length) {
|
|
2495
|
-
const index = nextIndex++;
|
|
2496
|
-
await worker(items[index], index);
|
|
2497
|
-
}
|
|
2498
|
-
});
|
|
2499
|
-
await Promise.all(runners);
|
|
2500
|
-
}
|
|
2501
2763
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2502
2764
|
const fm = enDoc.frontmatter;
|
|
2503
2765
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2506,73 +2768,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2506
2768
|
}
|
|
2507
2769
|
return enSlug;
|
|
2508
2770
|
}
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
const base = {
|
|
2771
|
+
function baseForItem(item) {
|
|
2772
|
+
return {
|
|
2512
2773
|
contentType: item.contentType,
|
|
2513
2774
|
enSlug: item.enSlug,
|
|
2514
2775
|
locale: item.locale
|
|
2515
2776
|
};
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2777
|
+
}
|
|
2778
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
2779
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
2780
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
2781
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
2782
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
2783
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
2784
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
2785
|
+
const db = openStore(config, "readonly");
|
|
2786
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
2787
|
+
db.close();
|
|
2788
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
2789
|
+
return {
|
|
2790
|
+
status: "done",
|
|
2791
|
+
result: {
|
|
2792
|
+
...baseForItem(item),
|
|
2529
2793
|
skipped: true,
|
|
2530
2794
|
reason: "fresh",
|
|
2531
2795
|
durationMs: Date.now() - startedAt
|
|
2532
|
-
}
|
|
2533
|
-
}
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2796
|
+
}
|
|
2797
|
+
};
|
|
2798
|
+
}
|
|
2799
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
2800
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
2801
|
+
const prompt = buildPageTranslationPrompt({
|
|
2802
|
+
resolved: resolvedTranslate,
|
|
2803
|
+
targetLocale: item.locale,
|
|
2804
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
2805
|
+
translatableFrontmatter: payload.frontmatter,
|
|
2806
|
+
enBody: payload.body,
|
|
2807
|
+
slugStrategy: type.slugStrategy
|
|
2808
|
+
});
|
|
2809
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
2810
|
+
return {
|
|
2811
|
+
status: "ready",
|
|
2812
|
+
prepared: {
|
|
2813
|
+
item,
|
|
2814
|
+
type,
|
|
2815
|
+
enDoc,
|
|
2816
|
+
payload,
|
|
2817
|
+
currentEnHash,
|
|
2818
|
+
existingSlug: existing?.slug,
|
|
2555
2819
|
model,
|
|
2820
|
+
prompt,
|
|
2556
2821
|
responseSchema: responseSchema ?? void 0
|
|
2557
|
-
}
|
|
2558
|
-
|
|
2822
|
+
}
|
|
2823
|
+
};
|
|
2824
|
+
}
|
|
2825
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
2826
|
+
const { item, type, enDoc, payload } = prepared;
|
|
2827
|
+
const base = baseForItem(item);
|
|
2828
|
+
try {
|
|
2829
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2559
2830
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2560
2831
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2561
2832
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2562
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
2833
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2563
2834
|
if (!validated.ok) {
|
|
2564
2835
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2565
2836
|
}
|
|
2566
2837
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2567
|
-
|
|
2838
|
+
output.parsed.body
|
|
2568
2839
|
);
|
|
2569
2840
|
const writeDb = openStore(config, "readwrite");
|
|
2570
|
-
const snapshotId = recordEnSnapshot(
|
|
2841
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2571
2842
|
config,
|
|
2572
2843
|
{
|
|
2573
2844
|
contentType: type.id,
|
|
2574
2845
|
enSlug: item.enSlug,
|
|
2575
|
-
enHash: currentEnHash,
|
|
2846
|
+
enHash: prepared.currentEnHash,
|
|
2576
2847
|
frontmatter: payload.frontmatter,
|
|
2577
2848
|
body: payload.body
|
|
2578
2849
|
},
|
|
@@ -2585,24 +2856,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2585
2856
|
slug,
|
|
2586
2857
|
frontmatter: validated.frontmatter,
|
|
2587
2858
|
body: translatedBody,
|
|
2588
|
-
enHash: currentEnHash,
|
|
2859
|
+
enHash: prepared.currentEnHash,
|
|
2589
2860
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2590
|
-
model:
|
|
2861
|
+
model: output.model,
|
|
2591
2862
|
snapshotId
|
|
2592
2863
|
});
|
|
2593
2864
|
writeDb.close();
|
|
2594
2865
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2595
|
-
normalizeGeminiDisplayName(
|
|
2596
|
-
|
|
2597
|
-
|
|
2866
|
+
normalizeGeminiDisplayName(output.model),
|
|
2867
|
+
output.usage.inputTokens,
|
|
2868
|
+
output.usage.outputTokens,
|
|
2869
|
+
options.costMode
|
|
2598
2870
|
);
|
|
2599
2871
|
return {
|
|
2600
2872
|
...base,
|
|
2601
2873
|
skipped: false,
|
|
2602
|
-
model:
|
|
2603
|
-
usage:
|
|
2874
|
+
model: output.model,
|
|
2875
|
+
usage: output.usage,
|
|
2604
2876
|
estimatedCostUsd,
|
|
2605
|
-
durationMs: Date.now() - startedAt,
|
|
2877
|
+
durationMs: Date.now() - options.startedAt,
|
|
2606
2878
|
slugAdjusted,
|
|
2607
2879
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2608
2880
|
};
|
|
@@ -2612,33 +2884,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2612
2884
|
skipped: false,
|
|
2613
2885
|
failed: true,
|
|
2614
2886
|
error: formatTranslateError(error),
|
|
2887
|
+
durationMs: Date.now() - options.startedAt
|
|
2888
|
+
};
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
function displayModelFor(prepared) {
|
|
2892
|
+
return normalizeGeminiDisplayName(
|
|
2893
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
2894
|
+
);
|
|
2895
|
+
}
|
|
2896
|
+
|
|
2897
|
+
// src/translate/batch-worklist.ts
|
|
2898
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
2899
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
2900
|
+
var INITIAL_POLL_MS = 5e3;
|
|
2901
|
+
var MAX_POLL_MS = 3e4;
|
|
2902
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
2903
|
+
function sleep(ms) {
|
|
2904
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2905
|
+
}
|
|
2906
|
+
function planBatchJobs(entries) {
|
|
2907
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
2908
|
+
for (const entry of entries) {
|
|
2909
|
+
const group = byModel.get(entry.apiModel);
|
|
2910
|
+
if (group) group.push(entry);
|
|
2911
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
2912
|
+
}
|
|
2913
|
+
const plans = [];
|
|
2914
|
+
for (const [apiModel, group] of byModel) {
|
|
2915
|
+
let current = [];
|
|
2916
|
+
let currentBytes = 0;
|
|
2917
|
+
for (const entry of group) {
|
|
2918
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
2919
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
2920
|
+
plans.push({ apiModel, entries: current });
|
|
2921
|
+
current = [];
|
|
2922
|
+
currentBytes = 0;
|
|
2923
|
+
}
|
|
2924
|
+
current.push(entry);
|
|
2925
|
+
currentBytes += size;
|
|
2926
|
+
}
|
|
2927
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
2928
|
+
}
|
|
2929
|
+
return plans;
|
|
2930
|
+
}
|
|
2931
|
+
function readPendingBatchWork(config) {
|
|
2932
|
+
const db = openStore(config, "readwrite");
|
|
2933
|
+
const jobs = listPendingBatchJobs(db);
|
|
2934
|
+
const pendingItems = listPendingBatchItems(db);
|
|
2935
|
+
db.close();
|
|
2936
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
2937
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
2938
|
+
}
|
|
2939
|
+
async function submitBatchJobPlan(config, input) {
|
|
2940
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
2941
|
+
const job = await createGeminiBatchJob({
|
|
2942
|
+
model: plan.apiModel,
|
|
2943
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
2944
|
+
contents: prepared.prompt,
|
|
2945
|
+
config: buildGeminiRequestConfig({
|
|
2946
|
+
apiModelId: plan.apiModel,
|
|
2947
|
+
responseSchema: prepared.responseSchema
|
|
2948
|
+
})
|
|
2949
|
+
})),
|
|
2950
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
2951
|
+
});
|
|
2952
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2953
|
+
const db = openStore(config, "readwrite");
|
|
2954
|
+
const jobId = insertBatchJob(db, {
|
|
2955
|
+
jobName: job.name,
|
|
2956
|
+
model: plan.apiModel,
|
|
2957
|
+
displayModel,
|
|
2958
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2959
|
+
createdAt
|
|
2960
|
+
});
|
|
2961
|
+
insertBatchItems(
|
|
2962
|
+
db,
|
|
2963
|
+
jobId,
|
|
2964
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
2965
|
+
requestIndex,
|
|
2966
|
+
contentType: prepared.item.contentType,
|
|
2967
|
+
enSlug: prepared.item.enSlug,
|
|
2968
|
+
locale: prepared.item.locale,
|
|
2969
|
+
enHash: prepared.currentEnHash,
|
|
2970
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
2971
|
+
// on the EN files still matching (or existing) on disk.
|
|
2972
|
+
snapshotId: recordEnSnapshot(
|
|
2973
|
+
config,
|
|
2974
|
+
{
|
|
2975
|
+
contentType: prepared.item.contentType,
|
|
2976
|
+
enSlug: prepared.item.enSlug,
|
|
2977
|
+
enHash: prepared.currentEnHash,
|
|
2978
|
+
frontmatter: prepared.payload.frontmatter,
|
|
2979
|
+
body: prepared.payload.body
|
|
2980
|
+
},
|
|
2981
|
+
db
|
|
2982
|
+
)
|
|
2983
|
+
}))
|
|
2984
|
+
);
|
|
2985
|
+
const row = {
|
|
2986
|
+
id: jobId,
|
|
2987
|
+
job_name: job.name,
|
|
2988
|
+
model: plan.apiModel,
|
|
2989
|
+
display_model: displayModel,
|
|
2990
|
+
created_at: createdAt,
|
|
2991
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2992
|
+
completed_at: null
|
|
2993
|
+
};
|
|
2994
|
+
db.close();
|
|
2995
|
+
input.onProgress?.({
|
|
2996
|
+
type: "batch-submitted",
|
|
2997
|
+
name: job.name,
|
|
2998
|
+
count: plan.entries.length,
|
|
2999
|
+
model: displayModel,
|
|
3000
|
+
jobIndex,
|
|
3001
|
+
jobCount,
|
|
3002
|
+
createdAt
|
|
3003
|
+
});
|
|
3004
|
+
return row;
|
|
3005
|
+
}
|
|
3006
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3007
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3008
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3009
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3010
|
+
if (!enDoc) {
|
|
3011
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3012
|
+
if (!snapshot) {
|
|
3013
|
+
throw new Error(
|
|
3014
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3015
|
+
);
|
|
3016
|
+
}
|
|
3017
|
+
enDoc = {
|
|
3018
|
+
slug: itemRow.en_slug,
|
|
3019
|
+
enSlug: itemRow.en_slug,
|
|
3020
|
+
locale: config.defaultLocale,
|
|
3021
|
+
noindex: false,
|
|
3022
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3023
|
+
content: snapshot.body
|
|
3024
|
+
};
|
|
3025
|
+
}
|
|
3026
|
+
const item = {
|
|
3027
|
+
contentType: itemRow.content_type,
|
|
3028
|
+
enSlug: itemRow.en_slug,
|
|
3029
|
+
locale: itemRow.locale,
|
|
3030
|
+
reason: "missing",
|
|
3031
|
+
currentEnHash: itemRow.en_hash
|
|
3032
|
+
};
|
|
3033
|
+
return {
|
|
3034
|
+
item,
|
|
3035
|
+
type,
|
|
3036
|
+
enDoc,
|
|
3037
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3038
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3039
|
+
currentEnHash: itemRow.en_hash,
|
|
3040
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3041
|
+
model: jobRow.display_model,
|
|
3042
|
+
prompt: "",
|
|
3043
|
+
responseSchema: void 0
|
|
3044
|
+
};
|
|
3045
|
+
}
|
|
3046
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3047
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3048
|
+
const db = openStore(config, "readwrite");
|
|
3049
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3050
|
+
db.close();
|
|
3051
|
+
return null;
|
|
3052
|
+
}
|
|
3053
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3054
|
+
const results = [];
|
|
3055
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3056
|
+
const result = {
|
|
3057
|
+
contentType: itemRow.content_type,
|
|
3058
|
+
enSlug: itemRow.en_slug,
|
|
3059
|
+
locale: itemRow.locale,
|
|
3060
|
+
skipped: false,
|
|
3061
|
+
failed: true,
|
|
3062
|
+
error: formatTranslateError(new Error(message)),
|
|
2615
3063
|
durationMs: Date.now() - startedAt
|
|
2616
3064
|
};
|
|
3065
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3066
|
+
results.push(result);
|
|
3067
|
+
onResult?.(result);
|
|
3068
|
+
};
|
|
3069
|
+
if (isSuccessfulBatchState(state)) {
|
|
3070
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3071
|
+
for (const itemRow of pendingItems) {
|
|
3072
|
+
const startedAt = Date.now();
|
|
3073
|
+
const inlined = responses[itemRow.request_index];
|
|
3074
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3075
|
+
failItem(
|
|
3076
|
+
itemRow,
|
|
3077
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3078
|
+
startedAt
|
|
3079
|
+
);
|
|
3080
|
+
continue;
|
|
3081
|
+
}
|
|
3082
|
+
let result;
|
|
3083
|
+
try {
|
|
3084
|
+
const response = inlined.response;
|
|
3085
|
+
const raw = textFromBatchResponse(response);
|
|
3086
|
+
const parsed = parseGeminiResponse(raw);
|
|
3087
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3088
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3089
|
+
}
|
|
3090
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3091
|
+
result = finalizeTranslation(
|
|
3092
|
+
config,
|
|
3093
|
+
prepared,
|
|
3094
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3095
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3096
|
+
);
|
|
3097
|
+
} catch (error) {
|
|
3098
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3099
|
+
continue;
|
|
3100
|
+
}
|
|
3101
|
+
updateBatchItemStatus(
|
|
3102
|
+
db,
|
|
3103
|
+
jobRow.id,
|
|
3104
|
+
itemRow.request_index,
|
|
3105
|
+
result.failed ? "failed" : "done",
|
|
3106
|
+
result.error
|
|
3107
|
+
);
|
|
3108
|
+
results.push(result);
|
|
3109
|
+
onResult?.(result);
|
|
3110
|
+
}
|
|
3111
|
+
} else {
|
|
3112
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3113
|
+
const startedAt = Date.now();
|
|
3114
|
+
for (const itemRow of pendingItems) {
|
|
3115
|
+
failItem(itemRow, message, startedAt);
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
db.close();
|
|
3119
|
+
return results;
|
|
3120
|
+
}
|
|
3121
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3122
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3123
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3124
|
+
const loopStartedAt = Date.now();
|
|
3125
|
+
let delay = INITIAL_POLL_MS;
|
|
3126
|
+
let firstRound = true;
|
|
3127
|
+
const elapsedFor = (row) => {
|
|
3128
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3129
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3130
|
+
};
|
|
3131
|
+
while (active.length > 0) {
|
|
3132
|
+
if (!firstRound) {
|
|
3133
|
+
await sleep(delay);
|
|
3134
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3135
|
+
}
|
|
3136
|
+
firstRound = false;
|
|
3137
|
+
for (const tracked of [...active]) {
|
|
3138
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3139
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3140
|
+
options.onProgress?.({
|
|
3141
|
+
type: "batch-polling",
|
|
3142
|
+
name: tracked.row.job_name,
|
|
3143
|
+
state,
|
|
3144
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3145
|
+
jobIndex: tracked.jobIndex,
|
|
3146
|
+
jobCount
|
|
3147
|
+
});
|
|
3148
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3149
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3150
|
+
const results = ingestBatchJob(
|
|
3151
|
+
config,
|
|
3152
|
+
tracked.row,
|
|
3153
|
+
batchJob,
|
|
3154
|
+
options.onResult
|
|
3155
|
+
);
|
|
3156
|
+
if (results === null) {
|
|
3157
|
+
options.onProgress?.({
|
|
3158
|
+
type: "batch-done",
|
|
3159
|
+
name: tracked.row.job_name,
|
|
3160
|
+
state,
|
|
3161
|
+
model: tracked.row.display_model,
|
|
3162
|
+
count: 0,
|
|
3163
|
+
jobIndex: tracked.jobIndex,
|
|
3164
|
+
jobCount,
|
|
3165
|
+
translated: 0,
|
|
3166
|
+
failed: 0,
|
|
3167
|
+
inputTokens: 0,
|
|
3168
|
+
outputTokens: 0,
|
|
3169
|
+
estimatedCostUsd: 0,
|
|
3170
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3171
|
+
alreadyIngested: true
|
|
3172
|
+
});
|
|
3173
|
+
continue;
|
|
3174
|
+
}
|
|
3175
|
+
options.onProgress?.({
|
|
3176
|
+
type: "batch-done",
|
|
3177
|
+
name: tracked.row.job_name,
|
|
3178
|
+
state,
|
|
3179
|
+
model: tracked.row.display_model,
|
|
3180
|
+
count: results.length,
|
|
3181
|
+
jobIndex: tracked.jobIndex,
|
|
3182
|
+
jobCount,
|
|
3183
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3184
|
+
failed: results.filter((r) => r.failed).length,
|
|
3185
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3186
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3187
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3188
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3189
|
+
});
|
|
3190
|
+
}
|
|
3191
|
+
}
|
|
2617
3192
|
}
|
|
2618
3193
|
}
|
|
3194
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3195
|
+
return {
|
|
3196
|
+
...baseForItem(item),
|
|
3197
|
+
skipped: false,
|
|
3198
|
+
failed: true,
|
|
3199
|
+
error: formatTranslateError(error),
|
|
3200
|
+
durationMs: Date.now() - startedAt
|
|
3201
|
+
};
|
|
3202
|
+
}
|
|
3203
|
+
|
|
3204
|
+
// src/translate/page-translator.ts
|
|
3205
|
+
async function runPool(items, concurrency, worker) {
|
|
3206
|
+
if (items.length === 0) return;
|
|
3207
|
+
let nextIndex = 0;
|
|
3208
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3209
|
+
while (nextIndex < items.length) {
|
|
3210
|
+
const index = nextIndex++;
|
|
3211
|
+
await worker(items[index], index);
|
|
3212
|
+
}
|
|
3213
|
+
});
|
|
3214
|
+
await Promise.all(runners);
|
|
3215
|
+
}
|
|
2619
3216
|
function labelForItem(item) {
|
|
2620
3217
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2621
3218
|
}
|
|
3219
|
+
async function translatePage(config, item, options = {}) {
|
|
3220
|
+
const startedAt = Date.now();
|
|
3221
|
+
let outcome;
|
|
3222
|
+
try {
|
|
3223
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3224
|
+
} catch (error) {
|
|
3225
|
+
return failedResultForItem(item, error, startedAt);
|
|
3226
|
+
}
|
|
3227
|
+
if (outcome.status === "done") return outcome.result;
|
|
3228
|
+
const prepared = outcome.prepared;
|
|
3229
|
+
if (options.dryRun) {
|
|
3230
|
+
return {
|
|
3231
|
+
...baseForItem(item),
|
|
3232
|
+
skipped: false,
|
|
3233
|
+
model: prepared.model,
|
|
3234
|
+
durationMs: Date.now() - startedAt
|
|
3235
|
+
};
|
|
3236
|
+
}
|
|
3237
|
+
try {
|
|
3238
|
+
const result = await translatePageWithGemini({
|
|
3239
|
+
prompt: prepared.prompt,
|
|
3240
|
+
model: prepared.model,
|
|
3241
|
+
responseSchema: prepared.responseSchema
|
|
3242
|
+
});
|
|
3243
|
+
return finalizeTranslation(
|
|
3244
|
+
config,
|
|
3245
|
+
prepared,
|
|
3246
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3247
|
+
{ costMode: "interactive", startedAt }
|
|
3248
|
+
);
|
|
3249
|
+
} catch (error) {
|
|
3250
|
+
return failedResultForItem(item, error, startedAt);
|
|
3251
|
+
}
|
|
3252
|
+
}
|
|
3253
|
+
function createResultCollector(items, onProgress) {
|
|
3254
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3255
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3256
|
+
const slots = new Array(items.length);
|
|
3257
|
+
const extras = [];
|
|
3258
|
+
return {
|
|
3259
|
+
emit(result) {
|
|
3260
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3261
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3262
|
+
else extras.push(result);
|
|
3263
|
+
onProgress?.({ type: "item-done", result });
|
|
3264
|
+
},
|
|
3265
|
+
assemble() {
|
|
3266
|
+
return [
|
|
3267
|
+
...slots.filter((result) => result !== void 0),
|
|
3268
|
+
...extras
|
|
3269
|
+
];
|
|
3270
|
+
}
|
|
3271
|
+
};
|
|
3272
|
+
}
|
|
2622
3273
|
async function translateWorklist(config, items, options = {}) {
|
|
3274
|
+
const mode = options.mode ?? "batch";
|
|
2623
3275
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2624
3276
|
const startedAt = Date.now();
|
|
2625
|
-
|
|
2626
|
-
|
|
3277
|
+
if (options.dryRun) {
|
|
3278
|
+
options.onProgress?.({
|
|
3279
|
+
type: "start",
|
|
3280
|
+
total: items.length,
|
|
3281
|
+
concurrency,
|
|
3282
|
+
dryRun: true,
|
|
3283
|
+
model: options.model,
|
|
3284
|
+
mode
|
|
3285
|
+
});
|
|
3286
|
+
const results2 = items.map((item) => {
|
|
3287
|
+
const itemStartedAt = Date.now();
|
|
3288
|
+
try {
|
|
3289
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3290
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3291
|
+
...baseForItem(item),
|
|
3292
|
+
skipped: false,
|
|
3293
|
+
model: outcome.prepared.model,
|
|
3294
|
+
durationMs: Date.now() - itemStartedAt
|
|
3295
|
+
};
|
|
3296
|
+
} catch (error) {
|
|
3297
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3298
|
+
}
|
|
3299
|
+
});
|
|
3300
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3301
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3302
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3303
|
+
return results2;
|
|
3304
|
+
}
|
|
3305
|
+
const pending = readPendingBatchWork(config);
|
|
3306
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3307
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3308
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3309
|
+
).length;
|
|
2627
3310
|
options.onProgress?.({
|
|
2628
3311
|
type: "start",
|
|
2629
|
-
total: items.length,
|
|
3312
|
+
total: items.length + resumedExtraCount,
|
|
2630
3313
|
concurrency,
|
|
2631
|
-
dryRun:
|
|
2632
|
-
model: options.model
|
|
3314
|
+
dryRun: false,
|
|
3315
|
+
model: options.model,
|
|
3316
|
+
mode
|
|
2633
3317
|
});
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
3318
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3319
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3320
|
+
if (mode === "direct") {
|
|
3321
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3322
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3323
|
+
const active = /* @__PURE__ */ new Set();
|
|
3324
|
+
await Promise.all([
|
|
3325
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3326
|
+
const label = labelForItem(item);
|
|
3327
|
+
active.add(label);
|
|
3328
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3329
|
+
const result = await translatePage(config, item, options);
|
|
3330
|
+
active.delete(label);
|
|
3331
|
+
collector.emit(result);
|
|
3332
|
+
}),
|
|
3333
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3334
|
+
onProgress: options.onProgress,
|
|
3335
|
+
onResult: collector.emit
|
|
3336
|
+
}) : Promise.resolve()
|
|
3337
|
+
]);
|
|
3338
|
+
} else {
|
|
3339
|
+
const entries = [];
|
|
3340
|
+
for (const item of workItems) {
|
|
3341
|
+
const itemStartedAt = Date.now();
|
|
3342
|
+
try {
|
|
3343
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3344
|
+
if (outcome.status === "done") {
|
|
3345
|
+
collector.emit(outcome.result);
|
|
3346
|
+
} else {
|
|
3347
|
+
entries.push({
|
|
3348
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3349
|
+
prompt: outcome.prepared.prompt,
|
|
3350
|
+
prepared: outcome.prepared
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
} catch (error) {
|
|
3354
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
const plans = planBatchJobs(entries);
|
|
3358
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3359
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3360
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3361
|
+
const submitted = await Promise.all(
|
|
3362
|
+
plans.map(async (plan, planIndex) => {
|
|
3363
|
+
try {
|
|
3364
|
+
return await submitBatchJobPlan(config, {
|
|
3365
|
+
plan,
|
|
3366
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3367
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3368
|
+
jobCount,
|
|
3369
|
+
onProgress: options.onProgress
|
|
3370
|
+
});
|
|
3371
|
+
} catch (error) {
|
|
3372
|
+
for (const entry of plan.entries) {
|
|
3373
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3374
|
+
}
|
|
3375
|
+
return void 0;
|
|
3376
|
+
}
|
|
3377
|
+
})
|
|
3378
|
+
);
|
|
3379
|
+
const jobsToPoll = [
|
|
3380
|
+
...pending.jobs,
|
|
3381
|
+
...submitted.filter((row) => row !== void 0)
|
|
3382
|
+
];
|
|
3383
|
+
if (jobsToPoll.length > 0) {
|
|
3384
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3385
|
+
jobCount,
|
|
3386
|
+
onProgress: options.onProgress,
|
|
3387
|
+
onResult: collector.emit
|
|
3388
|
+
});
|
|
3389
|
+
}
|
|
3390
|
+
}
|
|
3391
|
+
const results = collector.assemble();
|
|
3392
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3393
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3394
|
+
return results;
|
|
3395
|
+
}
|
|
3396
|
+
function countPendingItems(config, jobs) {
|
|
3397
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3398
|
+
const db = openStore(config, "readonly");
|
|
3399
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3400
|
+
for (const job of jobs) {
|
|
3401
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3402
|
+
}
|
|
3403
|
+
db.close();
|
|
3404
|
+
return counts;
|
|
3405
|
+
}
|
|
3406
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3407
|
+
jobs.forEach((job, jobIndex) => {
|
|
3408
|
+
onProgress?.({
|
|
3409
|
+
type: "batch-submitted",
|
|
3410
|
+
name: job.job_name,
|
|
3411
|
+
count: counts.get(job.id) ?? 0,
|
|
3412
|
+
model: job.display_model,
|
|
3413
|
+
jobIndex,
|
|
3414
|
+
jobCount,
|
|
3415
|
+
resumed: true,
|
|
3416
|
+
createdAt: job.created_at
|
|
3417
|
+
});
|
|
3418
|
+
});
|
|
3419
|
+
}
|
|
3420
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3421
|
+
const startedAt = Date.now();
|
|
3422
|
+
const pending = readPendingBatchWork(config);
|
|
3423
|
+
if (pending.jobs.length === 0) return null;
|
|
3424
|
+
options.onProgress?.({
|
|
3425
|
+
type: "start",
|
|
3426
|
+
total: pending.pendingItems.length,
|
|
3427
|
+
concurrency: 1,
|
|
3428
|
+
dryRun: false,
|
|
3429
|
+
mode: "batch"
|
|
3430
|
+
});
|
|
3431
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3432
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3433
|
+
const results = [];
|
|
3434
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3435
|
+
onProgress: options.onProgress,
|
|
3436
|
+
onResult: (result) => {
|
|
3437
|
+
results.push(result);
|
|
3438
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3439
|
+
}
|
|
2642
3440
|
});
|
|
2643
3441
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2644
3442
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3256,35 +4054,43 @@ async function promptContentType(config) {
|
|
|
3256
4054
|
})
|
|
3257
4055
|
);
|
|
3258
4056
|
}
|
|
4057
|
+
var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
|
|
4058
|
+
var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
|
|
4059
|
+
async function promptManualLocales(config) {
|
|
4060
|
+
const locales = nonDefaultLocales(config);
|
|
4061
|
+
if (locales.length <= 1) return {};
|
|
4062
|
+
const picked = await runPrompt(
|
|
4063
|
+
prompts.checkbox({
|
|
4064
|
+
message: "Locales to translate",
|
|
4065
|
+
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
4066
|
+
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
4067
|
+
})
|
|
4068
|
+
);
|
|
4069
|
+
return picked.length === locales.length ? {} : { locale: picked };
|
|
4070
|
+
}
|
|
3259
4071
|
async function promptLocalePreset(config) {
|
|
3260
4072
|
const presets = Object.entries(config.localePresets ?? {}).filter(
|
|
3261
4073
|
(entry) => Array.isArray(entry[1])
|
|
3262
4074
|
);
|
|
3263
4075
|
if (presets.length === 0) {
|
|
3264
|
-
|
|
3265
|
-
if (locales.length <= 1) return {};
|
|
3266
|
-
const picked = await runPrompt(
|
|
3267
|
-
prompts.checkbox({
|
|
3268
|
-
message: "Locales to translate",
|
|
3269
|
-
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
3270
|
-
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
3271
|
-
})
|
|
3272
|
-
);
|
|
3273
|
-
return picked.length === locales.length ? {} : { locale: picked };
|
|
4076
|
+
return promptManualLocales(config);
|
|
3274
4077
|
}
|
|
3275
4078
|
const choice = await runPrompt(
|
|
3276
4079
|
prompts.select({
|
|
3277
4080
|
message: "Locale preset",
|
|
3278
4081
|
choices: [
|
|
3279
|
-
{ name: "All locales", value:
|
|
4082
|
+
{ name: "All locales", value: ALL_LOCALES },
|
|
3280
4083
|
...presets.map(([name, locales]) => ({
|
|
3281
4084
|
name: `${name} (${locales.join(", ")})`,
|
|
3282
4085
|
value: name
|
|
3283
|
-
}))
|
|
4086
|
+
})),
|
|
4087
|
+
{ name: "Choose locales manually\u2026", value: PICK_LOCALES }
|
|
3284
4088
|
]
|
|
3285
4089
|
})
|
|
3286
4090
|
);
|
|
3287
|
-
|
|
4091
|
+
if (choice === PICK_LOCALES) return promptManualLocales(config);
|
|
4092
|
+
if (choice === ALL_LOCALES) return {};
|
|
4093
|
+
return { preset: choice };
|
|
3288
4094
|
}
|
|
3289
4095
|
async function promptStrategy() {
|
|
3290
4096
|
return runPrompt(
|
|
@@ -3298,12 +4104,25 @@ async function promptStrategy() {
|
|
|
3298
4104
|
})
|
|
3299
4105
|
);
|
|
3300
4106
|
}
|
|
4107
|
+
async function promptMode() {
|
|
4108
|
+
return runPrompt(
|
|
4109
|
+
prompts.select({
|
|
4110
|
+
message: "Translation mode",
|
|
4111
|
+
choices: [
|
|
4112
|
+
{ name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
|
|
4113
|
+
{ name: "Direct \u2014 immediate results, full price", value: "direct" }
|
|
4114
|
+
],
|
|
4115
|
+
default: "batch"
|
|
4116
|
+
})
|
|
4117
|
+
);
|
|
4118
|
+
}
|
|
3301
4119
|
async function promptTranslateSelection(config, flags) {
|
|
3302
4120
|
const selection = {
|
|
3303
4121
|
contentType: flags.type,
|
|
3304
4122
|
preset: flags.preset,
|
|
3305
4123
|
locale: flags.locale,
|
|
3306
|
-
strategy: flags.strategy
|
|
4124
|
+
strategy: flags.strategy,
|
|
4125
|
+
mode: flags.mode
|
|
3307
4126
|
};
|
|
3308
4127
|
if (!isInteractive()) return selection;
|
|
3309
4128
|
if (!selection.contentType) {
|
|
@@ -3315,6 +4134,9 @@ async function promptTranslateSelection(config, flags) {
|
|
|
3315
4134
|
if (!selection.strategy) {
|
|
3316
4135
|
selection.strategy = await promptStrategy();
|
|
3317
4136
|
}
|
|
4137
|
+
if (!selection.mode) {
|
|
4138
|
+
selection.mode = await promptMode();
|
|
4139
|
+
}
|
|
3318
4140
|
return selection;
|
|
3319
4141
|
}
|
|
3320
4142
|
|
|
@@ -3336,6 +4158,34 @@ function progressBar(ratio, width = 28) {
|
|
|
3336
4158
|
function labelForResult(result) {
|
|
3337
4159
|
return `${result.contentType}/${result.enSlug}@${result.locale}`;
|
|
3338
4160
|
}
|
|
4161
|
+
function shortJobId(name) {
|
|
4162
|
+
const segments = name.split("/");
|
|
4163
|
+
return segments[segments.length - 1] || name;
|
|
4164
|
+
}
|
|
4165
|
+
function formatElapsed(elapsedMs) {
|
|
4166
|
+
return `${Math.round(elapsedMs / 1e3)}s`;
|
|
4167
|
+
}
|
|
4168
|
+
function batchDoneParts(event) {
|
|
4169
|
+
if (event.alreadyIngested) {
|
|
4170
|
+
return [
|
|
4171
|
+
`job ${shortJobId(event.name)}`,
|
|
4172
|
+
event.model ?? "?",
|
|
4173
|
+
"already ingested by another scribe run (results saved there)",
|
|
4174
|
+
formatElapsed(event.elapsedMs)
|
|
4175
|
+
].join(" \xB7 ");
|
|
4176
|
+
}
|
|
4177
|
+
const parts = [
|
|
4178
|
+
`job ${shortJobId(event.name)}`,
|
|
4179
|
+
event.model ?? "?",
|
|
4180
|
+
`${event.count} request${event.count === 1 ? "" : "s"}`,
|
|
4181
|
+
`${event.translated} translated`
|
|
4182
|
+
];
|
|
4183
|
+
if (event.failed > 0) parts.push(`${event.failed} failed`);
|
|
4184
|
+
parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
|
|
4185
|
+
parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
|
|
4186
|
+
parts.push(formatElapsed(event.elapsedMs));
|
|
4187
|
+
return parts.join(" \xB7 ");
|
|
4188
|
+
}
|
|
3339
4189
|
function statusForResult(result, dryRun) {
|
|
3340
4190
|
if (result.failed) return red("failed");
|
|
3341
4191
|
if (result.skipped) return dim("skipped");
|
|
@@ -3381,6 +4231,25 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3381
4231
|
if (!enabled) {
|
|
3382
4232
|
return {
|
|
3383
4233
|
onEvent(event) {
|
|
4234
|
+
if (event.type === "batch-submitted") {
|
|
4235
|
+
const verb = event.resumed ? "Resuming" : "Submitted";
|
|
4236
|
+
console.log(
|
|
4237
|
+
`${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
|
|
4238
|
+
);
|
|
4239
|
+
return;
|
|
4240
|
+
}
|
|
4241
|
+
if (event.type === "batch-polling") {
|
|
4242
|
+
console.log(
|
|
4243
|
+
`batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
|
|
4244
|
+
);
|
|
4245
|
+
return;
|
|
4246
|
+
}
|
|
4247
|
+
if (event.type === "batch-done") {
|
|
4248
|
+
const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
|
|
4249
|
+
if (event.failed > 0 && event.translated === 0) console.error(line);
|
|
4250
|
+
else console.log(line);
|
|
4251
|
+
return;
|
|
4252
|
+
}
|
|
3384
4253
|
if (event.type === "item-done") {
|
|
3385
4254
|
const label = labelForResult(event.result);
|
|
3386
4255
|
const status = statusForResult(event.result, dryRun);
|
|
@@ -3411,6 +4280,9 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3411
4280
|
let outputTokens = 0;
|
|
3412
4281
|
let estimatedCostUsd = 0;
|
|
3413
4282
|
let active = [];
|
|
4283
|
+
let mode;
|
|
4284
|
+
const batchJobs = /* @__PURE__ */ new Map();
|
|
4285
|
+
let batchElapsedMs = 0;
|
|
3414
4286
|
const recent = [];
|
|
3415
4287
|
let renderedLines = 0;
|
|
3416
4288
|
let cursorHidden = false;
|
|
@@ -3426,6 +4298,15 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3426
4298
|
cursorHidden = false;
|
|
3427
4299
|
}
|
|
3428
4300
|
}
|
|
4301
|
+
function printPersistent(lines) {
|
|
4302
|
+
if (renderedLines > 0) {
|
|
4303
|
+
process.stdout.write(`\x1B[${renderedLines}A`);
|
|
4304
|
+
}
|
|
4305
|
+
for (const line of lines) {
|
|
4306
|
+
process.stdout.write("\x1B[2K" + line + "\n");
|
|
4307
|
+
}
|
|
4308
|
+
renderedLines = 0;
|
|
4309
|
+
}
|
|
3429
4310
|
function render() {
|
|
3430
4311
|
hideCursor();
|
|
3431
4312
|
if (renderedLines > 0) {
|
|
@@ -3433,12 +4314,34 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3433
4314
|
}
|
|
3434
4315
|
const lines = [];
|
|
3435
4316
|
const title = dryRun ? "Dry run" : "Translating";
|
|
3436
|
-
|
|
4317
|
+
const isBatch = mode === "batch" || batchJobs.size > 0;
|
|
4318
|
+
lines.push(
|
|
4319
|
+
cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
|
|
4320
|
+
);
|
|
3437
4321
|
lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
|
|
3438
4322
|
lines.push(
|
|
3439
4323
|
dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
|
|
3440
4324
|
);
|
|
3441
|
-
if (
|
|
4325
|
+
if (batchJobs.size > 0 && done < total) {
|
|
4326
|
+
const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
|
|
4327
|
+
let running = 0;
|
|
4328
|
+
let finished = 0;
|
|
4329
|
+
let requests = 0;
|
|
4330
|
+
for (const job of batchJobs.values()) {
|
|
4331
|
+
if (terminal.test(job.state)) finished += 1;
|
|
4332
|
+
else running += 1;
|
|
4333
|
+
requests += job.count;
|
|
4334
|
+
}
|
|
4335
|
+
lines.push(
|
|
4336
|
+
dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
|
|
4337
|
+
);
|
|
4338
|
+
for (const [name, job] of batchJobs) {
|
|
4339
|
+
if (terminal.test(job.state)) continue;
|
|
4340
|
+
lines.push(
|
|
4341
|
+
dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
|
|
4342
|
+
);
|
|
4343
|
+
}
|
|
4344
|
+
} else if (active.length > 0) {
|
|
3442
4345
|
lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
|
|
3443
4346
|
} else if (done < total) {
|
|
3444
4347
|
lines.push(dim("Active ") + "starting\u2026");
|
|
@@ -3466,12 +4369,58 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3466
4369
|
total = event.total;
|
|
3467
4370
|
concurrency = event.concurrency;
|
|
3468
4371
|
model = event.model;
|
|
4372
|
+
mode = event.mode;
|
|
3469
4373
|
render();
|
|
3470
4374
|
break;
|
|
3471
4375
|
case "item-start":
|
|
3472
4376
|
active = event.active;
|
|
3473
4377
|
render();
|
|
3474
4378
|
break;
|
|
4379
|
+
case "batch-submitted": {
|
|
4380
|
+
const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
|
|
4381
|
+
const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
|
|
4382
|
+
batchJobs.set(event.name, {
|
|
4383
|
+
state: event.resumed ? "RESUMED" : "SUBMITTED",
|
|
4384
|
+
count: event.count,
|
|
4385
|
+
model: event.model,
|
|
4386
|
+
createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
|
|
4387
|
+
elapsedMs
|
|
4388
|
+
});
|
|
4389
|
+
const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
|
|
4390
|
+
const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
|
|
4391
|
+
printPersistent([
|
|
4392
|
+
dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
|
|
4393
|
+
]);
|
|
4394
|
+
render();
|
|
4395
|
+
break;
|
|
4396
|
+
}
|
|
4397
|
+
case "batch-polling": {
|
|
4398
|
+
const job = batchJobs.get(event.name);
|
|
4399
|
+
batchJobs.set(event.name, {
|
|
4400
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4401
|
+
count: job?.count ?? 0,
|
|
4402
|
+
model: job?.model,
|
|
4403
|
+
createdAtMs: job?.createdAtMs,
|
|
4404
|
+
elapsedMs: event.elapsedMs
|
|
4405
|
+
});
|
|
4406
|
+
batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
|
|
4407
|
+
render();
|
|
4408
|
+
break;
|
|
4409
|
+
}
|
|
4410
|
+
case "batch-done": {
|
|
4411
|
+
const job = batchJobs.get(event.name);
|
|
4412
|
+
batchJobs.set(event.name, {
|
|
4413
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4414
|
+
count: job?.count ?? event.count,
|
|
4415
|
+
model: job?.model ?? event.model,
|
|
4416
|
+
createdAtMs: job?.createdAtMs,
|
|
4417
|
+
elapsedMs: event.elapsedMs
|
|
4418
|
+
});
|
|
4419
|
+
const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
|
|
4420
|
+
printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
|
|
4421
|
+
render();
|
|
4422
|
+
break;
|
|
4423
|
+
}
|
|
3475
4424
|
case "item-done": {
|
|
3476
4425
|
done += 1;
|
|
3477
4426
|
active = active.filter((label) => label !== labelForResult(event.result));
|
|
@@ -3594,6 +4543,21 @@ function parseArgs(argv) {
|
|
|
3594
4543
|
i++;
|
|
3595
4544
|
continue;
|
|
3596
4545
|
}
|
|
4546
|
+
if (arg === "--batch") {
|
|
4547
|
+
options.batch = true;
|
|
4548
|
+
i++;
|
|
4549
|
+
continue;
|
|
4550
|
+
}
|
|
4551
|
+
if (arg === "--direct") {
|
|
4552
|
+
options.direct = true;
|
|
4553
|
+
i++;
|
|
4554
|
+
continue;
|
|
4555
|
+
}
|
|
4556
|
+
if (arg === "--resume") {
|
|
4557
|
+
options.resume = true;
|
|
4558
|
+
i++;
|
|
4559
|
+
continue;
|
|
4560
|
+
}
|
|
3597
4561
|
if (arg === "--strategy") {
|
|
3598
4562
|
const value = argv[++i];
|
|
3599
4563
|
if (value !== "all" && value !== "missing-only") {
|
|
@@ -3662,11 +4626,19 @@ Translate flags:
|
|
|
3662
4626
|
--locale <code>... Target locale(s); overrides --preset
|
|
3663
4627
|
--slug <en-slug> Single English document
|
|
3664
4628
|
--model <id> Gemini model override
|
|
3665
|
-
--
|
|
4629
|
+
--batch Force Gemini Batch API mode (50% token cost; default)
|
|
4630
|
+
--direct Force direct interactive calls (immediate, full price)
|
|
4631
|
+
--resume Only poll/ingest pending batch jobs; submit nothing new
|
|
4632
|
+
--concurrency <n> Parallel translations in --direct mode (default: 3)
|
|
3666
4633
|
--dry-run List work without writing
|
|
3667
4634
|
--force Re-translate even when hashes match
|
|
3668
4635
|
--strategy <mode> all (default) or missing-only
|
|
3669
4636
|
--no-progress Plain line logging instead of live progress
|
|
4637
|
+
|
|
4638
|
+
Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
|
|
4639
|
+
during polling is safe: the next run (or --resume) picks the jobs back up and
|
|
4640
|
+
ingests their results. In a TTY without --batch/--direct, a prompt asks which
|
|
4641
|
+
mode to use.
|
|
3670
4642
|
`);
|
|
3671
4643
|
return;
|
|
3672
4644
|
}
|
|
@@ -3712,11 +4684,29 @@ Translate flags:
|
|
|
3712
4684
|
break;
|
|
3713
4685
|
}
|
|
3714
4686
|
case "translate": {
|
|
4687
|
+
if (options.batch && options.direct) {
|
|
4688
|
+
throw new Error("--batch and --direct are mutually exclusive");
|
|
4689
|
+
}
|
|
4690
|
+
if (options.resume) {
|
|
4691
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4692
|
+
enabled: !options.noProgress,
|
|
4693
|
+
dryRun: false
|
|
4694
|
+
});
|
|
4695
|
+
const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4696
|
+
reporter2.finish();
|
|
4697
|
+
if (results2 === null) {
|
|
4698
|
+
console.log("No pending translation batch jobs.");
|
|
4699
|
+
break;
|
|
4700
|
+
}
|
|
4701
|
+
if (results2.some((result) => result.failed)) process.exitCode = 1;
|
|
4702
|
+
break;
|
|
4703
|
+
}
|
|
3715
4704
|
const selection = await promptTranslateSelection(config, {
|
|
3716
4705
|
type: options.type,
|
|
3717
4706
|
preset: options.preset,
|
|
3718
4707
|
locale: options.locale,
|
|
3719
|
-
strategy: options.strategy
|
|
4708
|
+
strategy: options.strategy,
|
|
4709
|
+
mode: options.batch ? "batch" : options.direct ? "direct" : void 0
|
|
3720
4710
|
});
|
|
3721
4711
|
const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
|
|
3722
4712
|
const worklist = buildWorklist(config, {
|
|
@@ -3726,6 +4716,18 @@ Translate flags:
|
|
|
3726
4716
|
strategy: selection.strategy ?? "all"
|
|
3727
4717
|
});
|
|
3728
4718
|
if (worklist.length === 0) {
|
|
4719
|
+
if (!options.dryRun) {
|
|
4720
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4721
|
+
enabled: !options.noProgress,
|
|
4722
|
+
dryRun: false
|
|
4723
|
+
});
|
|
4724
|
+
const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4725
|
+
reporter2.finish();
|
|
4726
|
+
if (resumed !== null) {
|
|
4727
|
+
if (resumed.some((result) => result.failed)) process.exitCode = 1;
|
|
4728
|
+
break;
|
|
4729
|
+
}
|
|
4730
|
+
}
|
|
3729
4731
|
console.log("Nothing to translate.");
|
|
3730
4732
|
break;
|
|
3731
4733
|
}
|
|
@@ -3738,6 +4740,7 @@ Translate flags:
|
|
|
3738
4740
|
dryRun: options.dryRun,
|
|
3739
4741
|
force: options.force,
|
|
3740
4742
|
concurrency: options.concurrency,
|
|
4743
|
+
mode: selection.mode,
|
|
3741
4744
|
onProgress: reporter.onEvent
|
|
3742
4745
|
});
|
|
3743
4746
|
reporter.finish();
|