scribe-cms 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1258 -219
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1259 -220
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +1020 -181
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1021 -183
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +149 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +149 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +23 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/src/validate/validate-project.d.ts.map +1 -1
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.cjs
CHANGED
|
@@ -76,6 +76,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
function readSchemaVersion(db) {
|
|
79
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
80
|
+
if (!metaTable) return 0;
|
|
79
81
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
80
82
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
81
83
|
}
|
|
@@ -100,7 +102,7 @@ var SCHEMA_VERSION, MIGRATIONS;
|
|
|
100
102
|
var init_sqlite = __esm({
|
|
101
103
|
"src/storage/sqlite.ts"() {
|
|
102
104
|
init_cjs_shims();
|
|
103
|
-
SCHEMA_VERSION =
|
|
105
|
+
SCHEMA_VERSION = 5;
|
|
104
106
|
MIGRATIONS = [
|
|
105
107
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
106
108
|
key TEXT PRIMARY KEY,
|
|
@@ -148,7 +150,30 @@ var init_sqlite = __esm({
|
|
|
148
150
|
UNIQUE (content_type, en_slug, en_hash)
|
|
149
151
|
)`,
|
|
150
152
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
151
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
153
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
154
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
155
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
156
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
157
|
+
model TEXT NOT NULL,
|
|
158
|
+
display_model TEXT NOT NULL,
|
|
159
|
+
created_at TEXT NOT NULL,
|
|
160
|
+
state TEXT NOT NULL,
|
|
161
|
+
completed_at TEXT
|
|
162
|
+
)`,
|
|
163
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
164
|
+
job_id INTEGER NOT NULL,
|
|
165
|
+
request_index INTEGER NOT NULL,
|
|
166
|
+
content_type TEXT NOT NULL,
|
|
167
|
+
en_slug TEXT NOT NULL,
|
|
168
|
+
locale TEXT NOT NULL,
|
|
169
|
+
en_hash TEXT NOT NULL,
|
|
170
|
+
snapshot_id INTEGER NOT NULL,
|
|
171
|
+
status TEXT NOT NULL,
|
|
172
|
+
error TEXT,
|
|
173
|
+
PRIMARY KEY (job_id, request_index)
|
|
174
|
+
)`,
|
|
175
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
176
|
+
ON translation_batch_items(status)`
|
|
152
177
|
];
|
|
153
178
|
}
|
|
154
179
|
});
|
|
@@ -190,26 +215,6 @@ function getRelationTarget(schema) {
|
|
|
190
215
|
}
|
|
191
216
|
return null;
|
|
192
217
|
}
|
|
193
|
-
function unwrapSchema(schema) {
|
|
194
|
-
let current = schema;
|
|
195
|
-
for (let i = 0; i < 8; i++) {
|
|
196
|
-
const anySchema = current;
|
|
197
|
-
if (typeof anySchema.unwrap === "function") {
|
|
198
|
-
current = anySchema.unwrap();
|
|
199
|
-
continue;
|
|
200
|
-
}
|
|
201
|
-
if (typeof anySchema.removeDefault === "function") {
|
|
202
|
-
current = anySchema.removeDefault();
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
if (anySchema._def?.innerType) {
|
|
206
|
-
current = anySchema._def.innerType;
|
|
207
|
-
continue;
|
|
208
|
-
}
|
|
209
|
-
break;
|
|
210
|
-
}
|
|
211
|
-
return current;
|
|
212
|
-
}
|
|
213
218
|
function peelOptionalWrappers(schema) {
|
|
214
219
|
let current = schema;
|
|
215
220
|
for (let i = 0; i < 8; i++) {
|
|
@@ -355,6 +360,39 @@ function resolveConfig(input, baseDir) {
|
|
|
355
360
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
356
361
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
357
362
|
}
|
|
363
|
+
const localeFallbacks = {};
|
|
364
|
+
for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
|
|
365
|
+
if (!raw.locales.includes(locale)) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
const seen = /* @__PURE__ */ new Set();
|
|
371
|
+
for (const fallback of chain) {
|
|
372
|
+
if (!raw.locales.includes(fallback)) {
|
|
373
|
+
throw new Error(
|
|
374
|
+
`scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
if (fallback === locale) {
|
|
378
|
+
throw new Error(
|
|
379
|
+
`scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
if (fallback === defaultLocale) {
|
|
383
|
+
throw new Error(
|
|
384
|
+
`scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
if (seen.has(fallback)) {
|
|
388
|
+
throw new Error(
|
|
389
|
+
`scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
seen.add(fallback);
|
|
393
|
+
}
|
|
394
|
+
localeFallbacks[locale] = [...chain];
|
|
395
|
+
}
|
|
358
396
|
const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
359
397
|
const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
|
|
360
398
|
const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -384,6 +422,7 @@ function resolveConfig(input, baseDir) {
|
|
|
384
422
|
defaultLocale,
|
|
385
423
|
localeRouting,
|
|
386
424
|
localePresets: raw.localePresets,
|
|
425
|
+
localeFallbacks,
|
|
387
426
|
translate: raw.translate,
|
|
388
427
|
types
|
|
389
428
|
};
|
|
@@ -399,6 +438,12 @@ init_cjs_shims();
|
|
|
399
438
|
|
|
400
439
|
// src/core/introspect-schema.ts
|
|
401
440
|
init_cjs_shims();
|
|
441
|
+
function getArrayElement(schema) {
|
|
442
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
443
|
+
return schema.element;
|
|
444
|
+
}
|
|
445
|
+
return null;
|
|
446
|
+
}
|
|
402
447
|
function introspectSchema(schema, prefix = []) {
|
|
403
448
|
const relation = getRelationTarget(schema);
|
|
404
449
|
if (relation && prefix.length > 0) {
|
|
@@ -412,7 +457,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
412
457
|
}
|
|
413
458
|
];
|
|
414
459
|
}
|
|
415
|
-
|
|
460
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
461
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
462
|
+
}
|
|
463
|
+
const base = peelOptionalWrappers(schema);
|
|
416
464
|
if (base instanceof Object && "shape" in base) {
|
|
417
465
|
const shape = base.shape;
|
|
418
466
|
const fields = [];
|
|
@@ -421,9 +469,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
421
469
|
}
|
|
422
470
|
return fields;
|
|
423
471
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
const elementBase =
|
|
472
|
+
const element = getArrayElement(base);
|
|
473
|
+
if (element) {
|
|
474
|
+
const elementBase = peelOptionalWrappers(element);
|
|
427
475
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
428
476
|
const shape = elementBase.shape;
|
|
429
477
|
const fields = [];
|
|
@@ -435,15 +483,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
435
483
|
}
|
|
436
484
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
437
485
|
}
|
|
438
|
-
function
|
|
439
|
-
const
|
|
486
|
+
function buildPathTrie(paths) {
|
|
487
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
440
488
|
for (const path15 of paths) {
|
|
441
|
-
|
|
489
|
+
let node = root;
|
|
490
|
+
for (const segment of path15) {
|
|
491
|
+
let child = node.children.get(segment);
|
|
492
|
+
if (!child) {
|
|
493
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
494
|
+
node.children.set(segment, child);
|
|
495
|
+
}
|
|
496
|
+
node = child;
|
|
497
|
+
}
|
|
498
|
+
node.leaf = true;
|
|
499
|
+
}
|
|
500
|
+
return root;
|
|
501
|
+
}
|
|
502
|
+
function extractNode(data, trie) {
|
|
503
|
+
if (trie.leaf) return data;
|
|
504
|
+
const star = trie.children.get("*");
|
|
505
|
+
if (star) {
|
|
506
|
+
if (!Array.isArray(data)) return void 0;
|
|
507
|
+
return data.map(
|
|
508
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
512
|
+
const record = data;
|
|
513
|
+
const out = {};
|
|
514
|
+
let any = false;
|
|
515
|
+
for (const [key, child] of trie.children) {
|
|
516
|
+
const value = extractNode(record[key], child);
|
|
442
517
|
if (value !== void 0) {
|
|
443
|
-
|
|
518
|
+
out[key] = value;
|
|
519
|
+
any = true;
|
|
444
520
|
}
|
|
445
521
|
}
|
|
446
|
-
return out;
|
|
522
|
+
return any ? out : void 0;
|
|
523
|
+
}
|
|
524
|
+
function extractByPaths(data, paths) {
|
|
525
|
+
if (paths.length === 0) return {};
|
|
526
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
527
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
447
528
|
}
|
|
448
529
|
function pickTranslatable(data, schema) {
|
|
449
530
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -458,32 +539,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
458
539
|
const translatable = pickTranslatable(localeData, schema);
|
|
459
540
|
return deepMerge(structural, translatable);
|
|
460
541
|
}
|
|
461
|
-
function getAtPath(obj, path15) {
|
|
462
|
-
let current = obj;
|
|
463
|
-
for (const segment of path15) {
|
|
464
|
-
if (segment === "*") {
|
|
465
|
-
if (!Array.isArray(current)) return void 0;
|
|
466
|
-
return current.map(
|
|
467
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path15.slice(path15.indexOf("*") + 1)) : void 0
|
|
468
|
-
);
|
|
469
|
-
}
|
|
470
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
471
|
-
current = current[segment];
|
|
472
|
-
}
|
|
473
|
-
return current;
|
|
474
|
-
}
|
|
475
|
-
function setAtPath(obj, path15, value) {
|
|
476
|
-
if (path15.length === 0) return;
|
|
477
|
-
if (path15.length === 1) {
|
|
478
|
-
obj[path15[0]] = value;
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
const [head, ...rest] = path15;
|
|
482
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
483
|
-
obj[head] = {};
|
|
484
|
-
}
|
|
485
|
-
setAtPath(obj[head], rest, value);
|
|
486
|
-
}
|
|
487
542
|
function deepMerge(base, overlay) {
|
|
488
543
|
const out = { ...base };
|
|
489
544
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -774,7 +829,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
774
829
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
775
830
|
const attrStart = i;
|
|
776
831
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
777
|
-
body.slice(attrStart, i);
|
|
832
|
+
const attrName = body.slice(attrStart, i);
|
|
833
|
+
if (!attrName) break;
|
|
778
834
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
779
835
|
if (body[i] !== "=") {
|
|
780
836
|
tagOut += body.slice(attrStart, i);
|
|
@@ -783,10 +839,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
783
839
|
tagOut += body.slice(attrStart, i);
|
|
784
840
|
tagOut += "=";
|
|
785
841
|
i += 1;
|
|
842
|
+
const wsStart = i;
|
|
786
843
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
844
|
+
tagOut += body.slice(wsStart, i);
|
|
787
845
|
const quote = body[i];
|
|
788
846
|
if (quote !== '"' && quote !== "'") {
|
|
789
|
-
|
|
847
|
+
break;
|
|
790
848
|
}
|
|
791
849
|
if (quote === "'") {
|
|
792
850
|
const valStart2 = i + 1;
|
|
@@ -1108,7 +1166,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1108
1166
|
|
|
1109
1167
|
// src/i18n/resolve-document.ts
|
|
1110
1168
|
init_cjs_shims();
|
|
1111
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1169
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1112
1170
|
const urlBuilder = createUrlBuilder({
|
|
1113
1171
|
locales: [defaultLocale, locale],
|
|
1114
1172
|
defaultLocale,
|
|
@@ -1122,15 +1180,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1122
1180
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1123
1181
|
const found = docIdx.bySlug.get(slug);
|
|
1124
1182
|
if (!found) continue;
|
|
1125
|
-
const
|
|
1126
|
-
|
|
1183
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1184
|
+
let target;
|
|
1185
|
+
let targetLocale = locale;
|
|
1186
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1187
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1188
|
+
if (cand) {
|
|
1189
|
+
target = cand;
|
|
1190
|
+
targetLocale = candidateLocale;
|
|
1191
|
+
break;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
if (target) {
|
|
1195
|
+
if (target.slug === slug) {
|
|
1196
|
+
return { document: target, actualLocale: targetLocale };
|
|
1197
|
+
}
|
|
1127
1198
|
if (!type.path) {
|
|
1128
1199
|
return { document: null, actualLocale: locale };
|
|
1129
1200
|
}
|
|
1130
1201
|
return {
|
|
1131
1202
|
document: null,
|
|
1132
1203
|
actualLocale: locale,
|
|
1133
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1204
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1134
1205
|
};
|
|
1135
1206
|
}
|
|
1136
1207
|
if (docLocale === defaultLocale) {
|
|
@@ -1161,13 +1232,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1161
1232
|
}
|
|
1162
1233
|
return { document: null, actualLocale: locale };
|
|
1163
1234
|
}
|
|
1164
|
-
function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1165
|
-
if (sourceLocale === targetLocale) return document2.slug;
|
|
1166
|
-
const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
|
|
1167
|
-
if (!englishSlug) return null;
|
|
1168
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1169
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1170
|
-
}
|
|
1171
1235
|
|
|
1172
1236
|
// src/create-project.ts
|
|
1173
1237
|
function comparatorFor(orderBy) {
|
|
@@ -1220,7 +1284,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1220
1284
|
config.defaultLocale,
|
|
1221
1285
|
load(),
|
|
1222
1286
|
type,
|
|
1223
|
-
config.localeRouting
|
|
1287
|
+
config.localeRouting,
|
|
1288
|
+
config.localeFallbacks[locale] ?? []
|
|
1224
1289
|
);
|
|
1225
1290
|
if (result.document && type.path) {
|
|
1226
1291
|
return {
|
|
@@ -1242,8 +1307,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1242
1307
|
const params = [];
|
|
1243
1308
|
for (const locale of options.locales ?? config.locales) {
|
|
1244
1309
|
const localeIdx = all.get(locale);
|
|
1310
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1245
1311
|
for (const doc of enIdx.bySlug.values()) {
|
|
1246
|
-
|
|
1312
|
+
let slug;
|
|
1313
|
+
if (locale === config.defaultLocale) {
|
|
1314
|
+
slug = doc.slug;
|
|
1315
|
+
} else {
|
|
1316
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1317
|
+
}
|
|
1247
1318
|
params.push({ locale, slug });
|
|
1248
1319
|
}
|
|
1249
1320
|
}
|
|
@@ -1705,7 +1776,7 @@ init_sqlite();
|
|
|
1705
1776
|
|
|
1706
1777
|
// src/validate/validate-relations.ts
|
|
1707
1778
|
init_cjs_shims();
|
|
1708
|
-
function
|
|
1779
|
+
function getAtPath(obj, fieldPath) {
|
|
1709
1780
|
let current = obj;
|
|
1710
1781
|
for (const segment of fieldPath) {
|
|
1711
1782
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -1750,7 +1821,7 @@ function validateRelations(project) {
|
|
|
1750
1821
|
});
|
|
1751
1822
|
continue;
|
|
1752
1823
|
}
|
|
1753
|
-
const value =
|
|
1824
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
1754
1825
|
if (value === void 0 || value === null || value === "") continue;
|
|
1755
1826
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
1756
1827
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -1906,6 +1977,20 @@ function listEnSlugs3(rootDir, contentDir) {
|
|
|
1906
1977
|
if (!fs2__default.default.existsSync(dir)) return [];
|
|
1907
1978
|
return fs2__default.default.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
|
|
1908
1979
|
}
|
|
1980
|
+
function validateDocumentMdxBody(issues, input) {
|
|
1981
|
+
const preparedBody = prepareTranslatedMdxBody(input.body).body;
|
|
1982
|
+
const mdxValidation = validateMdxBody(preparedBody);
|
|
1983
|
+
if (!mdxValidation.ok) {
|
|
1984
|
+
issues.push({
|
|
1985
|
+
level: "error",
|
|
1986
|
+
contentType: input.contentType,
|
|
1987
|
+
enSlug: input.enSlug,
|
|
1988
|
+
locale: input.locale,
|
|
1989
|
+
field: "body",
|
|
1990
|
+
message: `Invalid MDX: ${mdxValidation.error}`
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1909
1994
|
function validateProject(config) {
|
|
1910
1995
|
const issues = [];
|
|
1911
1996
|
const project = createProject(config);
|
|
@@ -1964,6 +2049,12 @@ function validateProject(config) {
|
|
|
1964
2049
|
})) {
|
|
1965
2050
|
issues.push(issue);
|
|
1966
2051
|
}
|
|
2052
|
+
validateDocumentMdxBody(issues, {
|
|
2053
|
+
contentType: type.id,
|
|
2054
|
+
enSlug,
|
|
2055
|
+
locale: config.defaultLocale,
|
|
2056
|
+
body: enDoc.content
|
|
2057
|
+
});
|
|
1967
2058
|
for (const locale of config.locales) {
|
|
1968
2059
|
if (locale === config.defaultLocale) continue;
|
|
1969
2060
|
const row = getTranslation(db, type.id, enSlug, locale);
|
|
@@ -1989,17 +2080,12 @@ function validateProject(config) {
|
|
|
1989
2080
|
})) {
|
|
1990
2081
|
issues.push(issue);
|
|
1991
2082
|
}
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
locale,
|
|
1999
|
-
field: "body",
|
|
2000
|
-
message: `Invalid MDX: ${mdxValidation.error}`
|
|
2001
|
-
});
|
|
2002
|
-
}
|
|
2083
|
+
validateDocumentMdxBody(issues, {
|
|
2084
|
+
contentType: type.id,
|
|
2085
|
+
enSlug,
|
|
2086
|
+
locale,
|
|
2087
|
+
body: row.body
|
|
2088
|
+
});
|
|
2003
2089
|
}
|
|
2004
2090
|
}
|
|
2005
2091
|
try {
|
|
@@ -2119,6 +2205,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2119
2205
|
// src/translate/page-translator.ts
|
|
2120
2206
|
init_cjs_shims();
|
|
2121
2207
|
|
|
2208
|
+
// src/storage/batch-jobs.ts
|
|
2209
|
+
init_cjs_shims();
|
|
2210
|
+
function insertBatchJob(db, input) {
|
|
2211
|
+
const info = db.prepare(
|
|
2212
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2213
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2214
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2215
|
+
return Number(info.lastInsertRowid);
|
|
2216
|
+
}
|
|
2217
|
+
function insertBatchItems(db, jobId, items) {
|
|
2218
|
+
const stmt = db.prepare(
|
|
2219
|
+
`INSERT INTO translation_batch_items (
|
|
2220
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2221
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2222
|
+
);
|
|
2223
|
+
for (const item of items) {
|
|
2224
|
+
stmt.run(
|
|
2225
|
+
jobId,
|
|
2226
|
+
item.requestIndex,
|
|
2227
|
+
item.contentType,
|
|
2228
|
+
item.enSlug,
|
|
2229
|
+
item.locale,
|
|
2230
|
+
item.enHash,
|
|
2231
|
+
item.snapshotId
|
|
2232
|
+
);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
function listPendingBatchJobs(db) {
|
|
2236
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2237
|
+
}
|
|
2238
|
+
function listBatchItems(db, jobId) {
|
|
2239
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2240
|
+
}
|
|
2241
|
+
function listPendingBatchItems(db) {
|
|
2242
|
+
return db.prepare(
|
|
2243
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2244
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2245
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2246
|
+
ORDER BY i.job_id, i.request_index`
|
|
2247
|
+
).all();
|
|
2248
|
+
}
|
|
2249
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2250
|
+
const info = db.prepare(
|
|
2251
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2252
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2253
|
+
).run(state, completedAt, jobId);
|
|
2254
|
+
return info.changes > 0;
|
|
2255
|
+
}
|
|
2256
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2257
|
+
db.prepare(
|
|
2258
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2259
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
// src/translate/page-translator.ts
|
|
2263
|
+
init_sqlite();
|
|
2264
|
+
|
|
2265
|
+
// src/translate/batch-worklist.ts
|
|
2266
|
+
init_cjs_shims();
|
|
2267
|
+
|
|
2122
2268
|
// src/history/record-snapshot.ts
|
|
2123
2269
|
init_cjs_shims();
|
|
2124
2270
|
init_sqlite();
|
|
@@ -2136,9 +2282,111 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2136
2282
|
return id;
|
|
2137
2283
|
}
|
|
2138
2284
|
|
|
2139
|
-
// src/translate/
|
|
2285
|
+
// src/translate/batch-worklist.ts
|
|
2140
2286
|
init_sqlite();
|
|
2141
2287
|
|
|
2288
|
+
// src/translate/gemini-batch.ts
|
|
2289
|
+
init_cjs_shims();
|
|
2290
|
+
|
|
2291
|
+
// src/translate/retry.ts
|
|
2292
|
+
init_cjs_shims();
|
|
2293
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2294
|
+
function statusFromError(error) {
|
|
2295
|
+
if (error instanceof genai.ApiError && typeof error.status === "number") {
|
|
2296
|
+
return error.status;
|
|
2297
|
+
}
|
|
2298
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2299
|
+
const status = error.status;
|
|
2300
|
+
if (typeof status === "number") return status;
|
|
2301
|
+
}
|
|
2302
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2303
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2304
|
+
if (match) return Number(match[1]);
|
|
2305
|
+
return void 0;
|
|
2306
|
+
}
|
|
2307
|
+
function isNetworkError(error) {
|
|
2308
|
+
const err = error;
|
|
2309
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2310
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2311
|
+
return true;
|
|
2312
|
+
}
|
|
2313
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2314
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2315
|
+
}
|
|
2316
|
+
function isTransientError(error) {
|
|
2317
|
+
const status = statusFromError(error);
|
|
2318
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2319
|
+
return isNetworkError(error);
|
|
2320
|
+
}
|
|
2321
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2322
|
+
async function withRetry(fn, options = {}) {
|
|
2323
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2324
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2325
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2326
|
+
const random = options.random ?? Math.random;
|
|
2327
|
+
let lastError;
|
|
2328
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2329
|
+
try {
|
|
2330
|
+
return await fn();
|
|
2331
|
+
} catch (error) {
|
|
2332
|
+
lastError = error;
|
|
2333
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2334
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2335
|
+
const delay = backoff + random() * backoff;
|
|
2336
|
+
await sleep2(delay);
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
throw lastError;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
// src/translate/gemini-batch.ts
|
|
2343
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2344
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2345
|
+
"SUCCEEDED",
|
|
2346
|
+
"FAILED",
|
|
2347
|
+
"CANCELLED",
|
|
2348
|
+
"EXPIRED",
|
|
2349
|
+
"PARTIALLY_SUCCEEDED"
|
|
2350
|
+
]);
|
|
2351
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2352
|
+
function normalizeBatchState(state) {
|
|
2353
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2354
|
+
}
|
|
2355
|
+
function isTerminalBatchState(state) {
|
|
2356
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2357
|
+
}
|
|
2358
|
+
function isSuccessfulBatchState(state) {
|
|
2359
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2360
|
+
}
|
|
2361
|
+
function makeClient(apiKey) {
|
|
2362
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2363
|
+
if (!key) {
|
|
2364
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2365
|
+
}
|
|
2366
|
+
return new genai.GoogleGenAI({ apiKey: key });
|
|
2367
|
+
}
|
|
2368
|
+
function textFromBatchResponse(response) {
|
|
2369
|
+
if (typeof response.text === "string") return response.text;
|
|
2370
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2371
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2372
|
+
}
|
|
2373
|
+
async function createGeminiBatchJob(input) {
|
|
2374
|
+
const ai = makeClient(input.apiKey);
|
|
2375
|
+
const job = await withRetry(
|
|
2376
|
+
() => ai.batches.create({
|
|
2377
|
+
model: input.model,
|
|
2378
|
+
src: input.requests,
|
|
2379
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2380
|
+
})
|
|
2381
|
+
);
|
|
2382
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2383
|
+
return job;
|
|
2384
|
+
}
|
|
2385
|
+
async function getGeminiBatchJob(input) {
|
|
2386
|
+
const ai = makeClient(input.apiKey);
|
|
2387
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2142
2390
|
// src/translate/gemini-client.ts
|
|
2143
2391
|
init_cjs_shims();
|
|
2144
2392
|
|
|
@@ -2163,16 +2411,55 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2163
2411
|
if (alias) return alias[0];
|
|
2164
2412
|
return name;
|
|
2165
2413
|
}
|
|
2414
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2415
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2416
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
|
|
2417
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2418
|
+
return void 0;
|
|
2419
|
+
}
|
|
2166
2420
|
|
|
2167
2421
|
// src/translate/gemini-client.ts
|
|
2168
2422
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
2169
2423
|
function extractJson(text) {
|
|
2170
|
-
const
|
|
2424
|
+
const trimmed = text.trim();
|
|
2425
|
+
const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
|
2171
2426
|
if (fenced?.[1]) return fenced[1].trim();
|
|
2172
|
-
const start =
|
|
2173
|
-
const end =
|
|
2174
|
-
if (start >= 0 && end > start) return
|
|
2175
|
-
return
|
|
2427
|
+
const start = trimmed.indexOf("{");
|
|
2428
|
+
const end = trimmed.lastIndexOf("}");
|
|
2429
|
+
if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
|
|
2430
|
+
return trimmed;
|
|
2431
|
+
}
|
|
2432
|
+
function parseGeminiResponse(text) {
|
|
2433
|
+
let parsed;
|
|
2434
|
+
try {
|
|
2435
|
+
parsed = JSON.parse(text.trim());
|
|
2436
|
+
} catch {
|
|
2437
|
+
parsed = JSON.parse(extractJson(text));
|
|
2438
|
+
}
|
|
2439
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2440
|
+
parsed.frontmatter = {};
|
|
2441
|
+
}
|
|
2442
|
+
return parsed;
|
|
2443
|
+
}
|
|
2444
|
+
function buildGeminiRequestConfig(input) {
|
|
2445
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2446
|
+
return {
|
|
2447
|
+
responseMimeType: "application/json",
|
|
2448
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2449
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
function usageFromResponse(response) {
|
|
2453
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2454
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2455
|
+
return {
|
|
2456
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2457
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2458
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2459
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2460
|
+
thoughtsTokens,
|
|
2461
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2462
|
+
};
|
|
2176
2463
|
}
|
|
2177
2464
|
async function translatePageWithGemini(input) {
|
|
2178
2465
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2184,28 +2471,28 @@ async function translatePageWithGemini(input) {
|
|
|
2184
2471
|
);
|
|
2185
2472
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2186
2473
|
const ai = new genai.GoogleGenAI({ apiKey });
|
|
2187
|
-
const response = await
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2474
|
+
const response = await withRetry(
|
|
2475
|
+
() => ai.models.generateContent({
|
|
2476
|
+
model: apiModel,
|
|
2477
|
+
contents: input.prompt,
|
|
2478
|
+
config: buildGeminiRequestConfig({
|
|
2479
|
+
apiModelId: apiModel,
|
|
2480
|
+
responseSchema: input.responseSchema
|
|
2481
|
+
})
|
|
2482
|
+
})
|
|
2483
|
+
);
|
|
2195
2484
|
const raw = response.text ?? "";
|
|
2196
|
-
const parsed =
|
|
2485
|
+
const parsed = parseGeminiResponse(raw);
|
|
2197
2486
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2198
2487
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2199
2488
|
}
|
|
2200
|
-
|
|
2201
|
-
const usage = {
|
|
2202
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2203
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2204
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2205
|
-
};
|
|
2206
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2489
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2207
2490
|
}
|
|
2208
2491
|
|
|
2492
|
+
// src/translate/translate-core.ts
|
|
2493
|
+
init_cjs_shims();
|
|
2494
|
+
init_sqlite();
|
|
2495
|
+
|
|
2209
2496
|
// src/translate/gemini-pricing.ts
|
|
2210
2497
|
init_cjs_shims();
|
|
2211
2498
|
var CONTEXT_TIER_TOKENS = 2e5;
|
|
@@ -2224,12 +2511,14 @@ function resolveModelPricing(model) {
|
|
|
2224
2511
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2225
2512
|
return match?.[1];
|
|
2226
2513
|
}
|
|
2227
|
-
|
|
2514
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2515
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2228
2516
|
const pricing = resolveModelPricing(model);
|
|
2229
2517
|
if (!pricing) return void 0;
|
|
2230
2518
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2231
2519
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2232
|
-
|
|
2520
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2521
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2233
2522
|
}
|
|
2234
2523
|
function formatTokenCount(tokens) {
|
|
2235
2524
|
if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
|
|
@@ -2257,6 +2546,7 @@ var LOCALE_NAMES = {
|
|
|
2257
2546
|
"zh-CN": "Simplified Chinese",
|
|
2258
2547
|
"zh-TW": "Traditional Chinese",
|
|
2259
2548
|
pt: "Portuguese",
|
|
2549
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2260
2550
|
ja: "Japanese",
|
|
2261
2551
|
ru: "Russian",
|
|
2262
2552
|
it: "Italian",
|
|
@@ -2264,29 +2554,29 @@ var LOCALE_NAMES = {
|
|
|
2264
2554
|
};
|
|
2265
2555
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2266
2556
|
return [
|
|
2267
|
-
`Localize the content
|
|
2557
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2268
2558
|
"Do not translate word-for-word.",
|
|
2269
2559
|
"Preserve the source tone and brand voice.",
|
|
2270
2560
|
"Write as if a native speaker authored it for the target market."
|
|
2271
2561
|
].join(" ");
|
|
2272
2562
|
}
|
|
2563
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2564
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2565
|
+
if (!hasFrontmatter) {
|
|
2566
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2567
|
+
}
|
|
2568
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2569
|
+
}
|
|
2273
2570
|
function buildPageTranslationPrompt(input) {
|
|
2274
2571
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2275
|
-
const
|
|
2276
|
-
const
|
|
2277
|
-
|
|
2572
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2573
|
+
const prefix = [
|
|
2574
|
+
TASK_FRAMING,
|
|
2278
2575
|
"",
|
|
2279
2576
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2280
2577
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2281
2578
|
"## Rules",
|
|
2282
2579
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2283
|
-
...input.slugStrategy === "localized" ? [
|
|
2284
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2285
|
-
] : [],
|
|
2286
|
-
"",
|
|
2287
|
-
"## Output format",
|
|
2288
|
-
"Return ONLY valid JSON with keys:",
|
|
2289
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2290
2580
|
"",
|
|
2291
2581
|
"## EN translatable frontmatter (JSON)",
|
|
2292
2582
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2294,7 +2584,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2294
2584
|
"## EN body (MDX)",
|
|
2295
2585
|
input.enBody
|
|
2296
2586
|
];
|
|
2297
|
-
|
|
2587
|
+
const suffix = [
|
|
2588
|
+
"",
|
|
2589
|
+
"## Target language",
|
|
2590
|
+
localizationPrompt,
|
|
2591
|
+
...input.slugStrategy === "localized" ? [
|
|
2592
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2593
|
+
] : [],
|
|
2594
|
+
"",
|
|
2595
|
+
"## Output format",
|
|
2596
|
+
"Return ONLY valid JSON with keys:",
|
|
2597
|
+
buildOutputFormatLine(
|
|
2598
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2599
|
+
input.slugStrategy
|
|
2600
|
+
)
|
|
2601
|
+
];
|
|
2602
|
+
return [...prefix, ...suffix].join("\n");
|
|
2298
2603
|
}
|
|
2299
2604
|
|
|
2300
2605
|
// src/translate/response-schema.ts
|
|
@@ -2345,9 +2650,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2345
2650
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2346
2651
|
try {
|
|
2347
2652
|
const translatable = buildTranslatableSubschema(schema);
|
|
2348
|
-
if (!translatable) return null;
|
|
2349
2653
|
const responseShape = {
|
|
2350
|
-
frontmatter: translatable,
|
|
2654
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2351
2655
|
body: zod.z.string()
|
|
2352
2656
|
};
|
|
2353
2657
|
if (slugStrategy === "localized") {
|
|
@@ -2428,7 +2732,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2428
2732
|
return { ok: true, frontmatter };
|
|
2429
2733
|
}
|
|
2430
2734
|
|
|
2431
|
-
// src/translate/
|
|
2735
|
+
// src/translate/translate-core.ts
|
|
2736
|
+
function translationItemKey(item) {
|
|
2737
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2738
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2739
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2740
|
+
}
|
|
2432
2741
|
function summarizeResults(results, durationMs) {
|
|
2433
2742
|
return results.reduce(
|
|
2434
2743
|
(totals, result) => {
|
|
@@ -2464,17 +2773,6 @@ function formatTranslateError(error) {
|
|
|
2464
2773
|
}
|
|
2465
2774
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2466
2775
|
}
|
|
2467
|
-
async function runPool(items, concurrency, worker) {
|
|
2468
|
-
if (items.length === 0) return;
|
|
2469
|
-
let nextIndex = 0;
|
|
2470
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2471
|
-
while (nextIndex < items.length) {
|
|
2472
|
-
const index = nextIndex++;
|
|
2473
|
-
await worker(items[index], index);
|
|
2474
|
-
}
|
|
2475
|
-
});
|
|
2476
|
-
await Promise.all(runners);
|
|
2477
|
-
}
|
|
2478
2776
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2479
2777
|
const fm = enDoc.frontmatter;
|
|
2480
2778
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2483,73 +2781,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2483
2781
|
}
|
|
2484
2782
|
return enSlug;
|
|
2485
2783
|
}
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
const base = {
|
|
2784
|
+
function baseForItem(item) {
|
|
2785
|
+
return {
|
|
2489
2786
|
contentType: item.contentType,
|
|
2490
2787
|
enSlug: item.enSlug,
|
|
2491
2788
|
locale: item.locale
|
|
2492
2789
|
};
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2790
|
+
}
|
|
2791
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
2792
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
2793
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
2794
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
2795
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
2796
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
2797
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
2798
|
+
const db = openStore(config, "readonly");
|
|
2799
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
2800
|
+
db.close();
|
|
2801
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
2802
|
+
return {
|
|
2803
|
+
status: "done",
|
|
2804
|
+
result: {
|
|
2805
|
+
...baseForItem(item),
|
|
2506
2806
|
skipped: true,
|
|
2507
2807
|
reason: "fresh",
|
|
2508
2808
|
durationMs: Date.now() - startedAt
|
|
2509
|
-
}
|
|
2510
|
-
}
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2809
|
+
}
|
|
2810
|
+
};
|
|
2811
|
+
}
|
|
2812
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
2813
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
2814
|
+
const prompt = buildPageTranslationPrompt({
|
|
2815
|
+
resolved: resolvedTranslate,
|
|
2816
|
+
targetLocale: item.locale,
|
|
2817
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
2818
|
+
translatableFrontmatter: payload.frontmatter,
|
|
2819
|
+
enBody: payload.body,
|
|
2820
|
+
slugStrategy: type.slugStrategy
|
|
2821
|
+
});
|
|
2822
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
2823
|
+
return {
|
|
2824
|
+
status: "ready",
|
|
2825
|
+
prepared: {
|
|
2826
|
+
item,
|
|
2827
|
+
type,
|
|
2828
|
+
enDoc,
|
|
2829
|
+
payload,
|
|
2830
|
+
currentEnHash,
|
|
2831
|
+
existingSlug: existing?.slug,
|
|
2532
2832
|
model,
|
|
2833
|
+
prompt,
|
|
2533
2834
|
responseSchema: responseSchema ?? void 0
|
|
2534
|
-
}
|
|
2535
|
-
|
|
2835
|
+
}
|
|
2836
|
+
};
|
|
2837
|
+
}
|
|
2838
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
2839
|
+
const { item, type, enDoc, payload } = prepared;
|
|
2840
|
+
const base = baseForItem(item);
|
|
2841
|
+
try {
|
|
2842
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2536
2843
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2537
2844
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2538
2845
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2539
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
2846
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2540
2847
|
if (!validated.ok) {
|
|
2541
2848
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2542
2849
|
}
|
|
2543
2850
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2544
|
-
|
|
2851
|
+
output.parsed.body
|
|
2545
2852
|
);
|
|
2546
2853
|
const writeDb = openStore(config, "readwrite");
|
|
2547
|
-
const snapshotId = recordEnSnapshot(
|
|
2854
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2548
2855
|
config,
|
|
2549
2856
|
{
|
|
2550
2857
|
contentType: type.id,
|
|
2551
2858
|
enSlug: item.enSlug,
|
|
2552
|
-
enHash: currentEnHash,
|
|
2859
|
+
enHash: prepared.currentEnHash,
|
|
2553
2860
|
frontmatter: payload.frontmatter,
|
|
2554
2861
|
body: payload.body
|
|
2555
2862
|
},
|
|
@@ -2562,24 +2869,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2562
2869
|
slug,
|
|
2563
2870
|
frontmatter: validated.frontmatter,
|
|
2564
2871
|
body: translatedBody,
|
|
2565
|
-
enHash: currentEnHash,
|
|
2872
|
+
enHash: prepared.currentEnHash,
|
|
2566
2873
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2567
|
-
model:
|
|
2874
|
+
model: output.model,
|
|
2568
2875
|
snapshotId
|
|
2569
2876
|
});
|
|
2570
2877
|
writeDb.close();
|
|
2571
2878
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2572
|
-
normalizeGeminiDisplayName(
|
|
2573
|
-
|
|
2574
|
-
|
|
2879
|
+
normalizeGeminiDisplayName(output.model),
|
|
2880
|
+
output.usage.inputTokens,
|
|
2881
|
+
output.usage.outputTokens,
|
|
2882
|
+
options.costMode
|
|
2575
2883
|
);
|
|
2576
2884
|
return {
|
|
2577
2885
|
...base,
|
|
2578
2886
|
skipped: false,
|
|
2579
|
-
model:
|
|
2580
|
-
usage:
|
|
2887
|
+
model: output.model,
|
|
2888
|
+
usage: output.usage,
|
|
2581
2889
|
estimatedCostUsd,
|
|
2582
|
-
durationMs: Date.now() - startedAt,
|
|
2890
|
+
durationMs: Date.now() - options.startedAt,
|
|
2583
2891
|
slugAdjusted,
|
|
2584
2892
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2585
2893
|
};
|
|
@@ -2589,33 +2897,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2589
2897
|
skipped: false,
|
|
2590
2898
|
failed: true,
|
|
2591
2899
|
error: formatTranslateError(error),
|
|
2900
|
+
durationMs: Date.now() - options.startedAt
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
}
|
|
2904
|
+
function displayModelFor(prepared) {
|
|
2905
|
+
return normalizeGeminiDisplayName(
|
|
2906
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
2907
|
+
);
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
// src/translate/batch-worklist.ts
|
|
2911
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
2912
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
2913
|
+
var INITIAL_POLL_MS = 5e3;
|
|
2914
|
+
var MAX_POLL_MS = 3e4;
|
|
2915
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
2916
|
+
function sleep(ms) {
|
|
2917
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2918
|
+
}
|
|
2919
|
+
function planBatchJobs(entries) {
|
|
2920
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
2921
|
+
for (const entry of entries) {
|
|
2922
|
+
const group = byModel.get(entry.apiModel);
|
|
2923
|
+
if (group) group.push(entry);
|
|
2924
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
2925
|
+
}
|
|
2926
|
+
const plans = [];
|
|
2927
|
+
for (const [apiModel, group] of byModel) {
|
|
2928
|
+
let current = [];
|
|
2929
|
+
let currentBytes = 0;
|
|
2930
|
+
for (const entry of group) {
|
|
2931
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
2932
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
2933
|
+
plans.push({ apiModel, entries: current });
|
|
2934
|
+
current = [];
|
|
2935
|
+
currentBytes = 0;
|
|
2936
|
+
}
|
|
2937
|
+
current.push(entry);
|
|
2938
|
+
currentBytes += size;
|
|
2939
|
+
}
|
|
2940
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
2941
|
+
}
|
|
2942
|
+
return plans;
|
|
2943
|
+
}
|
|
2944
|
+
function readPendingBatchWork(config) {
|
|
2945
|
+
const db = openStore(config, "readwrite");
|
|
2946
|
+
const jobs = listPendingBatchJobs(db);
|
|
2947
|
+
const pendingItems = listPendingBatchItems(db);
|
|
2948
|
+
db.close();
|
|
2949
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
2950
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
2951
|
+
}
|
|
2952
|
+
async function submitBatchJobPlan(config, input) {
|
|
2953
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
2954
|
+
const job = await createGeminiBatchJob({
|
|
2955
|
+
model: plan.apiModel,
|
|
2956
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
2957
|
+
contents: prepared.prompt,
|
|
2958
|
+
config: buildGeminiRequestConfig({
|
|
2959
|
+
apiModelId: plan.apiModel,
|
|
2960
|
+
responseSchema: prepared.responseSchema
|
|
2961
|
+
})
|
|
2962
|
+
})),
|
|
2963
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
2964
|
+
});
|
|
2965
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2966
|
+
const db = openStore(config, "readwrite");
|
|
2967
|
+
const jobId = insertBatchJob(db, {
|
|
2968
|
+
jobName: job.name,
|
|
2969
|
+
model: plan.apiModel,
|
|
2970
|
+
displayModel,
|
|
2971
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2972
|
+
createdAt
|
|
2973
|
+
});
|
|
2974
|
+
insertBatchItems(
|
|
2975
|
+
db,
|
|
2976
|
+
jobId,
|
|
2977
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
2978
|
+
requestIndex,
|
|
2979
|
+
contentType: prepared.item.contentType,
|
|
2980
|
+
enSlug: prepared.item.enSlug,
|
|
2981
|
+
locale: prepared.item.locale,
|
|
2982
|
+
enHash: prepared.currentEnHash,
|
|
2983
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
2984
|
+
// on the EN files still matching (or existing) on disk.
|
|
2985
|
+
snapshotId: recordEnSnapshot(
|
|
2986
|
+
config,
|
|
2987
|
+
{
|
|
2988
|
+
contentType: prepared.item.contentType,
|
|
2989
|
+
enSlug: prepared.item.enSlug,
|
|
2990
|
+
enHash: prepared.currentEnHash,
|
|
2991
|
+
frontmatter: prepared.payload.frontmatter,
|
|
2992
|
+
body: prepared.payload.body
|
|
2993
|
+
},
|
|
2994
|
+
db
|
|
2995
|
+
)
|
|
2996
|
+
}))
|
|
2997
|
+
);
|
|
2998
|
+
const row = {
|
|
2999
|
+
id: jobId,
|
|
3000
|
+
job_name: job.name,
|
|
3001
|
+
model: plan.apiModel,
|
|
3002
|
+
display_model: displayModel,
|
|
3003
|
+
created_at: createdAt,
|
|
3004
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3005
|
+
completed_at: null
|
|
3006
|
+
};
|
|
3007
|
+
db.close();
|
|
3008
|
+
input.onProgress?.({
|
|
3009
|
+
type: "batch-submitted",
|
|
3010
|
+
name: job.name,
|
|
3011
|
+
count: plan.entries.length,
|
|
3012
|
+
model: displayModel,
|
|
3013
|
+
jobIndex,
|
|
3014
|
+
jobCount,
|
|
3015
|
+
createdAt
|
|
3016
|
+
});
|
|
3017
|
+
return row;
|
|
3018
|
+
}
|
|
3019
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3020
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3021
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3022
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3023
|
+
if (!enDoc) {
|
|
3024
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3025
|
+
if (!snapshot) {
|
|
3026
|
+
throw new Error(
|
|
3027
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3028
|
+
);
|
|
3029
|
+
}
|
|
3030
|
+
enDoc = {
|
|
3031
|
+
slug: itemRow.en_slug,
|
|
3032
|
+
enSlug: itemRow.en_slug,
|
|
3033
|
+
locale: config.defaultLocale,
|
|
3034
|
+
noindex: false,
|
|
3035
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3036
|
+
content: snapshot.body
|
|
3037
|
+
};
|
|
3038
|
+
}
|
|
3039
|
+
const item = {
|
|
3040
|
+
contentType: itemRow.content_type,
|
|
3041
|
+
enSlug: itemRow.en_slug,
|
|
3042
|
+
locale: itemRow.locale,
|
|
3043
|
+
reason: "missing",
|
|
3044
|
+
currentEnHash: itemRow.en_hash
|
|
3045
|
+
};
|
|
3046
|
+
return {
|
|
3047
|
+
item,
|
|
3048
|
+
type,
|
|
3049
|
+
enDoc,
|
|
3050
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3051
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3052
|
+
currentEnHash: itemRow.en_hash,
|
|
3053
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3054
|
+
model: jobRow.display_model,
|
|
3055
|
+
prompt: "",
|
|
3056
|
+
responseSchema: void 0
|
|
3057
|
+
};
|
|
3058
|
+
}
|
|
3059
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3060
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3061
|
+
const db = openStore(config, "readwrite");
|
|
3062
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3063
|
+
db.close();
|
|
3064
|
+
return null;
|
|
3065
|
+
}
|
|
3066
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3067
|
+
const results = [];
|
|
3068
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3069
|
+
const result = {
|
|
3070
|
+
contentType: itemRow.content_type,
|
|
3071
|
+
enSlug: itemRow.en_slug,
|
|
3072
|
+
locale: itemRow.locale,
|
|
3073
|
+
skipped: false,
|
|
3074
|
+
failed: true,
|
|
3075
|
+
error: formatTranslateError(new Error(message)),
|
|
2592
3076
|
durationMs: Date.now() - startedAt
|
|
2593
3077
|
};
|
|
3078
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3079
|
+
results.push(result);
|
|
3080
|
+
onResult?.(result);
|
|
3081
|
+
};
|
|
3082
|
+
if (isSuccessfulBatchState(state)) {
|
|
3083
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3084
|
+
for (const itemRow of pendingItems) {
|
|
3085
|
+
const startedAt = Date.now();
|
|
3086
|
+
const inlined = responses[itemRow.request_index];
|
|
3087
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3088
|
+
failItem(
|
|
3089
|
+
itemRow,
|
|
3090
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3091
|
+
startedAt
|
|
3092
|
+
);
|
|
3093
|
+
continue;
|
|
3094
|
+
}
|
|
3095
|
+
let result;
|
|
3096
|
+
try {
|
|
3097
|
+
const response = inlined.response;
|
|
3098
|
+
const raw = textFromBatchResponse(response);
|
|
3099
|
+
const parsed = parseGeminiResponse(raw);
|
|
3100
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3101
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3102
|
+
}
|
|
3103
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3104
|
+
result = finalizeTranslation(
|
|
3105
|
+
config,
|
|
3106
|
+
prepared,
|
|
3107
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3108
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3109
|
+
);
|
|
3110
|
+
} catch (error) {
|
|
3111
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3112
|
+
continue;
|
|
3113
|
+
}
|
|
3114
|
+
updateBatchItemStatus(
|
|
3115
|
+
db,
|
|
3116
|
+
jobRow.id,
|
|
3117
|
+
itemRow.request_index,
|
|
3118
|
+
result.failed ? "failed" : "done",
|
|
3119
|
+
result.error
|
|
3120
|
+
);
|
|
3121
|
+
results.push(result);
|
|
3122
|
+
onResult?.(result);
|
|
3123
|
+
}
|
|
3124
|
+
} else {
|
|
3125
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3126
|
+
const startedAt = Date.now();
|
|
3127
|
+
for (const itemRow of pendingItems) {
|
|
3128
|
+
failItem(itemRow, message, startedAt);
|
|
3129
|
+
}
|
|
2594
3130
|
}
|
|
3131
|
+
db.close();
|
|
3132
|
+
return results;
|
|
3133
|
+
}
|
|
3134
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3135
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3136
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3137
|
+
const loopStartedAt = Date.now();
|
|
3138
|
+
let delay = INITIAL_POLL_MS;
|
|
3139
|
+
let firstRound = true;
|
|
3140
|
+
const elapsedFor = (row) => {
|
|
3141
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3142
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3143
|
+
};
|
|
3144
|
+
while (active.length > 0) {
|
|
3145
|
+
if (!firstRound) {
|
|
3146
|
+
await sleep(delay);
|
|
3147
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3148
|
+
}
|
|
3149
|
+
firstRound = false;
|
|
3150
|
+
for (const tracked of [...active]) {
|
|
3151
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3152
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3153
|
+
options.onProgress?.({
|
|
3154
|
+
type: "batch-polling",
|
|
3155
|
+
name: tracked.row.job_name,
|
|
3156
|
+
state,
|
|
3157
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3158
|
+
jobIndex: tracked.jobIndex,
|
|
3159
|
+
jobCount
|
|
3160
|
+
});
|
|
3161
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3162
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3163
|
+
const results = ingestBatchJob(
|
|
3164
|
+
config,
|
|
3165
|
+
tracked.row,
|
|
3166
|
+
batchJob,
|
|
3167
|
+
options.onResult
|
|
3168
|
+
);
|
|
3169
|
+
if (results === null) {
|
|
3170
|
+
options.onProgress?.({
|
|
3171
|
+
type: "batch-done",
|
|
3172
|
+
name: tracked.row.job_name,
|
|
3173
|
+
state,
|
|
3174
|
+
model: tracked.row.display_model,
|
|
3175
|
+
count: 0,
|
|
3176
|
+
jobIndex: tracked.jobIndex,
|
|
3177
|
+
jobCount,
|
|
3178
|
+
translated: 0,
|
|
3179
|
+
failed: 0,
|
|
3180
|
+
inputTokens: 0,
|
|
3181
|
+
outputTokens: 0,
|
|
3182
|
+
estimatedCostUsd: 0,
|
|
3183
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3184
|
+
alreadyIngested: true
|
|
3185
|
+
});
|
|
3186
|
+
continue;
|
|
3187
|
+
}
|
|
3188
|
+
options.onProgress?.({
|
|
3189
|
+
type: "batch-done",
|
|
3190
|
+
name: tracked.row.job_name,
|
|
3191
|
+
state,
|
|
3192
|
+
model: tracked.row.display_model,
|
|
3193
|
+
count: results.length,
|
|
3194
|
+
jobIndex: tracked.jobIndex,
|
|
3195
|
+
jobCount,
|
|
3196
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3197
|
+
failed: results.filter((r) => r.failed).length,
|
|
3198
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3199
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3200
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3201
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3202
|
+
});
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3208
|
+
return {
|
|
3209
|
+
...baseForItem(item),
|
|
3210
|
+
skipped: false,
|
|
3211
|
+
failed: true,
|
|
3212
|
+
error: formatTranslateError(error),
|
|
3213
|
+
durationMs: Date.now() - startedAt
|
|
3214
|
+
};
|
|
3215
|
+
}
|
|
3216
|
+
|
|
3217
|
+
// src/translate/page-translator.ts
|
|
3218
|
+
async function runPool(items, concurrency, worker) {
|
|
3219
|
+
if (items.length === 0) return;
|
|
3220
|
+
let nextIndex = 0;
|
|
3221
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3222
|
+
while (nextIndex < items.length) {
|
|
3223
|
+
const index = nextIndex++;
|
|
3224
|
+
await worker(items[index], index);
|
|
3225
|
+
}
|
|
3226
|
+
});
|
|
3227
|
+
await Promise.all(runners);
|
|
2595
3228
|
}
|
|
2596
3229
|
function labelForItem(item) {
|
|
2597
3230
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2598
3231
|
}
|
|
3232
|
+
async function translatePage(config, item, options = {}) {
|
|
3233
|
+
const startedAt = Date.now();
|
|
3234
|
+
let outcome;
|
|
3235
|
+
try {
|
|
3236
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3237
|
+
} catch (error) {
|
|
3238
|
+
return failedResultForItem(item, error, startedAt);
|
|
3239
|
+
}
|
|
3240
|
+
if (outcome.status === "done") return outcome.result;
|
|
3241
|
+
const prepared = outcome.prepared;
|
|
3242
|
+
if (options.dryRun) {
|
|
3243
|
+
return {
|
|
3244
|
+
...baseForItem(item),
|
|
3245
|
+
skipped: false,
|
|
3246
|
+
model: prepared.model,
|
|
3247
|
+
durationMs: Date.now() - startedAt
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
try {
|
|
3251
|
+
const result = await translatePageWithGemini({
|
|
3252
|
+
prompt: prepared.prompt,
|
|
3253
|
+
model: prepared.model,
|
|
3254
|
+
responseSchema: prepared.responseSchema
|
|
3255
|
+
});
|
|
3256
|
+
return finalizeTranslation(
|
|
3257
|
+
config,
|
|
3258
|
+
prepared,
|
|
3259
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3260
|
+
{ costMode: "interactive", startedAt }
|
|
3261
|
+
);
|
|
3262
|
+
} catch (error) {
|
|
3263
|
+
return failedResultForItem(item, error, startedAt);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
function createResultCollector(items, onProgress) {
|
|
3267
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3268
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3269
|
+
const slots = new Array(items.length);
|
|
3270
|
+
const extras = [];
|
|
3271
|
+
return {
|
|
3272
|
+
emit(result) {
|
|
3273
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3274
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3275
|
+
else extras.push(result);
|
|
3276
|
+
onProgress?.({ type: "item-done", result });
|
|
3277
|
+
},
|
|
3278
|
+
assemble() {
|
|
3279
|
+
return [
|
|
3280
|
+
...slots.filter((result) => result !== void 0),
|
|
3281
|
+
...extras
|
|
3282
|
+
];
|
|
3283
|
+
}
|
|
3284
|
+
};
|
|
3285
|
+
}
|
|
2599
3286
|
async function translateWorklist(config, items, options = {}) {
|
|
3287
|
+
const mode = options.mode ?? "batch";
|
|
2600
3288
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2601
3289
|
const startedAt = Date.now();
|
|
2602
|
-
|
|
2603
|
-
|
|
3290
|
+
if (options.dryRun) {
|
|
3291
|
+
options.onProgress?.({
|
|
3292
|
+
type: "start",
|
|
3293
|
+
total: items.length,
|
|
3294
|
+
concurrency,
|
|
3295
|
+
dryRun: true,
|
|
3296
|
+
model: options.model,
|
|
3297
|
+
mode
|
|
3298
|
+
});
|
|
3299
|
+
const results2 = items.map((item) => {
|
|
3300
|
+
const itemStartedAt = Date.now();
|
|
3301
|
+
try {
|
|
3302
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3303
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3304
|
+
...baseForItem(item),
|
|
3305
|
+
skipped: false,
|
|
3306
|
+
model: outcome.prepared.model,
|
|
3307
|
+
durationMs: Date.now() - itemStartedAt
|
|
3308
|
+
};
|
|
3309
|
+
} catch (error) {
|
|
3310
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3311
|
+
}
|
|
3312
|
+
});
|
|
3313
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3314
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3315
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3316
|
+
return results2;
|
|
3317
|
+
}
|
|
3318
|
+
const pending = readPendingBatchWork(config);
|
|
3319
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3320
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3321
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3322
|
+
).length;
|
|
2604
3323
|
options.onProgress?.({
|
|
2605
3324
|
type: "start",
|
|
2606
|
-
total: items.length,
|
|
3325
|
+
total: items.length + resumedExtraCount,
|
|
2607
3326
|
concurrency,
|
|
2608
|
-
dryRun:
|
|
2609
|
-
model: options.model
|
|
3327
|
+
dryRun: false,
|
|
3328
|
+
model: options.model,
|
|
3329
|
+
mode
|
|
3330
|
+
});
|
|
3331
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3332
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3333
|
+
if (mode === "direct") {
|
|
3334
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3335
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3336
|
+
const active = /* @__PURE__ */ new Set();
|
|
3337
|
+
await Promise.all([
|
|
3338
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3339
|
+
const label = labelForItem(item);
|
|
3340
|
+
active.add(label);
|
|
3341
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3342
|
+
const result = await translatePage(config, item, options);
|
|
3343
|
+
active.delete(label);
|
|
3344
|
+
collector.emit(result);
|
|
3345
|
+
}),
|
|
3346
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3347
|
+
onProgress: options.onProgress,
|
|
3348
|
+
onResult: collector.emit
|
|
3349
|
+
}) : Promise.resolve()
|
|
3350
|
+
]);
|
|
3351
|
+
} else {
|
|
3352
|
+
const entries = [];
|
|
3353
|
+
for (const item of workItems) {
|
|
3354
|
+
const itemStartedAt = Date.now();
|
|
3355
|
+
try {
|
|
3356
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3357
|
+
if (outcome.status === "done") {
|
|
3358
|
+
collector.emit(outcome.result);
|
|
3359
|
+
} else {
|
|
3360
|
+
entries.push({
|
|
3361
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3362
|
+
prompt: outcome.prepared.prompt,
|
|
3363
|
+
prepared: outcome.prepared
|
|
3364
|
+
});
|
|
3365
|
+
}
|
|
3366
|
+
} catch (error) {
|
|
3367
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
const plans = planBatchJobs(entries);
|
|
3371
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3372
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3373
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3374
|
+
const submitted = await Promise.all(
|
|
3375
|
+
plans.map(async (plan, planIndex) => {
|
|
3376
|
+
try {
|
|
3377
|
+
return await submitBatchJobPlan(config, {
|
|
3378
|
+
plan,
|
|
3379
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3380
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3381
|
+
jobCount,
|
|
3382
|
+
onProgress: options.onProgress
|
|
3383
|
+
});
|
|
3384
|
+
} catch (error) {
|
|
3385
|
+
for (const entry of plan.entries) {
|
|
3386
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3387
|
+
}
|
|
3388
|
+
return void 0;
|
|
3389
|
+
}
|
|
3390
|
+
})
|
|
3391
|
+
);
|
|
3392
|
+
const jobsToPoll = [
|
|
3393
|
+
...pending.jobs,
|
|
3394
|
+
...submitted.filter((row) => row !== void 0)
|
|
3395
|
+
];
|
|
3396
|
+
if (jobsToPoll.length > 0) {
|
|
3397
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3398
|
+
jobCount,
|
|
3399
|
+
onProgress: options.onProgress,
|
|
3400
|
+
onResult: collector.emit
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
const results = collector.assemble();
|
|
3405
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3406
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3407
|
+
return results;
|
|
3408
|
+
}
|
|
3409
|
+
function countPendingItems(config, jobs) {
|
|
3410
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3411
|
+
const db = openStore(config, "readonly");
|
|
3412
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3413
|
+
for (const job of jobs) {
|
|
3414
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3415
|
+
}
|
|
3416
|
+
db.close();
|
|
3417
|
+
return counts;
|
|
3418
|
+
}
|
|
3419
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3420
|
+
jobs.forEach((job, jobIndex) => {
|
|
3421
|
+
onProgress?.({
|
|
3422
|
+
type: "batch-submitted",
|
|
3423
|
+
name: job.job_name,
|
|
3424
|
+
count: counts.get(job.id) ?? 0,
|
|
3425
|
+
model: job.display_model,
|
|
3426
|
+
jobIndex,
|
|
3427
|
+
jobCount,
|
|
3428
|
+
resumed: true,
|
|
3429
|
+
createdAt: job.created_at
|
|
3430
|
+
});
|
|
3431
|
+
});
|
|
3432
|
+
}
|
|
3433
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3434
|
+
const startedAt = Date.now();
|
|
3435
|
+
const pending = readPendingBatchWork(config);
|
|
3436
|
+
if (pending.jobs.length === 0) return null;
|
|
3437
|
+
options.onProgress?.({
|
|
3438
|
+
type: "start",
|
|
3439
|
+
total: pending.pendingItems.length,
|
|
3440
|
+
concurrency: 1,
|
|
3441
|
+
dryRun: false,
|
|
3442
|
+
mode: "batch"
|
|
2610
3443
|
});
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
3444
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3445
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3446
|
+
const results = [];
|
|
3447
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3448
|
+
onProgress: options.onProgress,
|
|
3449
|
+
onResult: (result) => {
|
|
3450
|
+
results.push(result);
|
|
3451
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3452
|
+
}
|
|
2619
3453
|
});
|
|
2620
3454
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2621
3455
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3233,35 +4067,43 @@ async function promptContentType(config) {
|
|
|
3233
4067
|
})
|
|
3234
4068
|
);
|
|
3235
4069
|
}
|
|
4070
|
+
var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
|
|
4071
|
+
var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
|
|
4072
|
+
async function promptManualLocales(config) {
|
|
4073
|
+
const locales = nonDefaultLocales(config);
|
|
4074
|
+
if (locales.length <= 1) return {};
|
|
4075
|
+
const picked = await runPrompt(
|
|
4076
|
+
prompts.checkbox({
|
|
4077
|
+
message: "Locales to translate",
|
|
4078
|
+
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
4079
|
+
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
4080
|
+
})
|
|
4081
|
+
);
|
|
4082
|
+
return picked.length === locales.length ? {} : { locale: picked };
|
|
4083
|
+
}
|
|
3236
4084
|
async function promptLocalePreset(config) {
|
|
3237
4085
|
const presets = Object.entries(config.localePresets ?? {}).filter(
|
|
3238
4086
|
(entry) => Array.isArray(entry[1])
|
|
3239
4087
|
);
|
|
3240
4088
|
if (presets.length === 0) {
|
|
3241
|
-
|
|
3242
|
-
if (locales.length <= 1) return {};
|
|
3243
|
-
const picked = await runPrompt(
|
|
3244
|
-
prompts.checkbox({
|
|
3245
|
-
message: "Locales to translate",
|
|
3246
|
-
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
3247
|
-
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
3248
|
-
})
|
|
3249
|
-
);
|
|
3250
|
-
return picked.length === locales.length ? {} : { locale: picked };
|
|
4089
|
+
return promptManualLocales(config);
|
|
3251
4090
|
}
|
|
3252
4091
|
const choice = await runPrompt(
|
|
3253
4092
|
prompts.select({
|
|
3254
4093
|
message: "Locale preset",
|
|
3255
4094
|
choices: [
|
|
3256
|
-
{ name: "All locales", value:
|
|
4095
|
+
{ name: "All locales", value: ALL_LOCALES },
|
|
3257
4096
|
...presets.map(([name, locales]) => ({
|
|
3258
4097
|
name: `${name} (${locales.join(", ")})`,
|
|
3259
4098
|
value: name
|
|
3260
|
-
}))
|
|
4099
|
+
})),
|
|
4100
|
+
{ name: "Choose locales manually\u2026", value: PICK_LOCALES }
|
|
3261
4101
|
]
|
|
3262
4102
|
})
|
|
3263
4103
|
);
|
|
3264
|
-
|
|
4104
|
+
if (choice === PICK_LOCALES) return promptManualLocales(config);
|
|
4105
|
+
if (choice === ALL_LOCALES) return {};
|
|
4106
|
+
return { preset: choice };
|
|
3265
4107
|
}
|
|
3266
4108
|
async function promptStrategy() {
|
|
3267
4109
|
return runPrompt(
|
|
@@ -3275,12 +4117,25 @@ async function promptStrategy() {
|
|
|
3275
4117
|
})
|
|
3276
4118
|
);
|
|
3277
4119
|
}
|
|
4120
|
+
async function promptMode() {
|
|
4121
|
+
return runPrompt(
|
|
4122
|
+
prompts.select({
|
|
4123
|
+
message: "Translation mode",
|
|
4124
|
+
choices: [
|
|
4125
|
+
{ name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
|
|
4126
|
+
{ name: "Direct \u2014 immediate results, full price", value: "direct" }
|
|
4127
|
+
],
|
|
4128
|
+
default: "batch"
|
|
4129
|
+
})
|
|
4130
|
+
);
|
|
4131
|
+
}
|
|
3278
4132
|
async function promptTranslateSelection(config, flags) {
|
|
3279
4133
|
const selection = {
|
|
3280
4134
|
contentType: flags.type,
|
|
3281
4135
|
preset: flags.preset,
|
|
3282
4136
|
locale: flags.locale,
|
|
3283
|
-
strategy: flags.strategy
|
|
4137
|
+
strategy: flags.strategy,
|
|
4138
|
+
mode: flags.mode
|
|
3284
4139
|
};
|
|
3285
4140
|
if (!isInteractive()) return selection;
|
|
3286
4141
|
if (!selection.contentType) {
|
|
@@ -3292,6 +4147,9 @@ async function promptTranslateSelection(config, flags) {
|
|
|
3292
4147
|
if (!selection.strategy) {
|
|
3293
4148
|
selection.strategy = await promptStrategy();
|
|
3294
4149
|
}
|
|
4150
|
+
if (!selection.mode) {
|
|
4151
|
+
selection.mode = await promptMode();
|
|
4152
|
+
}
|
|
3295
4153
|
return selection;
|
|
3296
4154
|
}
|
|
3297
4155
|
|
|
@@ -3313,6 +4171,34 @@ function progressBar(ratio, width = 28) {
|
|
|
3313
4171
|
function labelForResult(result) {
|
|
3314
4172
|
return `${result.contentType}/${result.enSlug}@${result.locale}`;
|
|
3315
4173
|
}
|
|
4174
|
+
function shortJobId(name) {
|
|
4175
|
+
const segments = name.split("/");
|
|
4176
|
+
return segments[segments.length - 1] || name;
|
|
4177
|
+
}
|
|
4178
|
+
function formatElapsed(elapsedMs) {
|
|
4179
|
+
return `${Math.round(elapsedMs / 1e3)}s`;
|
|
4180
|
+
}
|
|
4181
|
+
function batchDoneParts(event) {
|
|
4182
|
+
if (event.alreadyIngested) {
|
|
4183
|
+
return [
|
|
4184
|
+
`job ${shortJobId(event.name)}`,
|
|
4185
|
+
event.model ?? "?",
|
|
4186
|
+
"already ingested by another scribe run (results saved there)",
|
|
4187
|
+
formatElapsed(event.elapsedMs)
|
|
4188
|
+
].join(" \xB7 ");
|
|
4189
|
+
}
|
|
4190
|
+
const parts = [
|
|
4191
|
+
`job ${shortJobId(event.name)}`,
|
|
4192
|
+
event.model ?? "?",
|
|
4193
|
+
`${event.count} request${event.count === 1 ? "" : "s"}`,
|
|
4194
|
+
`${event.translated} translated`
|
|
4195
|
+
];
|
|
4196
|
+
if (event.failed > 0) parts.push(`${event.failed} failed`);
|
|
4197
|
+
parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
|
|
4198
|
+
parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
|
|
4199
|
+
parts.push(formatElapsed(event.elapsedMs));
|
|
4200
|
+
return parts.join(" \xB7 ");
|
|
4201
|
+
}
|
|
3316
4202
|
function statusForResult(result, dryRun) {
|
|
3317
4203
|
if (result.failed) return red("failed");
|
|
3318
4204
|
if (result.skipped) return dim("skipped");
|
|
@@ -3358,6 +4244,25 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3358
4244
|
if (!enabled) {
|
|
3359
4245
|
return {
|
|
3360
4246
|
onEvent(event) {
|
|
4247
|
+
if (event.type === "batch-submitted") {
|
|
4248
|
+
const verb = event.resumed ? "Resuming" : "Submitted";
|
|
4249
|
+
console.log(
|
|
4250
|
+
`${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
|
|
4251
|
+
);
|
|
4252
|
+
return;
|
|
4253
|
+
}
|
|
4254
|
+
if (event.type === "batch-polling") {
|
|
4255
|
+
console.log(
|
|
4256
|
+
`batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
|
|
4257
|
+
);
|
|
4258
|
+
return;
|
|
4259
|
+
}
|
|
4260
|
+
if (event.type === "batch-done") {
|
|
4261
|
+
const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
|
|
4262
|
+
if (event.failed > 0 && event.translated === 0) console.error(line);
|
|
4263
|
+
else console.log(line);
|
|
4264
|
+
return;
|
|
4265
|
+
}
|
|
3361
4266
|
if (event.type === "item-done") {
|
|
3362
4267
|
const label = labelForResult(event.result);
|
|
3363
4268
|
const status = statusForResult(event.result, dryRun);
|
|
@@ -3388,6 +4293,9 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3388
4293
|
let outputTokens = 0;
|
|
3389
4294
|
let estimatedCostUsd = 0;
|
|
3390
4295
|
let active = [];
|
|
4296
|
+
let mode;
|
|
4297
|
+
const batchJobs = /* @__PURE__ */ new Map();
|
|
4298
|
+
let batchElapsedMs = 0;
|
|
3391
4299
|
const recent = [];
|
|
3392
4300
|
let renderedLines = 0;
|
|
3393
4301
|
let cursorHidden = false;
|
|
@@ -3403,6 +4311,15 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3403
4311
|
cursorHidden = false;
|
|
3404
4312
|
}
|
|
3405
4313
|
}
|
|
4314
|
+
function printPersistent(lines) {
|
|
4315
|
+
if (renderedLines > 0) {
|
|
4316
|
+
process.stdout.write(`\x1B[${renderedLines}A`);
|
|
4317
|
+
}
|
|
4318
|
+
for (const line of lines) {
|
|
4319
|
+
process.stdout.write("\x1B[2K" + line + "\n");
|
|
4320
|
+
}
|
|
4321
|
+
renderedLines = 0;
|
|
4322
|
+
}
|
|
3406
4323
|
function render() {
|
|
3407
4324
|
hideCursor();
|
|
3408
4325
|
if (renderedLines > 0) {
|
|
@@ -3410,12 +4327,34 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3410
4327
|
}
|
|
3411
4328
|
const lines = [];
|
|
3412
4329
|
const title = dryRun ? "Dry run" : "Translating";
|
|
3413
|
-
|
|
4330
|
+
const isBatch = mode === "batch" || batchJobs.size > 0;
|
|
4331
|
+
lines.push(
|
|
4332
|
+
cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
|
|
4333
|
+
);
|
|
3414
4334
|
lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
|
|
3415
4335
|
lines.push(
|
|
3416
4336
|
dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
|
|
3417
4337
|
);
|
|
3418
|
-
if (
|
|
4338
|
+
if (batchJobs.size > 0 && done < total) {
|
|
4339
|
+
const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
|
|
4340
|
+
let running = 0;
|
|
4341
|
+
let finished = 0;
|
|
4342
|
+
let requests = 0;
|
|
4343
|
+
for (const job of batchJobs.values()) {
|
|
4344
|
+
if (terminal.test(job.state)) finished += 1;
|
|
4345
|
+
else running += 1;
|
|
4346
|
+
requests += job.count;
|
|
4347
|
+
}
|
|
4348
|
+
lines.push(
|
|
4349
|
+
dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
|
|
4350
|
+
);
|
|
4351
|
+
for (const [name, job] of batchJobs) {
|
|
4352
|
+
if (terminal.test(job.state)) continue;
|
|
4353
|
+
lines.push(
|
|
4354
|
+
dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
|
|
4355
|
+
);
|
|
4356
|
+
}
|
|
4357
|
+
} else if (active.length > 0) {
|
|
3419
4358
|
lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
|
|
3420
4359
|
} else if (done < total) {
|
|
3421
4360
|
lines.push(dim("Active ") + "starting\u2026");
|
|
@@ -3443,12 +4382,58 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3443
4382
|
total = event.total;
|
|
3444
4383
|
concurrency = event.concurrency;
|
|
3445
4384
|
model = event.model;
|
|
4385
|
+
mode = event.mode;
|
|
3446
4386
|
render();
|
|
3447
4387
|
break;
|
|
3448
4388
|
case "item-start":
|
|
3449
4389
|
active = event.active;
|
|
3450
4390
|
render();
|
|
3451
4391
|
break;
|
|
4392
|
+
case "batch-submitted": {
|
|
4393
|
+
const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
|
|
4394
|
+
const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
|
|
4395
|
+
batchJobs.set(event.name, {
|
|
4396
|
+
state: event.resumed ? "RESUMED" : "SUBMITTED",
|
|
4397
|
+
count: event.count,
|
|
4398
|
+
model: event.model,
|
|
4399
|
+
createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
|
|
4400
|
+
elapsedMs
|
|
4401
|
+
});
|
|
4402
|
+
const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
|
|
4403
|
+
const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
|
|
4404
|
+
printPersistent([
|
|
4405
|
+
dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
|
|
4406
|
+
]);
|
|
4407
|
+
render();
|
|
4408
|
+
break;
|
|
4409
|
+
}
|
|
4410
|
+
case "batch-polling": {
|
|
4411
|
+
const job = batchJobs.get(event.name);
|
|
4412
|
+
batchJobs.set(event.name, {
|
|
4413
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4414
|
+
count: job?.count ?? 0,
|
|
4415
|
+
model: job?.model,
|
|
4416
|
+
createdAtMs: job?.createdAtMs,
|
|
4417
|
+
elapsedMs: event.elapsedMs
|
|
4418
|
+
});
|
|
4419
|
+
batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
|
|
4420
|
+
render();
|
|
4421
|
+
break;
|
|
4422
|
+
}
|
|
4423
|
+
case "batch-done": {
|
|
4424
|
+
const job = batchJobs.get(event.name);
|
|
4425
|
+
batchJobs.set(event.name, {
|
|
4426
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4427
|
+
count: job?.count ?? event.count,
|
|
4428
|
+
model: job?.model ?? event.model,
|
|
4429
|
+
createdAtMs: job?.createdAtMs,
|
|
4430
|
+
elapsedMs: event.elapsedMs
|
|
4431
|
+
});
|
|
4432
|
+
const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
|
|
4433
|
+
printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
|
|
4434
|
+
render();
|
|
4435
|
+
break;
|
|
4436
|
+
}
|
|
3452
4437
|
case "item-done": {
|
|
3453
4438
|
done += 1;
|
|
3454
4439
|
active = active.filter((label) => label !== labelForResult(event.result));
|
|
@@ -3571,6 +4556,21 @@ function parseArgs(argv) {
|
|
|
3571
4556
|
i++;
|
|
3572
4557
|
continue;
|
|
3573
4558
|
}
|
|
4559
|
+
if (arg === "--batch") {
|
|
4560
|
+
options.batch = true;
|
|
4561
|
+
i++;
|
|
4562
|
+
continue;
|
|
4563
|
+
}
|
|
4564
|
+
if (arg === "--direct") {
|
|
4565
|
+
options.direct = true;
|
|
4566
|
+
i++;
|
|
4567
|
+
continue;
|
|
4568
|
+
}
|
|
4569
|
+
if (arg === "--resume") {
|
|
4570
|
+
options.resume = true;
|
|
4571
|
+
i++;
|
|
4572
|
+
continue;
|
|
4573
|
+
}
|
|
3574
4574
|
if (arg === "--strategy") {
|
|
3575
4575
|
const value = argv[++i];
|
|
3576
4576
|
if (value !== "all" && value !== "missing-only") {
|
|
@@ -3639,11 +4639,19 @@ Translate flags:
|
|
|
3639
4639
|
--locale <code>... Target locale(s); overrides --preset
|
|
3640
4640
|
--slug <en-slug> Single English document
|
|
3641
4641
|
--model <id> Gemini model override
|
|
3642
|
-
--
|
|
4642
|
+
--batch Force Gemini Batch API mode (50% token cost; default)
|
|
4643
|
+
--direct Force direct interactive calls (immediate, full price)
|
|
4644
|
+
--resume Only poll/ingest pending batch jobs; submit nothing new
|
|
4645
|
+
--concurrency <n> Parallel translations in --direct mode (default: 3)
|
|
3643
4646
|
--dry-run List work without writing
|
|
3644
4647
|
--force Re-translate even when hashes match
|
|
3645
4648
|
--strategy <mode> all (default) or missing-only
|
|
3646
4649
|
--no-progress Plain line logging instead of live progress
|
|
4650
|
+
|
|
4651
|
+
Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
|
|
4652
|
+
during polling is safe: the next run (or --resume) picks the jobs back up and
|
|
4653
|
+
ingests their results. In a TTY without --batch/--direct, a prompt asks which
|
|
4654
|
+
mode to use.
|
|
3647
4655
|
`);
|
|
3648
4656
|
return;
|
|
3649
4657
|
}
|
|
@@ -3689,11 +4697,29 @@ Translate flags:
|
|
|
3689
4697
|
break;
|
|
3690
4698
|
}
|
|
3691
4699
|
case "translate": {
|
|
4700
|
+
if (options.batch && options.direct) {
|
|
4701
|
+
throw new Error("--batch and --direct are mutually exclusive");
|
|
4702
|
+
}
|
|
4703
|
+
if (options.resume) {
|
|
4704
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4705
|
+
enabled: !options.noProgress,
|
|
4706
|
+
dryRun: false
|
|
4707
|
+
});
|
|
4708
|
+
const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4709
|
+
reporter2.finish();
|
|
4710
|
+
if (results2 === null) {
|
|
4711
|
+
console.log("No pending translation batch jobs.");
|
|
4712
|
+
break;
|
|
4713
|
+
}
|
|
4714
|
+
if (results2.some((result) => result.failed)) process.exitCode = 1;
|
|
4715
|
+
break;
|
|
4716
|
+
}
|
|
3692
4717
|
const selection = await promptTranslateSelection(config, {
|
|
3693
4718
|
type: options.type,
|
|
3694
4719
|
preset: options.preset,
|
|
3695
4720
|
locale: options.locale,
|
|
3696
|
-
strategy: options.strategy
|
|
4721
|
+
strategy: options.strategy,
|
|
4722
|
+
mode: options.batch ? "batch" : options.direct ? "direct" : void 0
|
|
3697
4723
|
});
|
|
3698
4724
|
const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
|
|
3699
4725
|
const worklist = buildWorklist(config, {
|
|
@@ -3703,6 +4729,18 @@ Translate flags:
|
|
|
3703
4729
|
strategy: selection.strategy ?? "all"
|
|
3704
4730
|
});
|
|
3705
4731
|
if (worklist.length === 0) {
|
|
4732
|
+
if (!options.dryRun) {
|
|
4733
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4734
|
+
enabled: !options.noProgress,
|
|
4735
|
+
dryRun: false
|
|
4736
|
+
});
|
|
4737
|
+
const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4738
|
+
reporter2.finish();
|
|
4739
|
+
if (resumed !== null) {
|
|
4740
|
+
if (resumed.some((result) => result.failed)) process.exitCode = 1;
|
|
4741
|
+
break;
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
3706
4744
|
console.log("Nothing to translate.");
|
|
3707
4745
|
break;
|
|
3708
4746
|
}
|
|
@@ -3715,6 +4753,7 @@ Translate flags:
|
|
|
3715
4753
|
dryRun: options.dryRun,
|
|
3716
4754
|
force: options.force,
|
|
3717
4755
|
concurrency: options.concurrency,
|
|
4756
|
+
mode: selection.mode,
|
|
3718
4757
|
onProgress: reporter.onEvent
|
|
3719
4758
|
});
|
|
3720
4759
|
reporter.finish();
|