scribe-cms 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1220 -204
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1221 -205
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +982 -166
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +983 -168
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +149 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +149 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +18 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.cjs
CHANGED
|
@@ -76,6 +76,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
function readSchemaVersion(db) {
|
|
79
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
80
|
+
if (!metaTable) return 0;
|
|
79
81
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
80
82
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
81
83
|
}
|
|
@@ -100,7 +102,7 @@ var SCHEMA_VERSION, MIGRATIONS;
|
|
|
100
102
|
var init_sqlite = __esm({
|
|
101
103
|
"src/storage/sqlite.ts"() {
|
|
102
104
|
init_cjs_shims();
|
|
103
|
-
SCHEMA_VERSION =
|
|
105
|
+
SCHEMA_VERSION = 5;
|
|
104
106
|
MIGRATIONS = [
|
|
105
107
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
106
108
|
key TEXT PRIMARY KEY,
|
|
@@ -148,7 +150,30 @@ var init_sqlite = __esm({
|
|
|
148
150
|
UNIQUE (content_type, en_slug, en_hash)
|
|
149
151
|
)`,
|
|
150
152
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
151
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
153
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
154
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
155
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
156
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
157
|
+
model TEXT NOT NULL,
|
|
158
|
+
display_model TEXT NOT NULL,
|
|
159
|
+
created_at TEXT NOT NULL,
|
|
160
|
+
state TEXT NOT NULL,
|
|
161
|
+
completed_at TEXT
|
|
162
|
+
)`,
|
|
163
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
164
|
+
job_id INTEGER NOT NULL,
|
|
165
|
+
request_index INTEGER NOT NULL,
|
|
166
|
+
content_type TEXT NOT NULL,
|
|
167
|
+
en_slug TEXT NOT NULL,
|
|
168
|
+
locale TEXT NOT NULL,
|
|
169
|
+
en_hash TEXT NOT NULL,
|
|
170
|
+
snapshot_id INTEGER NOT NULL,
|
|
171
|
+
status TEXT NOT NULL,
|
|
172
|
+
error TEXT,
|
|
173
|
+
PRIMARY KEY (job_id, request_index)
|
|
174
|
+
)`,
|
|
175
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
176
|
+
ON translation_batch_items(status)`
|
|
152
177
|
];
|
|
153
178
|
}
|
|
154
179
|
});
|
|
@@ -190,26 +215,6 @@ function getRelationTarget(schema) {
|
|
|
190
215
|
}
|
|
191
216
|
return null;
|
|
192
217
|
}
|
|
193
|
-
function unwrapSchema(schema) {
|
|
194
|
-
let current = schema;
|
|
195
|
-
for (let i = 0; i < 8; i++) {
|
|
196
|
-
const anySchema = current;
|
|
197
|
-
if (typeof anySchema.unwrap === "function") {
|
|
198
|
-
current = anySchema.unwrap();
|
|
199
|
-
continue;
|
|
200
|
-
}
|
|
201
|
-
if (typeof anySchema.removeDefault === "function") {
|
|
202
|
-
current = anySchema.removeDefault();
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
if (anySchema._def?.innerType) {
|
|
206
|
-
current = anySchema._def.innerType;
|
|
207
|
-
continue;
|
|
208
|
-
}
|
|
209
|
-
break;
|
|
210
|
-
}
|
|
211
|
-
return current;
|
|
212
|
-
}
|
|
213
218
|
function peelOptionalWrappers(schema) {
|
|
214
219
|
let current = schema;
|
|
215
220
|
for (let i = 0; i < 8; i++) {
|
|
@@ -355,6 +360,39 @@ function resolveConfig(input, baseDir) {
|
|
|
355
360
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
356
361
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
357
362
|
}
|
|
363
|
+
const localeFallbacks = {};
|
|
364
|
+
for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
|
|
365
|
+
if (!raw.locales.includes(locale)) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
const seen = /* @__PURE__ */ new Set();
|
|
371
|
+
for (const fallback of chain) {
|
|
372
|
+
if (!raw.locales.includes(fallback)) {
|
|
373
|
+
throw new Error(
|
|
374
|
+
`scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
if (fallback === locale) {
|
|
378
|
+
throw new Error(
|
|
379
|
+
`scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
if (fallback === defaultLocale) {
|
|
383
|
+
throw new Error(
|
|
384
|
+
`scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
if (seen.has(fallback)) {
|
|
388
|
+
throw new Error(
|
|
389
|
+
`scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
seen.add(fallback);
|
|
393
|
+
}
|
|
394
|
+
localeFallbacks[locale] = [...chain];
|
|
395
|
+
}
|
|
358
396
|
const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
359
397
|
const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
|
|
360
398
|
const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -384,6 +422,7 @@ function resolveConfig(input, baseDir) {
|
|
|
384
422
|
defaultLocale,
|
|
385
423
|
localeRouting,
|
|
386
424
|
localePresets: raw.localePresets,
|
|
425
|
+
localeFallbacks,
|
|
387
426
|
translate: raw.translate,
|
|
388
427
|
types
|
|
389
428
|
};
|
|
@@ -399,6 +438,12 @@ init_cjs_shims();
|
|
|
399
438
|
|
|
400
439
|
// src/core/introspect-schema.ts
|
|
401
440
|
init_cjs_shims();
|
|
441
|
+
function getArrayElement(schema) {
|
|
442
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
443
|
+
return schema.element;
|
|
444
|
+
}
|
|
445
|
+
return null;
|
|
446
|
+
}
|
|
402
447
|
function introspectSchema(schema, prefix = []) {
|
|
403
448
|
const relation = getRelationTarget(schema);
|
|
404
449
|
if (relation && prefix.length > 0) {
|
|
@@ -412,7 +457,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
412
457
|
}
|
|
413
458
|
];
|
|
414
459
|
}
|
|
415
|
-
|
|
460
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
461
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
462
|
+
}
|
|
463
|
+
const base = peelOptionalWrappers(schema);
|
|
416
464
|
if (base instanceof Object && "shape" in base) {
|
|
417
465
|
const shape = base.shape;
|
|
418
466
|
const fields = [];
|
|
@@ -421,9 +469,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
421
469
|
}
|
|
422
470
|
return fields;
|
|
423
471
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
const elementBase =
|
|
472
|
+
const element = getArrayElement(base);
|
|
473
|
+
if (element) {
|
|
474
|
+
const elementBase = peelOptionalWrappers(element);
|
|
427
475
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
428
476
|
const shape = elementBase.shape;
|
|
429
477
|
const fields = [];
|
|
@@ -435,15 +483,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
435
483
|
}
|
|
436
484
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
437
485
|
}
|
|
438
|
-
function
|
|
439
|
-
const
|
|
486
|
+
function buildPathTrie(paths) {
|
|
487
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
440
488
|
for (const path15 of paths) {
|
|
441
|
-
|
|
489
|
+
let node = root;
|
|
490
|
+
for (const segment of path15) {
|
|
491
|
+
let child = node.children.get(segment);
|
|
492
|
+
if (!child) {
|
|
493
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
494
|
+
node.children.set(segment, child);
|
|
495
|
+
}
|
|
496
|
+
node = child;
|
|
497
|
+
}
|
|
498
|
+
node.leaf = true;
|
|
499
|
+
}
|
|
500
|
+
return root;
|
|
501
|
+
}
|
|
502
|
+
function extractNode(data, trie) {
|
|
503
|
+
if (trie.leaf) return data;
|
|
504
|
+
const star = trie.children.get("*");
|
|
505
|
+
if (star) {
|
|
506
|
+
if (!Array.isArray(data)) return void 0;
|
|
507
|
+
return data.map(
|
|
508
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
512
|
+
const record = data;
|
|
513
|
+
const out = {};
|
|
514
|
+
let any = false;
|
|
515
|
+
for (const [key, child] of trie.children) {
|
|
516
|
+
const value = extractNode(record[key], child);
|
|
442
517
|
if (value !== void 0) {
|
|
443
|
-
|
|
518
|
+
out[key] = value;
|
|
519
|
+
any = true;
|
|
444
520
|
}
|
|
445
521
|
}
|
|
446
|
-
return out;
|
|
522
|
+
return any ? out : void 0;
|
|
523
|
+
}
|
|
524
|
+
function extractByPaths(data, paths) {
|
|
525
|
+
if (paths.length === 0) return {};
|
|
526
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
527
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
447
528
|
}
|
|
448
529
|
function pickTranslatable(data, schema) {
|
|
449
530
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -458,32 +539,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
458
539
|
const translatable = pickTranslatable(localeData, schema);
|
|
459
540
|
return deepMerge(structural, translatable);
|
|
460
541
|
}
|
|
461
|
-
function getAtPath(obj, path15) {
|
|
462
|
-
let current = obj;
|
|
463
|
-
for (const segment of path15) {
|
|
464
|
-
if (segment === "*") {
|
|
465
|
-
if (!Array.isArray(current)) return void 0;
|
|
466
|
-
return current.map(
|
|
467
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path15.slice(path15.indexOf("*") + 1)) : void 0
|
|
468
|
-
);
|
|
469
|
-
}
|
|
470
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
471
|
-
current = current[segment];
|
|
472
|
-
}
|
|
473
|
-
return current;
|
|
474
|
-
}
|
|
475
|
-
function setAtPath(obj, path15, value) {
|
|
476
|
-
if (path15.length === 0) return;
|
|
477
|
-
if (path15.length === 1) {
|
|
478
|
-
obj[path15[0]] = value;
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
const [head, ...rest] = path15;
|
|
482
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
483
|
-
obj[head] = {};
|
|
484
|
-
}
|
|
485
|
-
setAtPath(obj[head], rest, value);
|
|
486
|
-
}
|
|
487
542
|
function deepMerge(base, overlay) {
|
|
488
543
|
const out = { ...base };
|
|
489
544
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -774,7 +829,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
774
829
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
775
830
|
const attrStart = i;
|
|
776
831
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
777
|
-
body.slice(attrStart, i);
|
|
832
|
+
const attrName = body.slice(attrStart, i);
|
|
833
|
+
if (!attrName) break;
|
|
778
834
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
779
835
|
if (body[i] !== "=") {
|
|
780
836
|
tagOut += body.slice(attrStart, i);
|
|
@@ -783,10 +839,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
783
839
|
tagOut += body.slice(attrStart, i);
|
|
784
840
|
tagOut += "=";
|
|
785
841
|
i += 1;
|
|
842
|
+
const wsStart = i;
|
|
786
843
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
844
|
+
tagOut += body.slice(wsStart, i);
|
|
787
845
|
const quote = body[i];
|
|
788
846
|
if (quote !== '"' && quote !== "'") {
|
|
789
|
-
|
|
847
|
+
break;
|
|
790
848
|
}
|
|
791
849
|
if (quote === "'") {
|
|
792
850
|
const valStart2 = i + 1;
|
|
@@ -1108,7 +1166,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1108
1166
|
|
|
1109
1167
|
// src/i18n/resolve-document.ts
|
|
1110
1168
|
init_cjs_shims();
|
|
1111
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1169
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1112
1170
|
const urlBuilder = createUrlBuilder({
|
|
1113
1171
|
locales: [defaultLocale, locale],
|
|
1114
1172
|
defaultLocale,
|
|
@@ -1122,15 +1180,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1122
1180
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1123
1181
|
const found = docIdx.bySlug.get(slug);
|
|
1124
1182
|
if (!found) continue;
|
|
1125
|
-
const
|
|
1126
|
-
|
|
1183
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1184
|
+
let target;
|
|
1185
|
+
let targetLocale = locale;
|
|
1186
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1187
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1188
|
+
if (cand) {
|
|
1189
|
+
target = cand;
|
|
1190
|
+
targetLocale = candidateLocale;
|
|
1191
|
+
break;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
if (target) {
|
|
1195
|
+
if (target.slug === slug) {
|
|
1196
|
+
return { document: target, actualLocale: targetLocale };
|
|
1197
|
+
}
|
|
1127
1198
|
if (!type.path) {
|
|
1128
1199
|
return { document: null, actualLocale: locale };
|
|
1129
1200
|
}
|
|
1130
1201
|
return {
|
|
1131
1202
|
document: null,
|
|
1132
1203
|
actualLocale: locale,
|
|
1133
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1204
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1134
1205
|
};
|
|
1135
1206
|
}
|
|
1136
1207
|
if (docLocale === defaultLocale) {
|
|
@@ -1161,13 +1232,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1161
1232
|
}
|
|
1162
1233
|
return { document: null, actualLocale: locale };
|
|
1163
1234
|
}
|
|
1164
|
-
function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1165
|
-
if (sourceLocale === targetLocale) return document2.slug;
|
|
1166
|
-
const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
|
|
1167
|
-
if (!englishSlug) return null;
|
|
1168
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1169
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1170
|
-
}
|
|
1171
1235
|
|
|
1172
1236
|
// src/create-project.ts
|
|
1173
1237
|
function comparatorFor(orderBy) {
|
|
@@ -1220,7 +1284,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1220
1284
|
config.defaultLocale,
|
|
1221
1285
|
load(),
|
|
1222
1286
|
type,
|
|
1223
|
-
config.localeRouting
|
|
1287
|
+
config.localeRouting,
|
|
1288
|
+
config.localeFallbacks[locale] ?? []
|
|
1224
1289
|
);
|
|
1225
1290
|
if (result.document && type.path) {
|
|
1226
1291
|
return {
|
|
@@ -1242,8 +1307,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1242
1307
|
const params = [];
|
|
1243
1308
|
for (const locale of options.locales ?? config.locales) {
|
|
1244
1309
|
const localeIdx = all.get(locale);
|
|
1310
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1245
1311
|
for (const doc of enIdx.bySlug.values()) {
|
|
1246
|
-
|
|
1312
|
+
let slug;
|
|
1313
|
+
if (locale === config.defaultLocale) {
|
|
1314
|
+
slug = doc.slug;
|
|
1315
|
+
} else {
|
|
1316
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1317
|
+
}
|
|
1247
1318
|
params.push({ locale, slug });
|
|
1248
1319
|
}
|
|
1249
1320
|
}
|
|
@@ -1705,7 +1776,7 @@ init_sqlite();
|
|
|
1705
1776
|
|
|
1706
1777
|
// src/validate/validate-relations.ts
|
|
1707
1778
|
init_cjs_shims();
|
|
1708
|
-
function
|
|
1779
|
+
function getAtPath(obj, fieldPath) {
|
|
1709
1780
|
let current = obj;
|
|
1710
1781
|
for (const segment of fieldPath) {
|
|
1711
1782
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -1750,7 +1821,7 @@ function validateRelations(project) {
|
|
|
1750
1821
|
});
|
|
1751
1822
|
continue;
|
|
1752
1823
|
}
|
|
1753
|
-
const value =
|
|
1824
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
1754
1825
|
if (value === void 0 || value === null || value === "") continue;
|
|
1755
1826
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
1756
1827
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2134,6 +2205,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2134
2205
|
// src/translate/page-translator.ts
|
|
2135
2206
|
init_cjs_shims();
|
|
2136
2207
|
|
|
2208
|
+
// src/storage/batch-jobs.ts
|
|
2209
|
+
init_cjs_shims();
|
|
2210
|
+
function insertBatchJob(db, input) {
|
|
2211
|
+
const info = db.prepare(
|
|
2212
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2213
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2214
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2215
|
+
return Number(info.lastInsertRowid);
|
|
2216
|
+
}
|
|
2217
|
+
function insertBatchItems(db, jobId, items) {
|
|
2218
|
+
const stmt = db.prepare(
|
|
2219
|
+
`INSERT INTO translation_batch_items (
|
|
2220
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2221
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2222
|
+
);
|
|
2223
|
+
for (const item of items) {
|
|
2224
|
+
stmt.run(
|
|
2225
|
+
jobId,
|
|
2226
|
+
item.requestIndex,
|
|
2227
|
+
item.contentType,
|
|
2228
|
+
item.enSlug,
|
|
2229
|
+
item.locale,
|
|
2230
|
+
item.enHash,
|
|
2231
|
+
item.snapshotId
|
|
2232
|
+
);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
function listPendingBatchJobs(db) {
|
|
2236
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2237
|
+
}
|
|
2238
|
+
function listBatchItems(db, jobId) {
|
|
2239
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2240
|
+
}
|
|
2241
|
+
function listPendingBatchItems(db) {
|
|
2242
|
+
return db.prepare(
|
|
2243
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2244
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2245
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2246
|
+
ORDER BY i.job_id, i.request_index`
|
|
2247
|
+
).all();
|
|
2248
|
+
}
|
|
2249
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2250
|
+
const info = db.prepare(
|
|
2251
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2252
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2253
|
+
).run(state, completedAt, jobId);
|
|
2254
|
+
return info.changes > 0;
|
|
2255
|
+
}
|
|
2256
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2257
|
+
db.prepare(
|
|
2258
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2259
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
// src/translate/page-translator.ts
|
|
2263
|
+
init_sqlite();
|
|
2264
|
+
|
|
2265
|
+
// src/translate/batch-worklist.ts
|
|
2266
|
+
init_cjs_shims();
|
|
2267
|
+
|
|
2137
2268
|
// src/history/record-snapshot.ts
|
|
2138
2269
|
init_cjs_shims();
|
|
2139
2270
|
init_sqlite();
|
|
@@ -2151,9 +2282,111 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2151
2282
|
return id;
|
|
2152
2283
|
}
|
|
2153
2284
|
|
|
2154
|
-
// src/translate/
|
|
2285
|
+
// src/translate/batch-worklist.ts
|
|
2155
2286
|
init_sqlite();
|
|
2156
2287
|
|
|
2288
|
+
// src/translate/gemini-batch.ts
|
|
2289
|
+
init_cjs_shims();
|
|
2290
|
+
|
|
2291
|
+
// src/translate/retry.ts
|
|
2292
|
+
init_cjs_shims();
|
|
2293
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2294
|
+
function statusFromError(error) {
|
|
2295
|
+
if (error instanceof genai.ApiError && typeof error.status === "number") {
|
|
2296
|
+
return error.status;
|
|
2297
|
+
}
|
|
2298
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2299
|
+
const status = error.status;
|
|
2300
|
+
if (typeof status === "number") return status;
|
|
2301
|
+
}
|
|
2302
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2303
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2304
|
+
if (match) return Number(match[1]);
|
|
2305
|
+
return void 0;
|
|
2306
|
+
}
|
|
2307
|
+
function isNetworkError(error) {
|
|
2308
|
+
const err = error;
|
|
2309
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2310
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2311
|
+
return true;
|
|
2312
|
+
}
|
|
2313
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2314
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2315
|
+
}
|
|
2316
|
+
function isTransientError(error) {
|
|
2317
|
+
const status = statusFromError(error);
|
|
2318
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2319
|
+
return isNetworkError(error);
|
|
2320
|
+
}
|
|
2321
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2322
|
+
async function withRetry(fn, options = {}) {
|
|
2323
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2324
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2325
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2326
|
+
const random = options.random ?? Math.random;
|
|
2327
|
+
let lastError;
|
|
2328
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2329
|
+
try {
|
|
2330
|
+
return await fn();
|
|
2331
|
+
} catch (error) {
|
|
2332
|
+
lastError = error;
|
|
2333
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2334
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2335
|
+
const delay = backoff + random() * backoff;
|
|
2336
|
+
await sleep2(delay);
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
throw lastError;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
// src/translate/gemini-batch.ts
|
|
2343
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2344
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2345
|
+
"SUCCEEDED",
|
|
2346
|
+
"FAILED",
|
|
2347
|
+
"CANCELLED",
|
|
2348
|
+
"EXPIRED",
|
|
2349
|
+
"PARTIALLY_SUCCEEDED"
|
|
2350
|
+
]);
|
|
2351
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2352
|
+
function normalizeBatchState(state) {
|
|
2353
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2354
|
+
}
|
|
2355
|
+
function isTerminalBatchState(state) {
|
|
2356
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2357
|
+
}
|
|
2358
|
+
function isSuccessfulBatchState(state) {
|
|
2359
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2360
|
+
}
|
|
2361
|
+
function makeClient(apiKey) {
|
|
2362
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2363
|
+
if (!key) {
|
|
2364
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2365
|
+
}
|
|
2366
|
+
return new genai.GoogleGenAI({ apiKey: key });
|
|
2367
|
+
}
|
|
2368
|
+
function textFromBatchResponse(response) {
|
|
2369
|
+
if (typeof response.text === "string") return response.text;
|
|
2370
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2371
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2372
|
+
}
|
|
2373
|
+
async function createGeminiBatchJob(input) {
|
|
2374
|
+
const ai = makeClient(input.apiKey);
|
|
2375
|
+
const job = await withRetry(
|
|
2376
|
+
() => ai.batches.create({
|
|
2377
|
+
model: input.model,
|
|
2378
|
+
src: input.requests,
|
|
2379
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2380
|
+
})
|
|
2381
|
+
);
|
|
2382
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2383
|
+
return job;
|
|
2384
|
+
}
|
|
2385
|
+
async function getGeminiBatchJob(input) {
|
|
2386
|
+
const ai = makeClient(input.apiKey);
|
|
2387
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2157
2390
|
// src/translate/gemini-client.ts
|
|
2158
2391
|
init_cjs_shims();
|
|
2159
2392
|
|
|
@@ -2178,6 +2411,12 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2178
2411
|
if (alias) return alias[0];
|
|
2179
2412
|
return name;
|
|
2180
2413
|
}
|
|
2414
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2415
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2416
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
|
|
2417
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2418
|
+
return void 0;
|
|
2419
|
+
}
|
|
2181
2420
|
|
|
2182
2421
|
// src/translate/gemini-client.ts
|
|
2183
2422
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
@@ -2191,11 +2430,36 @@ function extractJson(text) {
|
|
|
2191
2430
|
return trimmed;
|
|
2192
2431
|
}
|
|
2193
2432
|
function parseGeminiResponse(text) {
|
|
2433
|
+
let parsed;
|
|
2194
2434
|
try {
|
|
2195
|
-
|
|
2435
|
+
parsed = JSON.parse(text.trim());
|
|
2196
2436
|
} catch {
|
|
2197
|
-
|
|
2437
|
+
parsed = JSON.parse(extractJson(text));
|
|
2198
2438
|
}
|
|
2439
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2440
|
+
parsed.frontmatter = {};
|
|
2441
|
+
}
|
|
2442
|
+
return parsed;
|
|
2443
|
+
}
|
|
2444
|
+
function buildGeminiRequestConfig(input) {
|
|
2445
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2446
|
+
return {
|
|
2447
|
+
responseMimeType: "application/json",
|
|
2448
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2449
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
function usageFromResponse(response) {
|
|
2453
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2454
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2455
|
+
return {
|
|
2456
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2457
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2458
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2459
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2460
|
+
thoughtsTokens,
|
|
2461
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2462
|
+
};
|
|
2199
2463
|
}
|
|
2200
2464
|
async function translatePageWithGemini(input) {
|
|
2201
2465
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2207,28 +2471,28 @@ async function translatePageWithGemini(input) {
|
|
|
2207
2471
|
);
|
|
2208
2472
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2209
2473
|
const ai = new genai.GoogleGenAI({ apiKey });
|
|
2210
|
-
const response = await
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2474
|
+
const response = await withRetry(
|
|
2475
|
+
() => ai.models.generateContent({
|
|
2476
|
+
model: apiModel,
|
|
2477
|
+
contents: input.prompt,
|
|
2478
|
+
config: buildGeminiRequestConfig({
|
|
2479
|
+
apiModelId: apiModel,
|
|
2480
|
+
responseSchema: input.responseSchema
|
|
2481
|
+
})
|
|
2482
|
+
})
|
|
2483
|
+
);
|
|
2218
2484
|
const raw = response.text ?? "";
|
|
2219
2485
|
const parsed = parseGeminiResponse(raw);
|
|
2220
2486
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2221
2487
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2222
2488
|
}
|
|
2223
|
-
|
|
2224
|
-
const usage = {
|
|
2225
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2226
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2227
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2228
|
-
};
|
|
2229
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2489
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2230
2490
|
}
|
|
2231
2491
|
|
|
2492
|
+
// src/translate/translate-core.ts
|
|
2493
|
+
init_cjs_shims();
|
|
2494
|
+
init_sqlite();
|
|
2495
|
+
|
|
2232
2496
|
// src/translate/gemini-pricing.ts
|
|
2233
2497
|
init_cjs_shims();
|
|
2234
2498
|
var CONTEXT_TIER_TOKENS = 2e5;
|
|
@@ -2247,12 +2511,14 @@ function resolveModelPricing(model) {
|
|
|
2247
2511
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2248
2512
|
return match?.[1];
|
|
2249
2513
|
}
|
|
2250
|
-
|
|
2514
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2515
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2251
2516
|
const pricing = resolveModelPricing(model);
|
|
2252
2517
|
if (!pricing) return void 0;
|
|
2253
2518
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2254
2519
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2255
|
-
|
|
2520
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2521
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2256
2522
|
}
|
|
2257
2523
|
function formatTokenCount(tokens) {
|
|
2258
2524
|
if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
|
|
@@ -2280,6 +2546,7 @@ var LOCALE_NAMES = {
|
|
|
2280
2546
|
"zh-CN": "Simplified Chinese",
|
|
2281
2547
|
"zh-TW": "Traditional Chinese",
|
|
2282
2548
|
pt: "Portuguese",
|
|
2549
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2283
2550
|
ja: "Japanese",
|
|
2284
2551
|
ru: "Russian",
|
|
2285
2552
|
it: "Italian",
|
|
@@ -2287,29 +2554,29 @@ var LOCALE_NAMES = {
|
|
|
2287
2554
|
};
|
|
2288
2555
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2289
2556
|
return [
|
|
2290
|
-
`Localize the content
|
|
2557
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2291
2558
|
"Do not translate word-for-word.",
|
|
2292
2559
|
"Preserve the source tone and brand voice.",
|
|
2293
2560
|
"Write as if a native speaker authored it for the target market."
|
|
2294
2561
|
].join(" ");
|
|
2295
2562
|
}
|
|
2563
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2564
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2565
|
+
if (!hasFrontmatter) {
|
|
2566
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2567
|
+
}
|
|
2568
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2569
|
+
}
|
|
2296
2570
|
function buildPageTranslationPrompt(input) {
|
|
2297
2571
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2298
|
-
const
|
|
2299
|
-
const
|
|
2300
|
-
|
|
2572
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2573
|
+
const prefix = [
|
|
2574
|
+
TASK_FRAMING,
|
|
2301
2575
|
"",
|
|
2302
2576
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2303
2577
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2304
2578
|
"## Rules",
|
|
2305
2579
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2306
|
-
...input.slugStrategy === "localized" ? [
|
|
2307
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2308
|
-
] : [],
|
|
2309
|
-
"",
|
|
2310
|
-
"## Output format",
|
|
2311
|
-
"Return ONLY valid JSON with keys:",
|
|
2312
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2313
2580
|
"",
|
|
2314
2581
|
"## EN translatable frontmatter (JSON)",
|
|
2315
2582
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2317,7 +2584,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2317
2584
|
"## EN body (MDX)",
|
|
2318
2585
|
input.enBody
|
|
2319
2586
|
];
|
|
2320
|
-
|
|
2587
|
+
const suffix = [
|
|
2588
|
+
"",
|
|
2589
|
+
"## Target language",
|
|
2590
|
+
localizationPrompt,
|
|
2591
|
+
...input.slugStrategy === "localized" ? [
|
|
2592
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2593
|
+
] : [],
|
|
2594
|
+
"",
|
|
2595
|
+
"## Output format",
|
|
2596
|
+
"Return ONLY valid JSON with keys:",
|
|
2597
|
+
buildOutputFormatLine(
|
|
2598
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2599
|
+
input.slugStrategy
|
|
2600
|
+
)
|
|
2601
|
+
];
|
|
2602
|
+
return [...prefix, ...suffix].join("\n");
|
|
2321
2603
|
}
|
|
2322
2604
|
|
|
2323
2605
|
// src/translate/response-schema.ts
|
|
@@ -2368,9 +2650,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2368
2650
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2369
2651
|
try {
|
|
2370
2652
|
const translatable = buildTranslatableSubschema(schema);
|
|
2371
|
-
if (!translatable) return null;
|
|
2372
2653
|
const responseShape = {
|
|
2373
|
-
frontmatter: translatable,
|
|
2654
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2374
2655
|
body: zod.z.string()
|
|
2375
2656
|
};
|
|
2376
2657
|
if (slugStrategy === "localized") {
|
|
@@ -2451,7 +2732,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2451
2732
|
return { ok: true, frontmatter };
|
|
2452
2733
|
}
|
|
2453
2734
|
|
|
2454
|
-
// src/translate/
|
|
2735
|
+
// src/translate/translate-core.ts
|
|
2736
|
+
function translationItemKey(item) {
|
|
2737
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2738
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2739
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2740
|
+
}
|
|
2455
2741
|
function summarizeResults(results, durationMs) {
|
|
2456
2742
|
return results.reduce(
|
|
2457
2743
|
(totals, result) => {
|
|
@@ -2487,17 +2773,6 @@ function formatTranslateError(error) {
|
|
|
2487
2773
|
}
|
|
2488
2774
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2489
2775
|
}
|
|
2490
|
-
async function runPool(items, concurrency, worker) {
|
|
2491
|
-
if (items.length === 0) return;
|
|
2492
|
-
let nextIndex = 0;
|
|
2493
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2494
|
-
while (nextIndex < items.length) {
|
|
2495
|
-
const index = nextIndex++;
|
|
2496
|
-
await worker(items[index], index);
|
|
2497
|
-
}
|
|
2498
|
-
});
|
|
2499
|
-
await Promise.all(runners);
|
|
2500
|
-
}
|
|
2501
2776
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2502
2777
|
const fm = enDoc.frontmatter;
|
|
2503
2778
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2506,73 +2781,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2506
2781
|
}
|
|
2507
2782
|
return enSlug;
|
|
2508
2783
|
}
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
const base = {
|
|
2784
|
+
function baseForItem(item) {
|
|
2785
|
+
return {
|
|
2512
2786
|
contentType: item.contentType,
|
|
2513
2787
|
enSlug: item.enSlug,
|
|
2514
2788
|
locale: item.locale
|
|
2515
2789
|
};
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2790
|
+
}
|
|
2791
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
2792
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
2793
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
2794
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
2795
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
2796
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
2797
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
2798
|
+
const db = openStore(config, "readonly");
|
|
2799
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
2800
|
+
db.close();
|
|
2801
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
2802
|
+
return {
|
|
2803
|
+
status: "done",
|
|
2804
|
+
result: {
|
|
2805
|
+
...baseForItem(item),
|
|
2529
2806
|
skipped: true,
|
|
2530
2807
|
reason: "fresh",
|
|
2531
2808
|
durationMs: Date.now() - startedAt
|
|
2532
|
-
}
|
|
2533
|
-
}
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2809
|
+
}
|
|
2810
|
+
};
|
|
2811
|
+
}
|
|
2812
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
2813
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
2814
|
+
const prompt = buildPageTranslationPrompt({
|
|
2815
|
+
resolved: resolvedTranslate,
|
|
2816
|
+
targetLocale: item.locale,
|
|
2817
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
2818
|
+
translatableFrontmatter: payload.frontmatter,
|
|
2819
|
+
enBody: payload.body,
|
|
2820
|
+
slugStrategy: type.slugStrategy
|
|
2821
|
+
});
|
|
2822
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
2823
|
+
return {
|
|
2824
|
+
status: "ready",
|
|
2825
|
+
prepared: {
|
|
2826
|
+
item,
|
|
2827
|
+
type,
|
|
2828
|
+
enDoc,
|
|
2829
|
+
payload,
|
|
2830
|
+
currentEnHash,
|
|
2831
|
+
existingSlug: existing?.slug,
|
|
2555
2832
|
model,
|
|
2833
|
+
prompt,
|
|
2556
2834
|
responseSchema: responseSchema ?? void 0
|
|
2557
|
-
}
|
|
2558
|
-
|
|
2835
|
+
}
|
|
2836
|
+
};
|
|
2837
|
+
}
|
|
2838
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
2839
|
+
const { item, type, enDoc, payload } = prepared;
|
|
2840
|
+
const base = baseForItem(item);
|
|
2841
|
+
try {
|
|
2842
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2559
2843
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2560
2844
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2561
2845
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2562
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
2846
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2563
2847
|
if (!validated.ok) {
|
|
2564
2848
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2565
2849
|
}
|
|
2566
2850
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2567
|
-
|
|
2851
|
+
output.parsed.body
|
|
2568
2852
|
);
|
|
2569
2853
|
const writeDb = openStore(config, "readwrite");
|
|
2570
|
-
const snapshotId = recordEnSnapshot(
|
|
2854
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2571
2855
|
config,
|
|
2572
2856
|
{
|
|
2573
2857
|
contentType: type.id,
|
|
2574
2858
|
enSlug: item.enSlug,
|
|
2575
|
-
enHash: currentEnHash,
|
|
2859
|
+
enHash: prepared.currentEnHash,
|
|
2576
2860
|
frontmatter: payload.frontmatter,
|
|
2577
2861
|
body: payload.body
|
|
2578
2862
|
},
|
|
@@ -2585,24 +2869,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2585
2869
|
slug,
|
|
2586
2870
|
frontmatter: validated.frontmatter,
|
|
2587
2871
|
body: translatedBody,
|
|
2588
|
-
enHash: currentEnHash,
|
|
2872
|
+
enHash: prepared.currentEnHash,
|
|
2589
2873
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2590
|
-
model:
|
|
2874
|
+
model: output.model,
|
|
2591
2875
|
snapshotId
|
|
2592
2876
|
});
|
|
2593
2877
|
writeDb.close();
|
|
2594
2878
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2595
|
-
normalizeGeminiDisplayName(
|
|
2596
|
-
|
|
2597
|
-
|
|
2879
|
+
normalizeGeminiDisplayName(output.model),
|
|
2880
|
+
output.usage.inputTokens,
|
|
2881
|
+
output.usage.outputTokens,
|
|
2882
|
+
options.costMode
|
|
2598
2883
|
);
|
|
2599
2884
|
return {
|
|
2600
2885
|
...base,
|
|
2601
2886
|
skipped: false,
|
|
2602
|
-
model:
|
|
2603
|
-
usage:
|
|
2887
|
+
model: output.model,
|
|
2888
|
+
usage: output.usage,
|
|
2604
2889
|
estimatedCostUsd,
|
|
2605
|
-
durationMs: Date.now() - startedAt,
|
|
2890
|
+
durationMs: Date.now() - options.startedAt,
|
|
2606
2891
|
slugAdjusted,
|
|
2607
2892
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2608
2893
|
};
|
|
@@ -2612,33 +2897,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2612
2897
|
skipped: false,
|
|
2613
2898
|
failed: true,
|
|
2614
2899
|
error: formatTranslateError(error),
|
|
2900
|
+
durationMs: Date.now() - options.startedAt
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
}
|
|
2904
|
+
function displayModelFor(prepared) {
|
|
2905
|
+
return normalizeGeminiDisplayName(
|
|
2906
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
2907
|
+
);
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
// src/translate/batch-worklist.ts
|
|
2911
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
2912
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
2913
|
+
var INITIAL_POLL_MS = 5e3;
|
|
2914
|
+
var MAX_POLL_MS = 3e4;
|
|
2915
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
2916
|
+
function sleep(ms) {
|
|
2917
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2918
|
+
}
|
|
2919
|
+
function planBatchJobs(entries) {
|
|
2920
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
2921
|
+
for (const entry of entries) {
|
|
2922
|
+
const group = byModel.get(entry.apiModel);
|
|
2923
|
+
if (group) group.push(entry);
|
|
2924
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
2925
|
+
}
|
|
2926
|
+
const plans = [];
|
|
2927
|
+
for (const [apiModel, group] of byModel) {
|
|
2928
|
+
let current = [];
|
|
2929
|
+
let currentBytes = 0;
|
|
2930
|
+
for (const entry of group) {
|
|
2931
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
2932
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
2933
|
+
plans.push({ apiModel, entries: current });
|
|
2934
|
+
current = [];
|
|
2935
|
+
currentBytes = 0;
|
|
2936
|
+
}
|
|
2937
|
+
current.push(entry);
|
|
2938
|
+
currentBytes += size;
|
|
2939
|
+
}
|
|
2940
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
2941
|
+
}
|
|
2942
|
+
return plans;
|
|
2943
|
+
}
|
|
2944
|
+
function readPendingBatchWork(config) {
|
|
2945
|
+
const db = openStore(config, "readwrite");
|
|
2946
|
+
const jobs = listPendingBatchJobs(db);
|
|
2947
|
+
const pendingItems = listPendingBatchItems(db);
|
|
2948
|
+
db.close();
|
|
2949
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
2950
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
2951
|
+
}
|
|
2952
|
+
async function submitBatchJobPlan(config, input) {
|
|
2953
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
2954
|
+
const job = await createGeminiBatchJob({
|
|
2955
|
+
model: plan.apiModel,
|
|
2956
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
2957
|
+
contents: prepared.prompt,
|
|
2958
|
+
config: buildGeminiRequestConfig({
|
|
2959
|
+
apiModelId: plan.apiModel,
|
|
2960
|
+
responseSchema: prepared.responseSchema
|
|
2961
|
+
})
|
|
2962
|
+
})),
|
|
2963
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
2964
|
+
});
|
|
2965
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2966
|
+
const db = openStore(config, "readwrite");
|
|
2967
|
+
const jobId = insertBatchJob(db, {
|
|
2968
|
+
jobName: job.name,
|
|
2969
|
+
model: plan.apiModel,
|
|
2970
|
+
displayModel,
|
|
2971
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2972
|
+
createdAt
|
|
2973
|
+
});
|
|
2974
|
+
insertBatchItems(
|
|
2975
|
+
db,
|
|
2976
|
+
jobId,
|
|
2977
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
2978
|
+
requestIndex,
|
|
2979
|
+
contentType: prepared.item.contentType,
|
|
2980
|
+
enSlug: prepared.item.enSlug,
|
|
2981
|
+
locale: prepared.item.locale,
|
|
2982
|
+
enHash: prepared.currentEnHash,
|
|
2983
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
2984
|
+
// on the EN files still matching (or existing) on disk.
|
|
2985
|
+
snapshotId: recordEnSnapshot(
|
|
2986
|
+
config,
|
|
2987
|
+
{
|
|
2988
|
+
contentType: prepared.item.contentType,
|
|
2989
|
+
enSlug: prepared.item.enSlug,
|
|
2990
|
+
enHash: prepared.currentEnHash,
|
|
2991
|
+
frontmatter: prepared.payload.frontmatter,
|
|
2992
|
+
body: prepared.payload.body
|
|
2993
|
+
},
|
|
2994
|
+
db
|
|
2995
|
+
)
|
|
2996
|
+
}))
|
|
2997
|
+
);
|
|
2998
|
+
const row = {
|
|
2999
|
+
id: jobId,
|
|
3000
|
+
job_name: job.name,
|
|
3001
|
+
model: plan.apiModel,
|
|
3002
|
+
display_model: displayModel,
|
|
3003
|
+
created_at: createdAt,
|
|
3004
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3005
|
+
completed_at: null
|
|
3006
|
+
};
|
|
3007
|
+
db.close();
|
|
3008
|
+
input.onProgress?.({
|
|
3009
|
+
type: "batch-submitted",
|
|
3010
|
+
name: job.name,
|
|
3011
|
+
count: plan.entries.length,
|
|
3012
|
+
model: displayModel,
|
|
3013
|
+
jobIndex,
|
|
3014
|
+
jobCount,
|
|
3015
|
+
createdAt
|
|
3016
|
+
});
|
|
3017
|
+
return row;
|
|
3018
|
+
}
|
|
3019
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3020
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3021
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3022
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3023
|
+
if (!enDoc) {
|
|
3024
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3025
|
+
if (!snapshot) {
|
|
3026
|
+
throw new Error(
|
|
3027
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3028
|
+
);
|
|
3029
|
+
}
|
|
3030
|
+
enDoc = {
|
|
3031
|
+
slug: itemRow.en_slug,
|
|
3032
|
+
enSlug: itemRow.en_slug,
|
|
3033
|
+
locale: config.defaultLocale,
|
|
3034
|
+
noindex: false,
|
|
3035
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3036
|
+
content: snapshot.body
|
|
3037
|
+
};
|
|
3038
|
+
}
|
|
3039
|
+
const item = {
|
|
3040
|
+
contentType: itemRow.content_type,
|
|
3041
|
+
enSlug: itemRow.en_slug,
|
|
3042
|
+
locale: itemRow.locale,
|
|
3043
|
+
reason: "missing",
|
|
3044
|
+
currentEnHash: itemRow.en_hash
|
|
3045
|
+
};
|
|
3046
|
+
return {
|
|
3047
|
+
item,
|
|
3048
|
+
type,
|
|
3049
|
+
enDoc,
|
|
3050
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3051
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3052
|
+
currentEnHash: itemRow.en_hash,
|
|
3053
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3054
|
+
model: jobRow.display_model,
|
|
3055
|
+
prompt: "",
|
|
3056
|
+
responseSchema: void 0
|
|
3057
|
+
};
|
|
3058
|
+
}
|
|
3059
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3060
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3061
|
+
const db = openStore(config, "readwrite");
|
|
3062
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3063
|
+
db.close();
|
|
3064
|
+
return null;
|
|
3065
|
+
}
|
|
3066
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3067
|
+
const results = [];
|
|
3068
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3069
|
+
const result = {
|
|
3070
|
+
contentType: itemRow.content_type,
|
|
3071
|
+
enSlug: itemRow.en_slug,
|
|
3072
|
+
locale: itemRow.locale,
|
|
3073
|
+
skipped: false,
|
|
3074
|
+
failed: true,
|
|
3075
|
+
error: formatTranslateError(new Error(message)),
|
|
2615
3076
|
durationMs: Date.now() - startedAt
|
|
2616
3077
|
};
|
|
3078
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3079
|
+
results.push(result);
|
|
3080
|
+
onResult?.(result);
|
|
3081
|
+
};
|
|
3082
|
+
if (isSuccessfulBatchState(state)) {
|
|
3083
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3084
|
+
for (const itemRow of pendingItems) {
|
|
3085
|
+
const startedAt = Date.now();
|
|
3086
|
+
const inlined = responses[itemRow.request_index];
|
|
3087
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3088
|
+
failItem(
|
|
3089
|
+
itemRow,
|
|
3090
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3091
|
+
startedAt
|
|
3092
|
+
);
|
|
3093
|
+
continue;
|
|
3094
|
+
}
|
|
3095
|
+
let result;
|
|
3096
|
+
try {
|
|
3097
|
+
const response = inlined.response;
|
|
3098
|
+
const raw = textFromBatchResponse(response);
|
|
3099
|
+
const parsed = parseGeminiResponse(raw);
|
|
3100
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3101
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3102
|
+
}
|
|
3103
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3104
|
+
result = finalizeTranslation(
|
|
3105
|
+
config,
|
|
3106
|
+
prepared,
|
|
3107
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3108
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3109
|
+
);
|
|
3110
|
+
} catch (error) {
|
|
3111
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3112
|
+
continue;
|
|
3113
|
+
}
|
|
3114
|
+
updateBatchItemStatus(
|
|
3115
|
+
db,
|
|
3116
|
+
jobRow.id,
|
|
3117
|
+
itemRow.request_index,
|
|
3118
|
+
result.failed ? "failed" : "done",
|
|
3119
|
+
result.error
|
|
3120
|
+
);
|
|
3121
|
+
results.push(result);
|
|
3122
|
+
onResult?.(result);
|
|
3123
|
+
}
|
|
3124
|
+
} else {
|
|
3125
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3126
|
+
const startedAt = Date.now();
|
|
3127
|
+
for (const itemRow of pendingItems) {
|
|
3128
|
+
failItem(itemRow, message, startedAt);
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
db.close();
|
|
3132
|
+
return results;
|
|
3133
|
+
}
|
|
3134
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3135
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3136
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3137
|
+
const loopStartedAt = Date.now();
|
|
3138
|
+
let delay = INITIAL_POLL_MS;
|
|
3139
|
+
let firstRound = true;
|
|
3140
|
+
const elapsedFor = (row) => {
|
|
3141
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3142
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3143
|
+
};
|
|
3144
|
+
while (active.length > 0) {
|
|
3145
|
+
if (!firstRound) {
|
|
3146
|
+
await sleep(delay);
|
|
3147
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3148
|
+
}
|
|
3149
|
+
firstRound = false;
|
|
3150
|
+
for (const tracked of [...active]) {
|
|
3151
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3152
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3153
|
+
options.onProgress?.({
|
|
3154
|
+
type: "batch-polling",
|
|
3155
|
+
name: tracked.row.job_name,
|
|
3156
|
+
state,
|
|
3157
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3158
|
+
jobIndex: tracked.jobIndex,
|
|
3159
|
+
jobCount
|
|
3160
|
+
});
|
|
3161
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3162
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3163
|
+
const results = ingestBatchJob(
|
|
3164
|
+
config,
|
|
3165
|
+
tracked.row,
|
|
3166
|
+
batchJob,
|
|
3167
|
+
options.onResult
|
|
3168
|
+
);
|
|
3169
|
+
if (results === null) {
|
|
3170
|
+
options.onProgress?.({
|
|
3171
|
+
type: "batch-done",
|
|
3172
|
+
name: tracked.row.job_name,
|
|
3173
|
+
state,
|
|
3174
|
+
model: tracked.row.display_model,
|
|
3175
|
+
count: 0,
|
|
3176
|
+
jobIndex: tracked.jobIndex,
|
|
3177
|
+
jobCount,
|
|
3178
|
+
translated: 0,
|
|
3179
|
+
failed: 0,
|
|
3180
|
+
inputTokens: 0,
|
|
3181
|
+
outputTokens: 0,
|
|
3182
|
+
estimatedCostUsd: 0,
|
|
3183
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3184
|
+
alreadyIngested: true
|
|
3185
|
+
});
|
|
3186
|
+
continue;
|
|
3187
|
+
}
|
|
3188
|
+
options.onProgress?.({
|
|
3189
|
+
type: "batch-done",
|
|
3190
|
+
name: tracked.row.job_name,
|
|
3191
|
+
state,
|
|
3192
|
+
model: tracked.row.display_model,
|
|
3193
|
+
count: results.length,
|
|
3194
|
+
jobIndex: tracked.jobIndex,
|
|
3195
|
+
jobCount,
|
|
3196
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3197
|
+
failed: results.filter((r) => r.failed).length,
|
|
3198
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3199
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3200
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3201
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3202
|
+
});
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
2617
3205
|
}
|
|
2618
3206
|
}
|
|
3207
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3208
|
+
return {
|
|
3209
|
+
...baseForItem(item),
|
|
3210
|
+
skipped: false,
|
|
3211
|
+
failed: true,
|
|
3212
|
+
error: formatTranslateError(error),
|
|
3213
|
+
durationMs: Date.now() - startedAt
|
|
3214
|
+
};
|
|
3215
|
+
}
|
|
3216
|
+
|
|
3217
|
+
// src/translate/page-translator.ts
|
|
3218
|
+
async function runPool(items, concurrency, worker) {
|
|
3219
|
+
if (items.length === 0) return;
|
|
3220
|
+
let nextIndex = 0;
|
|
3221
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3222
|
+
while (nextIndex < items.length) {
|
|
3223
|
+
const index = nextIndex++;
|
|
3224
|
+
await worker(items[index], index);
|
|
3225
|
+
}
|
|
3226
|
+
});
|
|
3227
|
+
await Promise.all(runners);
|
|
3228
|
+
}
|
|
2619
3229
|
function labelForItem(item) {
|
|
2620
3230
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2621
3231
|
}
|
|
3232
|
+
async function translatePage(config, item, options = {}) {
|
|
3233
|
+
const startedAt = Date.now();
|
|
3234
|
+
let outcome;
|
|
3235
|
+
try {
|
|
3236
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3237
|
+
} catch (error) {
|
|
3238
|
+
return failedResultForItem(item, error, startedAt);
|
|
3239
|
+
}
|
|
3240
|
+
if (outcome.status === "done") return outcome.result;
|
|
3241
|
+
const prepared = outcome.prepared;
|
|
3242
|
+
if (options.dryRun) {
|
|
3243
|
+
return {
|
|
3244
|
+
...baseForItem(item),
|
|
3245
|
+
skipped: false,
|
|
3246
|
+
model: prepared.model,
|
|
3247
|
+
durationMs: Date.now() - startedAt
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
try {
|
|
3251
|
+
const result = await translatePageWithGemini({
|
|
3252
|
+
prompt: prepared.prompt,
|
|
3253
|
+
model: prepared.model,
|
|
3254
|
+
responseSchema: prepared.responseSchema
|
|
3255
|
+
});
|
|
3256
|
+
return finalizeTranslation(
|
|
3257
|
+
config,
|
|
3258
|
+
prepared,
|
|
3259
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3260
|
+
{ costMode: "interactive", startedAt }
|
|
3261
|
+
);
|
|
3262
|
+
} catch (error) {
|
|
3263
|
+
return failedResultForItem(item, error, startedAt);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
function createResultCollector(items, onProgress) {
|
|
3267
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3268
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3269
|
+
const slots = new Array(items.length);
|
|
3270
|
+
const extras = [];
|
|
3271
|
+
return {
|
|
3272
|
+
emit(result) {
|
|
3273
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3274
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3275
|
+
else extras.push(result);
|
|
3276
|
+
onProgress?.({ type: "item-done", result });
|
|
3277
|
+
},
|
|
3278
|
+
assemble() {
|
|
3279
|
+
return [
|
|
3280
|
+
...slots.filter((result) => result !== void 0),
|
|
3281
|
+
...extras
|
|
3282
|
+
];
|
|
3283
|
+
}
|
|
3284
|
+
};
|
|
3285
|
+
}
|
|
2622
3286
|
async function translateWorklist(config, items, options = {}) {
|
|
3287
|
+
const mode = options.mode ?? "batch";
|
|
2623
3288
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2624
3289
|
const startedAt = Date.now();
|
|
2625
|
-
|
|
2626
|
-
|
|
3290
|
+
if (options.dryRun) {
|
|
3291
|
+
options.onProgress?.({
|
|
3292
|
+
type: "start",
|
|
3293
|
+
total: items.length,
|
|
3294
|
+
concurrency,
|
|
3295
|
+
dryRun: true,
|
|
3296
|
+
model: options.model,
|
|
3297
|
+
mode
|
|
3298
|
+
});
|
|
3299
|
+
const results2 = items.map((item) => {
|
|
3300
|
+
const itemStartedAt = Date.now();
|
|
3301
|
+
try {
|
|
3302
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3303
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3304
|
+
...baseForItem(item),
|
|
3305
|
+
skipped: false,
|
|
3306
|
+
model: outcome.prepared.model,
|
|
3307
|
+
durationMs: Date.now() - itemStartedAt
|
|
3308
|
+
};
|
|
3309
|
+
} catch (error) {
|
|
3310
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3311
|
+
}
|
|
3312
|
+
});
|
|
3313
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3314
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3315
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3316
|
+
return results2;
|
|
3317
|
+
}
|
|
3318
|
+
const pending = readPendingBatchWork(config);
|
|
3319
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3320
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3321
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3322
|
+
).length;
|
|
2627
3323
|
options.onProgress?.({
|
|
2628
3324
|
type: "start",
|
|
2629
|
-
total: items.length,
|
|
3325
|
+
total: items.length + resumedExtraCount,
|
|
2630
3326
|
concurrency,
|
|
2631
|
-
dryRun:
|
|
2632
|
-
model: options.model
|
|
3327
|
+
dryRun: false,
|
|
3328
|
+
model: options.model,
|
|
3329
|
+
mode
|
|
2633
3330
|
});
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
3331
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3332
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3333
|
+
if (mode === "direct") {
|
|
3334
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3335
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3336
|
+
const active = /* @__PURE__ */ new Set();
|
|
3337
|
+
await Promise.all([
|
|
3338
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3339
|
+
const label = labelForItem(item);
|
|
3340
|
+
active.add(label);
|
|
3341
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3342
|
+
const result = await translatePage(config, item, options);
|
|
3343
|
+
active.delete(label);
|
|
3344
|
+
collector.emit(result);
|
|
3345
|
+
}),
|
|
3346
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3347
|
+
onProgress: options.onProgress,
|
|
3348
|
+
onResult: collector.emit
|
|
3349
|
+
}) : Promise.resolve()
|
|
3350
|
+
]);
|
|
3351
|
+
} else {
|
|
3352
|
+
const entries = [];
|
|
3353
|
+
for (const item of workItems) {
|
|
3354
|
+
const itemStartedAt = Date.now();
|
|
3355
|
+
try {
|
|
3356
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3357
|
+
if (outcome.status === "done") {
|
|
3358
|
+
collector.emit(outcome.result);
|
|
3359
|
+
} else {
|
|
3360
|
+
entries.push({
|
|
3361
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3362
|
+
prompt: outcome.prepared.prompt,
|
|
3363
|
+
prepared: outcome.prepared
|
|
3364
|
+
});
|
|
3365
|
+
}
|
|
3366
|
+
} catch (error) {
|
|
3367
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
const plans = planBatchJobs(entries);
|
|
3371
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3372
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3373
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3374
|
+
const submitted = await Promise.all(
|
|
3375
|
+
plans.map(async (plan, planIndex) => {
|
|
3376
|
+
try {
|
|
3377
|
+
return await submitBatchJobPlan(config, {
|
|
3378
|
+
plan,
|
|
3379
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3380
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3381
|
+
jobCount,
|
|
3382
|
+
onProgress: options.onProgress
|
|
3383
|
+
});
|
|
3384
|
+
} catch (error) {
|
|
3385
|
+
for (const entry of plan.entries) {
|
|
3386
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3387
|
+
}
|
|
3388
|
+
return void 0;
|
|
3389
|
+
}
|
|
3390
|
+
})
|
|
3391
|
+
);
|
|
3392
|
+
const jobsToPoll = [
|
|
3393
|
+
...pending.jobs,
|
|
3394
|
+
...submitted.filter((row) => row !== void 0)
|
|
3395
|
+
];
|
|
3396
|
+
if (jobsToPoll.length > 0) {
|
|
3397
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3398
|
+
jobCount,
|
|
3399
|
+
onProgress: options.onProgress,
|
|
3400
|
+
onResult: collector.emit
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
const results = collector.assemble();
|
|
3405
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3406
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3407
|
+
return results;
|
|
3408
|
+
}
|
|
3409
|
+
function countPendingItems(config, jobs) {
|
|
3410
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3411
|
+
const db = openStore(config, "readonly");
|
|
3412
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3413
|
+
for (const job of jobs) {
|
|
3414
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3415
|
+
}
|
|
3416
|
+
db.close();
|
|
3417
|
+
return counts;
|
|
3418
|
+
}
|
|
3419
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3420
|
+
jobs.forEach((job, jobIndex) => {
|
|
3421
|
+
onProgress?.({
|
|
3422
|
+
type: "batch-submitted",
|
|
3423
|
+
name: job.job_name,
|
|
3424
|
+
count: counts.get(job.id) ?? 0,
|
|
3425
|
+
model: job.display_model,
|
|
3426
|
+
jobIndex,
|
|
3427
|
+
jobCount,
|
|
3428
|
+
resumed: true,
|
|
3429
|
+
createdAt: job.created_at
|
|
3430
|
+
});
|
|
3431
|
+
});
|
|
3432
|
+
}
|
|
3433
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3434
|
+
const startedAt = Date.now();
|
|
3435
|
+
const pending = readPendingBatchWork(config);
|
|
3436
|
+
if (pending.jobs.length === 0) return null;
|
|
3437
|
+
options.onProgress?.({
|
|
3438
|
+
type: "start",
|
|
3439
|
+
total: pending.pendingItems.length,
|
|
3440
|
+
concurrency: 1,
|
|
3441
|
+
dryRun: false,
|
|
3442
|
+
mode: "batch"
|
|
3443
|
+
});
|
|
3444
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3445
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3446
|
+
const results = [];
|
|
3447
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3448
|
+
onProgress: options.onProgress,
|
|
3449
|
+
onResult: (result) => {
|
|
3450
|
+
results.push(result);
|
|
3451
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3452
|
+
}
|
|
2642
3453
|
});
|
|
2643
3454
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2644
3455
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3256,35 +4067,43 @@ async function promptContentType(config) {
|
|
|
3256
4067
|
})
|
|
3257
4068
|
);
|
|
3258
4069
|
}
|
|
4070
|
+
var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
|
|
4071
|
+
var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
|
|
4072
|
+
async function promptManualLocales(config) {
|
|
4073
|
+
const locales = nonDefaultLocales(config);
|
|
4074
|
+
if (locales.length <= 1) return {};
|
|
4075
|
+
const picked = await runPrompt(
|
|
4076
|
+
prompts.checkbox({
|
|
4077
|
+
message: "Locales to translate",
|
|
4078
|
+
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
4079
|
+
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
4080
|
+
})
|
|
4081
|
+
);
|
|
4082
|
+
return picked.length === locales.length ? {} : { locale: picked };
|
|
4083
|
+
}
|
|
3259
4084
|
async function promptLocalePreset(config) {
|
|
3260
4085
|
const presets = Object.entries(config.localePresets ?? {}).filter(
|
|
3261
4086
|
(entry) => Array.isArray(entry[1])
|
|
3262
4087
|
);
|
|
3263
4088
|
if (presets.length === 0) {
|
|
3264
|
-
|
|
3265
|
-
if (locales.length <= 1) return {};
|
|
3266
|
-
const picked = await runPrompt(
|
|
3267
|
-
prompts.checkbox({
|
|
3268
|
-
message: "Locales to translate",
|
|
3269
|
-
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
3270
|
-
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
3271
|
-
})
|
|
3272
|
-
);
|
|
3273
|
-
return picked.length === locales.length ? {} : { locale: picked };
|
|
4089
|
+
return promptManualLocales(config);
|
|
3274
4090
|
}
|
|
3275
4091
|
const choice = await runPrompt(
|
|
3276
4092
|
prompts.select({
|
|
3277
4093
|
message: "Locale preset",
|
|
3278
4094
|
choices: [
|
|
3279
|
-
{ name: "All locales", value:
|
|
4095
|
+
{ name: "All locales", value: ALL_LOCALES },
|
|
3280
4096
|
...presets.map(([name, locales]) => ({
|
|
3281
4097
|
name: `${name} (${locales.join(", ")})`,
|
|
3282
4098
|
value: name
|
|
3283
|
-
}))
|
|
4099
|
+
})),
|
|
4100
|
+
{ name: "Choose locales manually\u2026", value: PICK_LOCALES }
|
|
3284
4101
|
]
|
|
3285
4102
|
})
|
|
3286
4103
|
);
|
|
3287
|
-
|
|
4104
|
+
if (choice === PICK_LOCALES) return promptManualLocales(config);
|
|
4105
|
+
if (choice === ALL_LOCALES) return {};
|
|
4106
|
+
return { preset: choice };
|
|
3288
4107
|
}
|
|
3289
4108
|
async function promptStrategy() {
|
|
3290
4109
|
return runPrompt(
|
|
@@ -3298,12 +4117,25 @@ async function promptStrategy() {
|
|
|
3298
4117
|
})
|
|
3299
4118
|
);
|
|
3300
4119
|
}
|
|
4120
|
+
async function promptMode() {
|
|
4121
|
+
return runPrompt(
|
|
4122
|
+
prompts.select({
|
|
4123
|
+
message: "Translation mode",
|
|
4124
|
+
choices: [
|
|
4125
|
+
{ name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
|
|
4126
|
+
{ name: "Direct \u2014 immediate results, full price", value: "direct" }
|
|
4127
|
+
],
|
|
4128
|
+
default: "batch"
|
|
4129
|
+
})
|
|
4130
|
+
);
|
|
4131
|
+
}
|
|
3301
4132
|
async function promptTranslateSelection(config, flags) {
|
|
3302
4133
|
const selection = {
|
|
3303
4134
|
contentType: flags.type,
|
|
3304
4135
|
preset: flags.preset,
|
|
3305
4136
|
locale: flags.locale,
|
|
3306
|
-
strategy: flags.strategy
|
|
4137
|
+
strategy: flags.strategy,
|
|
4138
|
+
mode: flags.mode
|
|
3307
4139
|
};
|
|
3308
4140
|
if (!isInteractive()) return selection;
|
|
3309
4141
|
if (!selection.contentType) {
|
|
@@ -3315,6 +4147,9 @@ async function promptTranslateSelection(config, flags) {
|
|
|
3315
4147
|
if (!selection.strategy) {
|
|
3316
4148
|
selection.strategy = await promptStrategy();
|
|
3317
4149
|
}
|
|
4150
|
+
if (!selection.mode) {
|
|
4151
|
+
selection.mode = await promptMode();
|
|
4152
|
+
}
|
|
3318
4153
|
return selection;
|
|
3319
4154
|
}
|
|
3320
4155
|
|
|
@@ -3336,6 +4171,34 @@ function progressBar(ratio, width = 28) {
|
|
|
3336
4171
|
function labelForResult(result) {
|
|
3337
4172
|
return `${result.contentType}/${result.enSlug}@${result.locale}`;
|
|
3338
4173
|
}
|
|
4174
|
+
function shortJobId(name) {
|
|
4175
|
+
const segments = name.split("/");
|
|
4176
|
+
return segments[segments.length - 1] || name;
|
|
4177
|
+
}
|
|
4178
|
+
function formatElapsed(elapsedMs) {
|
|
4179
|
+
return `${Math.round(elapsedMs / 1e3)}s`;
|
|
4180
|
+
}
|
|
4181
|
+
function batchDoneParts(event) {
|
|
4182
|
+
if (event.alreadyIngested) {
|
|
4183
|
+
return [
|
|
4184
|
+
`job ${shortJobId(event.name)}`,
|
|
4185
|
+
event.model ?? "?",
|
|
4186
|
+
"already ingested by another scribe run (results saved there)",
|
|
4187
|
+
formatElapsed(event.elapsedMs)
|
|
4188
|
+
].join(" \xB7 ");
|
|
4189
|
+
}
|
|
4190
|
+
const parts = [
|
|
4191
|
+
`job ${shortJobId(event.name)}`,
|
|
4192
|
+
event.model ?? "?",
|
|
4193
|
+
`${event.count} request${event.count === 1 ? "" : "s"}`,
|
|
4194
|
+
`${event.translated} translated`
|
|
4195
|
+
];
|
|
4196
|
+
if (event.failed > 0) parts.push(`${event.failed} failed`);
|
|
4197
|
+
parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
|
|
4198
|
+
parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
|
|
4199
|
+
parts.push(formatElapsed(event.elapsedMs));
|
|
4200
|
+
return parts.join(" \xB7 ");
|
|
4201
|
+
}
|
|
3339
4202
|
function statusForResult(result, dryRun) {
|
|
3340
4203
|
if (result.failed) return red("failed");
|
|
3341
4204
|
if (result.skipped) return dim("skipped");
|
|
@@ -3381,6 +4244,25 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3381
4244
|
if (!enabled) {
|
|
3382
4245
|
return {
|
|
3383
4246
|
onEvent(event) {
|
|
4247
|
+
if (event.type === "batch-submitted") {
|
|
4248
|
+
const verb = event.resumed ? "Resuming" : "Submitted";
|
|
4249
|
+
console.log(
|
|
4250
|
+
`${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
|
|
4251
|
+
);
|
|
4252
|
+
return;
|
|
4253
|
+
}
|
|
4254
|
+
if (event.type === "batch-polling") {
|
|
4255
|
+
console.log(
|
|
4256
|
+
`batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
|
|
4257
|
+
);
|
|
4258
|
+
return;
|
|
4259
|
+
}
|
|
4260
|
+
if (event.type === "batch-done") {
|
|
4261
|
+
const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
|
|
4262
|
+
if (event.failed > 0 && event.translated === 0) console.error(line);
|
|
4263
|
+
else console.log(line);
|
|
4264
|
+
return;
|
|
4265
|
+
}
|
|
3384
4266
|
if (event.type === "item-done") {
|
|
3385
4267
|
const label = labelForResult(event.result);
|
|
3386
4268
|
const status = statusForResult(event.result, dryRun);
|
|
@@ -3411,6 +4293,9 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3411
4293
|
let outputTokens = 0;
|
|
3412
4294
|
let estimatedCostUsd = 0;
|
|
3413
4295
|
let active = [];
|
|
4296
|
+
let mode;
|
|
4297
|
+
const batchJobs = /* @__PURE__ */ new Map();
|
|
4298
|
+
let batchElapsedMs = 0;
|
|
3414
4299
|
const recent = [];
|
|
3415
4300
|
let renderedLines = 0;
|
|
3416
4301
|
let cursorHidden = false;
|
|
@@ -3426,6 +4311,15 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3426
4311
|
cursorHidden = false;
|
|
3427
4312
|
}
|
|
3428
4313
|
}
|
|
4314
|
+
function printPersistent(lines) {
|
|
4315
|
+
if (renderedLines > 0) {
|
|
4316
|
+
process.stdout.write(`\x1B[${renderedLines}A`);
|
|
4317
|
+
}
|
|
4318
|
+
for (const line of lines) {
|
|
4319
|
+
process.stdout.write("\x1B[2K" + line + "\n");
|
|
4320
|
+
}
|
|
4321
|
+
renderedLines = 0;
|
|
4322
|
+
}
|
|
3429
4323
|
function render() {
|
|
3430
4324
|
hideCursor();
|
|
3431
4325
|
if (renderedLines > 0) {
|
|
@@ -3433,12 +4327,34 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3433
4327
|
}
|
|
3434
4328
|
const lines = [];
|
|
3435
4329
|
const title = dryRun ? "Dry run" : "Translating";
|
|
3436
|
-
|
|
4330
|
+
const isBatch = mode === "batch" || batchJobs.size > 0;
|
|
4331
|
+
lines.push(
|
|
4332
|
+
cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
|
|
4333
|
+
);
|
|
3437
4334
|
lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
|
|
3438
4335
|
lines.push(
|
|
3439
4336
|
dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
|
|
3440
4337
|
);
|
|
3441
|
-
if (
|
|
4338
|
+
if (batchJobs.size > 0 && done < total) {
|
|
4339
|
+
const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
|
|
4340
|
+
let running = 0;
|
|
4341
|
+
let finished = 0;
|
|
4342
|
+
let requests = 0;
|
|
4343
|
+
for (const job of batchJobs.values()) {
|
|
4344
|
+
if (terminal.test(job.state)) finished += 1;
|
|
4345
|
+
else running += 1;
|
|
4346
|
+
requests += job.count;
|
|
4347
|
+
}
|
|
4348
|
+
lines.push(
|
|
4349
|
+
dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
|
|
4350
|
+
);
|
|
4351
|
+
for (const [name, job] of batchJobs) {
|
|
4352
|
+
if (terminal.test(job.state)) continue;
|
|
4353
|
+
lines.push(
|
|
4354
|
+
dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
|
|
4355
|
+
);
|
|
4356
|
+
}
|
|
4357
|
+
} else if (active.length > 0) {
|
|
3442
4358
|
lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
|
|
3443
4359
|
} else if (done < total) {
|
|
3444
4360
|
lines.push(dim("Active ") + "starting\u2026");
|
|
@@ -3466,12 +4382,58 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3466
4382
|
total = event.total;
|
|
3467
4383
|
concurrency = event.concurrency;
|
|
3468
4384
|
model = event.model;
|
|
4385
|
+
mode = event.mode;
|
|
3469
4386
|
render();
|
|
3470
4387
|
break;
|
|
3471
4388
|
case "item-start":
|
|
3472
4389
|
active = event.active;
|
|
3473
4390
|
render();
|
|
3474
4391
|
break;
|
|
4392
|
+
case "batch-submitted": {
|
|
4393
|
+
const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
|
|
4394
|
+
const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
|
|
4395
|
+
batchJobs.set(event.name, {
|
|
4396
|
+
state: event.resumed ? "RESUMED" : "SUBMITTED",
|
|
4397
|
+
count: event.count,
|
|
4398
|
+
model: event.model,
|
|
4399
|
+
createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
|
|
4400
|
+
elapsedMs
|
|
4401
|
+
});
|
|
4402
|
+
const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
|
|
4403
|
+
const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
|
|
4404
|
+
printPersistent([
|
|
4405
|
+
dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
|
|
4406
|
+
]);
|
|
4407
|
+
render();
|
|
4408
|
+
break;
|
|
4409
|
+
}
|
|
4410
|
+
case "batch-polling": {
|
|
4411
|
+
const job = batchJobs.get(event.name);
|
|
4412
|
+
batchJobs.set(event.name, {
|
|
4413
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4414
|
+
count: job?.count ?? 0,
|
|
4415
|
+
model: job?.model,
|
|
4416
|
+
createdAtMs: job?.createdAtMs,
|
|
4417
|
+
elapsedMs: event.elapsedMs
|
|
4418
|
+
});
|
|
4419
|
+
batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
|
|
4420
|
+
render();
|
|
4421
|
+
break;
|
|
4422
|
+
}
|
|
4423
|
+
case "batch-done": {
|
|
4424
|
+
const job = batchJobs.get(event.name);
|
|
4425
|
+
batchJobs.set(event.name, {
|
|
4426
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4427
|
+
count: job?.count ?? event.count,
|
|
4428
|
+
model: job?.model ?? event.model,
|
|
4429
|
+
createdAtMs: job?.createdAtMs,
|
|
4430
|
+
elapsedMs: event.elapsedMs
|
|
4431
|
+
});
|
|
4432
|
+
const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
|
|
4433
|
+
printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
|
|
4434
|
+
render();
|
|
4435
|
+
break;
|
|
4436
|
+
}
|
|
3475
4437
|
case "item-done": {
|
|
3476
4438
|
done += 1;
|
|
3477
4439
|
active = active.filter((label) => label !== labelForResult(event.result));
|
|
@@ -3594,6 +4556,21 @@ function parseArgs(argv) {
|
|
|
3594
4556
|
i++;
|
|
3595
4557
|
continue;
|
|
3596
4558
|
}
|
|
4559
|
+
if (arg === "--batch") {
|
|
4560
|
+
options.batch = true;
|
|
4561
|
+
i++;
|
|
4562
|
+
continue;
|
|
4563
|
+
}
|
|
4564
|
+
if (arg === "--direct") {
|
|
4565
|
+
options.direct = true;
|
|
4566
|
+
i++;
|
|
4567
|
+
continue;
|
|
4568
|
+
}
|
|
4569
|
+
if (arg === "--resume") {
|
|
4570
|
+
options.resume = true;
|
|
4571
|
+
i++;
|
|
4572
|
+
continue;
|
|
4573
|
+
}
|
|
3597
4574
|
if (arg === "--strategy") {
|
|
3598
4575
|
const value = argv[++i];
|
|
3599
4576
|
if (value !== "all" && value !== "missing-only") {
|
|
@@ -3662,11 +4639,19 @@ Translate flags:
|
|
|
3662
4639
|
--locale <code>... Target locale(s); overrides --preset
|
|
3663
4640
|
--slug <en-slug> Single English document
|
|
3664
4641
|
--model <id> Gemini model override
|
|
3665
|
-
--
|
|
4642
|
+
--batch Force Gemini Batch API mode (50% token cost; default)
|
|
4643
|
+
--direct Force direct interactive calls (immediate, full price)
|
|
4644
|
+
--resume Only poll/ingest pending batch jobs; submit nothing new
|
|
4645
|
+
--concurrency <n> Parallel translations in --direct mode (default: 3)
|
|
3666
4646
|
--dry-run List work without writing
|
|
3667
4647
|
--force Re-translate even when hashes match
|
|
3668
4648
|
--strategy <mode> all (default) or missing-only
|
|
3669
4649
|
--no-progress Plain line logging instead of live progress
|
|
4650
|
+
|
|
4651
|
+
Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
|
|
4652
|
+
during polling is safe: the next run (or --resume) picks the jobs back up and
|
|
4653
|
+
ingests their results. In a TTY without --batch/--direct, a prompt asks which
|
|
4654
|
+
mode to use.
|
|
3670
4655
|
`);
|
|
3671
4656
|
return;
|
|
3672
4657
|
}
|
|
@@ -3712,11 +4697,29 @@ Translate flags:
|
|
|
3712
4697
|
break;
|
|
3713
4698
|
}
|
|
3714
4699
|
case "translate": {
|
|
4700
|
+
if (options.batch && options.direct) {
|
|
4701
|
+
throw new Error("--batch and --direct are mutually exclusive");
|
|
4702
|
+
}
|
|
4703
|
+
if (options.resume) {
|
|
4704
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4705
|
+
enabled: !options.noProgress,
|
|
4706
|
+
dryRun: false
|
|
4707
|
+
});
|
|
4708
|
+
const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4709
|
+
reporter2.finish();
|
|
4710
|
+
if (results2 === null) {
|
|
4711
|
+
console.log("No pending translation batch jobs.");
|
|
4712
|
+
break;
|
|
4713
|
+
}
|
|
4714
|
+
if (results2.some((result) => result.failed)) process.exitCode = 1;
|
|
4715
|
+
break;
|
|
4716
|
+
}
|
|
3715
4717
|
const selection = await promptTranslateSelection(config, {
|
|
3716
4718
|
type: options.type,
|
|
3717
4719
|
preset: options.preset,
|
|
3718
4720
|
locale: options.locale,
|
|
3719
|
-
strategy: options.strategy
|
|
4721
|
+
strategy: options.strategy,
|
|
4722
|
+
mode: options.batch ? "batch" : options.direct ? "direct" : void 0
|
|
3720
4723
|
});
|
|
3721
4724
|
const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
|
|
3722
4725
|
const worklist = buildWorklist(config, {
|
|
@@ -3726,6 +4729,18 @@ Translate flags:
|
|
|
3726
4729
|
strategy: selection.strategy ?? "all"
|
|
3727
4730
|
});
|
|
3728
4731
|
if (worklist.length === 0) {
|
|
4732
|
+
if (!options.dryRun) {
|
|
4733
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4734
|
+
enabled: !options.noProgress,
|
|
4735
|
+
dryRun: false
|
|
4736
|
+
});
|
|
4737
|
+
const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4738
|
+
reporter2.finish();
|
|
4739
|
+
if (resumed !== null) {
|
|
4740
|
+
if (resumed.some((result) => result.failed)) process.exitCode = 1;
|
|
4741
|
+
break;
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
3729
4744
|
console.log("Nothing to translate.");
|
|
3730
4745
|
break;
|
|
3731
4746
|
}
|
|
@@ -3738,6 +4753,7 @@ Translate flags:
|
|
|
3738
4753
|
dryRun: options.dryRun,
|
|
3739
4754
|
force: options.force,
|
|
3740
4755
|
concurrency: options.concurrency,
|
|
4756
|
+
mode: selection.mode,
|
|
3741
4757
|
onProgress: reporter.onEvent
|
|
3742
4758
|
});
|
|
3743
4759
|
reporter.finish();
|