scribe-cms 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1258 -219
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1259 -220
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +1020 -181
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1021 -183
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +149 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +149 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +23 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/src/validate/validate-project.d.ts.map +1 -1
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import remarkParse from 'remark-parse';
|
|
|
10
10
|
import { unified } from 'unified';
|
|
11
11
|
import { createJiti } from 'jiti';
|
|
12
12
|
import { createHash } from 'crypto';
|
|
13
|
-
import { GoogleGenAI } from '@google/genai';
|
|
13
|
+
import { GoogleGenAI, ThinkingLevel, ApiError } from '@google/genai';
|
|
14
14
|
import dotenv from 'dotenv';
|
|
15
15
|
import { serve } from '@hono/node-server';
|
|
16
16
|
import { Hono } from 'hono';
|
|
@@ -59,6 +59,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
function readSchemaVersion(db) {
|
|
62
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
63
|
+
if (!metaTable) return 0;
|
|
62
64
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
63
65
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
64
66
|
}
|
|
@@ -83,7 +85,7 @@ var SCHEMA_VERSION, MIGRATIONS;
|
|
|
83
85
|
var init_sqlite = __esm({
|
|
84
86
|
"src/storage/sqlite.ts"() {
|
|
85
87
|
init_esm_shims();
|
|
86
|
-
SCHEMA_VERSION =
|
|
88
|
+
SCHEMA_VERSION = 5;
|
|
87
89
|
MIGRATIONS = [
|
|
88
90
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
89
91
|
key TEXT PRIMARY KEY,
|
|
@@ -131,7 +133,30 @@ var init_sqlite = __esm({
|
|
|
131
133
|
UNIQUE (content_type, en_slug, en_hash)
|
|
132
134
|
)`,
|
|
133
135
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
134
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
136
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
137
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
138
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
139
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
140
|
+
model TEXT NOT NULL,
|
|
141
|
+
display_model TEXT NOT NULL,
|
|
142
|
+
created_at TEXT NOT NULL,
|
|
143
|
+
state TEXT NOT NULL,
|
|
144
|
+
completed_at TEXT
|
|
145
|
+
)`,
|
|
146
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
147
|
+
job_id INTEGER NOT NULL,
|
|
148
|
+
request_index INTEGER NOT NULL,
|
|
149
|
+
content_type TEXT NOT NULL,
|
|
150
|
+
en_slug TEXT NOT NULL,
|
|
151
|
+
locale TEXT NOT NULL,
|
|
152
|
+
en_hash TEXT NOT NULL,
|
|
153
|
+
snapshot_id INTEGER NOT NULL,
|
|
154
|
+
status TEXT NOT NULL,
|
|
155
|
+
error TEXT,
|
|
156
|
+
PRIMARY KEY (job_id, request_index)
|
|
157
|
+
)`,
|
|
158
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
159
|
+
ON translation_batch_items(status)`
|
|
135
160
|
];
|
|
136
161
|
}
|
|
137
162
|
});
|
|
@@ -173,26 +198,6 @@ function getRelationTarget(schema) {
|
|
|
173
198
|
}
|
|
174
199
|
return null;
|
|
175
200
|
}
|
|
176
|
-
function unwrapSchema(schema) {
|
|
177
|
-
let current = schema;
|
|
178
|
-
for (let i = 0; i < 8; i++) {
|
|
179
|
-
const anySchema = current;
|
|
180
|
-
if (typeof anySchema.unwrap === "function") {
|
|
181
|
-
current = anySchema.unwrap();
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
if (typeof anySchema.removeDefault === "function") {
|
|
185
|
-
current = anySchema.removeDefault();
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
if (anySchema._def?.innerType) {
|
|
189
|
-
current = anySchema._def.innerType;
|
|
190
|
-
continue;
|
|
191
|
-
}
|
|
192
|
-
break;
|
|
193
|
-
}
|
|
194
|
-
return current;
|
|
195
|
-
}
|
|
196
201
|
function peelOptionalWrappers(schema) {
|
|
197
202
|
let current = schema;
|
|
198
203
|
for (let i = 0; i < 8; i++) {
|
|
@@ -338,6 +343,39 @@ function resolveConfig(input, baseDir) {
|
|
|
338
343
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
339
344
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
340
345
|
}
|
|
346
|
+
const localeFallbacks = {};
|
|
347
|
+
for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
|
|
348
|
+
if (!raw.locales.includes(locale)) {
|
|
349
|
+
throw new Error(
|
|
350
|
+
`scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
const seen = /* @__PURE__ */ new Set();
|
|
354
|
+
for (const fallback of chain) {
|
|
355
|
+
if (!raw.locales.includes(fallback)) {
|
|
356
|
+
throw new Error(
|
|
357
|
+
`scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
if (fallback === locale) {
|
|
361
|
+
throw new Error(
|
|
362
|
+
`scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
if (fallback === defaultLocale) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
if (seen.has(fallback)) {
|
|
371
|
+
throw new Error(
|
|
372
|
+
`scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
seen.add(fallback);
|
|
376
|
+
}
|
|
377
|
+
localeFallbacks[locale] = [...chain];
|
|
378
|
+
}
|
|
341
379
|
const projectRoot = path4.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
342
380
|
const contentRoot = path4.resolve(projectRoot, raw.contentDir ?? "content");
|
|
343
381
|
const storePath = path4.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -367,6 +405,7 @@ function resolveConfig(input, baseDir) {
|
|
|
367
405
|
defaultLocale,
|
|
368
406
|
localeRouting,
|
|
369
407
|
localePresets: raw.localePresets,
|
|
408
|
+
localeFallbacks,
|
|
370
409
|
translate: raw.translate,
|
|
371
410
|
types
|
|
372
411
|
};
|
|
@@ -382,6 +421,12 @@ init_esm_shims();
|
|
|
382
421
|
|
|
383
422
|
// src/core/introspect-schema.ts
|
|
384
423
|
init_esm_shims();
|
|
424
|
+
function getArrayElement(schema) {
|
|
425
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
426
|
+
return schema.element;
|
|
427
|
+
}
|
|
428
|
+
return null;
|
|
429
|
+
}
|
|
385
430
|
function introspectSchema(schema, prefix = []) {
|
|
386
431
|
const relation = getRelationTarget(schema);
|
|
387
432
|
if (relation && prefix.length > 0) {
|
|
@@ -395,7 +440,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
395
440
|
}
|
|
396
441
|
];
|
|
397
442
|
}
|
|
398
|
-
|
|
443
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
444
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
445
|
+
}
|
|
446
|
+
const base = peelOptionalWrappers(schema);
|
|
399
447
|
if (base instanceof Object && "shape" in base) {
|
|
400
448
|
const shape = base.shape;
|
|
401
449
|
const fields = [];
|
|
@@ -404,9 +452,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
404
452
|
}
|
|
405
453
|
return fields;
|
|
406
454
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
const elementBase =
|
|
455
|
+
const element = getArrayElement(base);
|
|
456
|
+
if (element) {
|
|
457
|
+
const elementBase = peelOptionalWrappers(element);
|
|
410
458
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
411
459
|
const shape = elementBase.shape;
|
|
412
460
|
const fields = [];
|
|
@@ -418,15 +466,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
418
466
|
}
|
|
419
467
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
420
468
|
}
|
|
421
|
-
function
|
|
422
|
-
const
|
|
469
|
+
function buildPathTrie(paths) {
|
|
470
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
423
471
|
for (const path16 of paths) {
|
|
424
|
-
|
|
472
|
+
let node = root;
|
|
473
|
+
for (const segment of path16) {
|
|
474
|
+
let child = node.children.get(segment);
|
|
475
|
+
if (!child) {
|
|
476
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
477
|
+
node.children.set(segment, child);
|
|
478
|
+
}
|
|
479
|
+
node = child;
|
|
480
|
+
}
|
|
481
|
+
node.leaf = true;
|
|
482
|
+
}
|
|
483
|
+
return root;
|
|
484
|
+
}
|
|
485
|
+
function extractNode(data, trie) {
|
|
486
|
+
if (trie.leaf) return data;
|
|
487
|
+
const star = trie.children.get("*");
|
|
488
|
+
if (star) {
|
|
489
|
+
if (!Array.isArray(data)) return void 0;
|
|
490
|
+
return data.map(
|
|
491
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
495
|
+
const record = data;
|
|
496
|
+
const out = {};
|
|
497
|
+
let any = false;
|
|
498
|
+
for (const [key, child] of trie.children) {
|
|
499
|
+
const value = extractNode(record[key], child);
|
|
425
500
|
if (value !== void 0) {
|
|
426
|
-
|
|
501
|
+
out[key] = value;
|
|
502
|
+
any = true;
|
|
427
503
|
}
|
|
428
504
|
}
|
|
429
|
-
return out;
|
|
505
|
+
return any ? out : void 0;
|
|
506
|
+
}
|
|
507
|
+
function extractByPaths(data, paths) {
|
|
508
|
+
if (paths.length === 0) return {};
|
|
509
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
510
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
430
511
|
}
|
|
431
512
|
function pickTranslatable(data, schema) {
|
|
432
513
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -441,32 +522,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
441
522
|
const translatable = pickTranslatable(localeData, schema);
|
|
442
523
|
return deepMerge(structural, translatable);
|
|
443
524
|
}
|
|
444
|
-
function getAtPath(obj, path16) {
|
|
445
|
-
let current = obj;
|
|
446
|
-
for (const segment of path16) {
|
|
447
|
-
if (segment === "*") {
|
|
448
|
-
if (!Array.isArray(current)) return void 0;
|
|
449
|
-
return current.map(
|
|
450
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path16.slice(path16.indexOf("*") + 1)) : void 0
|
|
451
|
-
);
|
|
452
|
-
}
|
|
453
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
454
|
-
current = current[segment];
|
|
455
|
-
}
|
|
456
|
-
return current;
|
|
457
|
-
}
|
|
458
|
-
function setAtPath(obj, path16, value) {
|
|
459
|
-
if (path16.length === 0) return;
|
|
460
|
-
if (path16.length === 1) {
|
|
461
|
-
obj[path16[0]] = value;
|
|
462
|
-
return;
|
|
463
|
-
}
|
|
464
|
-
const [head, ...rest] = path16;
|
|
465
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
466
|
-
obj[head] = {};
|
|
467
|
-
}
|
|
468
|
-
setAtPath(obj[head], rest, value);
|
|
469
|
-
}
|
|
470
525
|
function deepMerge(base, overlay) {
|
|
471
526
|
const out = { ...base };
|
|
472
527
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -757,7 +812,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
757
812
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
758
813
|
const attrStart = i;
|
|
759
814
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
760
|
-
body.slice(attrStart, i);
|
|
815
|
+
const attrName = body.slice(attrStart, i);
|
|
816
|
+
if (!attrName) break;
|
|
761
817
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
762
818
|
if (body[i] !== "=") {
|
|
763
819
|
tagOut += body.slice(attrStart, i);
|
|
@@ -766,10 +822,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
766
822
|
tagOut += body.slice(attrStart, i);
|
|
767
823
|
tagOut += "=";
|
|
768
824
|
i += 1;
|
|
825
|
+
const wsStart = i;
|
|
769
826
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
827
|
+
tagOut += body.slice(wsStart, i);
|
|
770
828
|
const quote = body[i];
|
|
771
829
|
if (quote !== '"' && quote !== "'") {
|
|
772
|
-
|
|
830
|
+
break;
|
|
773
831
|
}
|
|
774
832
|
if (quote === "'") {
|
|
775
833
|
const valStart2 = i + 1;
|
|
@@ -1091,7 +1149,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1091
1149
|
|
|
1092
1150
|
// src/i18n/resolve-document.ts
|
|
1093
1151
|
init_esm_shims();
|
|
1094
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1152
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1095
1153
|
const urlBuilder = createUrlBuilder({
|
|
1096
1154
|
locales: [defaultLocale, locale],
|
|
1097
1155
|
defaultLocale,
|
|
@@ -1105,15 +1163,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1105
1163
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1106
1164
|
const found = docIdx.bySlug.get(slug);
|
|
1107
1165
|
if (!found) continue;
|
|
1108
|
-
const
|
|
1109
|
-
|
|
1166
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1167
|
+
let target;
|
|
1168
|
+
let targetLocale = locale;
|
|
1169
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1170
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1171
|
+
if (cand) {
|
|
1172
|
+
target = cand;
|
|
1173
|
+
targetLocale = candidateLocale;
|
|
1174
|
+
break;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
if (target) {
|
|
1178
|
+
if (target.slug === slug) {
|
|
1179
|
+
return { document: target, actualLocale: targetLocale };
|
|
1180
|
+
}
|
|
1110
1181
|
if (!type.path) {
|
|
1111
1182
|
return { document: null, actualLocale: locale };
|
|
1112
1183
|
}
|
|
1113
1184
|
return {
|
|
1114
1185
|
document: null,
|
|
1115
1186
|
actualLocale: locale,
|
|
1116
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1187
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1117
1188
|
};
|
|
1118
1189
|
}
|
|
1119
1190
|
if (docLocale === defaultLocale) {
|
|
@@ -1144,13 +1215,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1144
1215
|
}
|
|
1145
1216
|
return { document: null, actualLocale: locale };
|
|
1146
1217
|
}
|
|
1147
|
-
function getSlugForLocale(document, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1148
|
-
if (sourceLocale === targetLocale) return document.slug;
|
|
1149
|
-
const englishSlug = sourceLocale === defaultLocale ? document.slug : document.enSlug;
|
|
1150
|
-
if (!englishSlug) return null;
|
|
1151
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1152
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1153
|
-
}
|
|
1154
1218
|
|
|
1155
1219
|
// src/create-project.ts
|
|
1156
1220
|
function comparatorFor(orderBy) {
|
|
@@ -1203,7 +1267,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1203
1267
|
config.defaultLocale,
|
|
1204
1268
|
load(),
|
|
1205
1269
|
type,
|
|
1206
|
-
config.localeRouting
|
|
1270
|
+
config.localeRouting,
|
|
1271
|
+
config.localeFallbacks[locale] ?? []
|
|
1207
1272
|
);
|
|
1208
1273
|
if (result.document && type.path) {
|
|
1209
1274
|
return {
|
|
@@ -1225,8 +1290,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1225
1290
|
const params = [];
|
|
1226
1291
|
for (const locale of options.locales ?? config.locales) {
|
|
1227
1292
|
const localeIdx = all.get(locale);
|
|
1293
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1228
1294
|
for (const doc of enIdx.bySlug.values()) {
|
|
1229
|
-
|
|
1295
|
+
let slug;
|
|
1296
|
+
if (locale === config.defaultLocale) {
|
|
1297
|
+
slug = doc.slug;
|
|
1298
|
+
} else {
|
|
1299
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1300
|
+
}
|
|
1230
1301
|
params.push({ locale, slug });
|
|
1231
1302
|
}
|
|
1232
1303
|
}
|
|
@@ -1688,7 +1759,7 @@ init_sqlite();
|
|
|
1688
1759
|
|
|
1689
1760
|
// src/validate/validate-relations.ts
|
|
1690
1761
|
init_esm_shims();
|
|
1691
|
-
function
|
|
1762
|
+
function getAtPath(obj, fieldPath) {
|
|
1692
1763
|
let current = obj;
|
|
1693
1764
|
for (const segment of fieldPath) {
|
|
1694
1765
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -1733,7 +1804,7 @@ function validateRelations(project) {
|
|
|
1733
1804
|
});
|
|
1734
1805
|
continue;
|
|
1735
1806
|
}
|
|
1736
|
-
const value =
|
|
1807
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
1737
1808
|
if (value === void 0 || value === null || value === "") continue;
|
|
1738
1809
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
1739
1810
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -1889,6 +1960,20 @@ function listEnSlugs3(rootDir, contentDir) {
|
|
|
1889
1960
|
if (!fs2.existsSync(dir)) return [];
|
|
1890
1961
|
return fs2.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
|
|
1891
1962
|
}
|
|
1963
|
+
function validateDocumentMdxBody(issues, input) {
|
|
1964
|
+
const preparedBody = prepareTranslatedMdxBody(input.body).body;
|
|
1965
|
+
const mdxValidation = validateMdxBody(preparedBody);
|
|
1966
|
+
if (!mdxValidation.ok) {
|
|
1967
|
+
issues.push({
|
|
1968
|
+
level: "error",
|
|
1969
|
+
contentType: input.contentType,
|
|
1970
|
+
enSlug: input.enSlug,
|
|
1971
|
+
locale: input.locale,
|
|
1972
|
+
field: "body",
|
|
1973
|
+
message: `Invalid MDX: ${mdxValidation.error}`
|
|
1974
|
+
});
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1892
1977
|
function validateProject(config) {
|
|
1893
1978
|
const issues = [];
|
|
1894
1979
|
const project = createProject(config);
|
|
@@ -1947,6 +2032,12 @@ function validateProject(config) {
|
|
|
1947
2032
|
})) {
|
|
1948
2033
|
issues.push(issue);
|
|
1949
2034
|
}
|
|
2035
|
+
validateDocumentMdxBody(issues, {
|
|
2036
|
+
contentType: type.id,
|
|
2037
|
+
enSlug,
|
|
2038
|
+
locale: config.defaultLocale,
|
|
2039
|
+
body: enDoc.content
|
|
2040
|
+
});
|
|
1950
2041
|
for (const locale of config.locales) {
|
|
1951
2042
|
if (locale === config.defaultLocale) continue;
|
|
1952
2043
|
const row = getTranslation(db, type.id, enSlug, locale);
|
|
@@ -1972,17 +2063,12 @@ function validateProject(config) {
|
|
|
1972
2063
|
})) {
|
|
1973
2064
|
issues.push(issue);
|
|
1974
2065
|
}
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
locale,
|
|
1982
|
-
field: "body",
|
|
1983
|
-
message: `Invalid MDX: ${mdxValidation.error}`
|
|
1984
|
-
});
|
|
1985
|
-
}
|
|
2066
|
+
validateDocumentMdxBody(issues, {
|
|
2067
|
+
contentType: type.id,
|
|
2068
|
+
enSlug,
|
|
2069
|
+
locale,
|
|
2070
|
+
body: row.body
|
|
2071
|
+
});
|
|
1986
2072
|
}
|
|
1987
2073
|
}
|
|
1988
2074
|
try {
|
|
@@ -2102,6 +2188,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2102
2188
|
// src/translate/page-translator.ts
|
|
2103
2189
|
init_esm_shims();
|
|
2104
2190
|
|
|
2191
|
+
// src/storage/batch-jobs.ts
|
|
2192
|
+
init_esm_shims();
|
|
2193
|
+
function insertBatchJob(db, input) {
|
|
2194
|
+
const info = db.prepare(
|
|
2195
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2196
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2197
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2198
|
+
return Number(info.lastInsertRowid);
|
|
2199
|
+
}
|
|
2200
|
+
function insertBatchItems(db, jobId, items) {
|
|
2201
|
+
const stmt = db.prepare(
|
|
2202
|
+
`INSERT INTO translation_batch_items (
|
|
2203
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2204
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2205
|
+
);
|
|
2206
|
+
for (const item of items) {
|
|
2207
|
+
stmt.run(
|
|
2208
|
+
jobId,
|
|
2209
|
+
item.requestIndex,
|
|
2210
|
+
item.contentType,
|
|
2211
|
+
item.enSlug,
|
|
2212
|
+
item.locale,
|
|
2213
|
+
item.enHash,
|
|
2214
|
+
item.snapshotId
|
|
2215
|
+
);
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
function listPendingBatchJobs(db) {
|
|
2219
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2220
|
+
}
|
|
2221
|
+
function listBatchItems(db, jobId) {
|
|
2222
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2223
|
+
}
|
|
2224
|
+
function listPendingBatchItems(db) {
|
|
2225
|
+
return db.prepare(
|
|
2226
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2227
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2228
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2229
|
+
ORDER BY i.job_id, i.request_index`
|
|
2230
|
+
).all();
|
|
2231
|
+
}
|
|
2232
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2233
|
+
const info = db.prepare(
|
|
2234
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2235
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2236
|
+
).run(state, completedAt, jobId);
|
|
2237
|
+
return info.changes > 0;
|
|
2238
|
+
}
|
|
2239
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2240
|
+
db.prepare(
|
|
2241
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2242
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
// src/translate/page-translator.ts
|
|
2246
|
+
init_sqlite();
|
|
2247
|
+
|
|
2248
|
+
// src/translate/batch-worklist.ts
|
|
2249
|
+
init_esm_shims();
|
|
2250
|
+
|
|
2105
2251
|
// src/history/record-snapshot.ts
|
|
2106
2252
|
init_esm_shims();
|
|
2107
2253
|
init_sqlite();
|
|
@@ -2119,9 +2265,111 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2119
2265
|
return id;
|
|
2120
2266
|
}
|
|
2121
2267
|
|
|
2122
|
-
// src/translate/
|
|
2268
|
+
// src/translate/batch-worklist.ts
|
|
2123
2269
|
init_sqlite();
|
|
2124
2270
|
|
|
2271
|
+
// src/translate/gemini-batch.ts
|
|
2272
|
+
init_esm_shims();
|
|
2273
|
+
|
|
2274
|
+
// src/translate/retry.ts
|
|
2275
|
+
init_esm_shims();
|
|
2276
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2277
|
+
function statusFromError(error) {
|
|
2278
|
+
if (error instanceof ApiError && typeof error.status === "number") {
|
|
2279
|
+
return error.status;
|
|
2280
|
+
}
|
|
2281
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2282
|
+
const status = error.status;
|
|
2283
|
+
if (typeof status === "number") return status;
|
|
2284
|
+
}
|
|
2285
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2286
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2287
|
+
if (match) return Number(match[1]);
|
|
2288
|
+
return void 0;
|
|
2289
|
+
}
|
|
2290
|
+
function isNetworkError(error) {
|
|
2291
|
+
const err = error;
|
|
2292
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2293
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2294
|
+
return true;
|
|
2295
|
+
}
|
|
2296
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2297
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2298
|
+
}
|
|
2299
|
+
function isTransientError(error) {
|
|
2300
|
+
const status = statusFromError(error);
|
|
2301
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2302
|
+
return isNetworkError(error);
|
|
2303
|
+
}
|
|
2304
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2305
|
+
async function withRetry(fn, options = {}) {
|
|
2306
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2307
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2308
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2309
|
+
const random = options.random ?? Math.random;
|
|
2310
|
+
let lastError;
|
|
2311
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2312
|
+
try {
|
|
2313
|
+
return await fn();
|
|
2314
|
+
} catch (error) {
|
|
2315
|
+
lastError = error;
|
|
2316
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2317
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2318
|
+
const delay = backoff + random() * backoff;
|
|
2319
|
+
await sleep2(delay);
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
throw lastError;
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
// src/translate/gemini-batch.ts
|
|
2326
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2327
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2328
|
+
"SUCCEEDED",
|
|
2329
|
+
"FAILED",
|
|
2330
|
+
"CANCELLED",
|
|
2331
|
+
"EXPIRED",
|
|
2332
|
+
"PARTIALLY_SUCCEEDED"
|
|
2333
|
+
]);
|
|
2334
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2335
|
+
function normalizeBatchState(state) {
|
|
2336
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2337
|
+
}
|
|
2338
|
+
function isTerminalBatchState(state) {
|
|
2339
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2340
|
+
}
|
|
2341
|
+
function isSuccessfulBatchState(state) {
|
|
2342
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2343
|
+
}
|
|
2344
|
+
function makeClient(apiKey) {
|
|
2345
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2346
|
+
if (!key) {
|
|
2347
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2348
|
+
}
|
|
2349
|
+
return new GoogleGenAI({ apiKey: key });
|
|
2350
|
+
}
|
|
2351
|
+
function textFromBatchResponse(response) {
|
|
2352
|
+
if (typeof response.text === "string") return response.text;
|
|
2353
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2354
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2355
|
+
}
|
|
2356
|
+
async function createGeminiBatchJob(input) {
|
|
2357
|
+
const ai = makeClient(input.apiKey);
|
|
2358
|
+
const job = await withRetry(
|
|
2359
|
+
() => ai.batches.create({
|
|
2360
|
+
model: input.model,
|
|
2361
|
+
src: input.requests,
|
|
2362
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2363
|
+
})
|
|
2364
|
+
);
|
|
2365
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2366
|
+
return job;
|
|
2367
|
+
}
|
|
2368
|
+
async function getGeminiBatchJob(input) {
|
|
2369
|
+
const ai = makeClient(input.apiKey);
|
|
2370
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2125
2373
|
// src/translate/gemini-client.ts
|
|
2126
2374
|
init_esm_shims();
|
|
2127
2375
|
|
|
@@ -2146,16 +2394,55 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2146
2394
|
if (alias) return alias[0];
|
|
2147
2395
|
return name;
|
|
2148
2396
|
}
|
|
2397
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2398
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2399
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: ThinkingLevel.LOW };
|
|
2400
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2401
|
+
return void 0;
|
|
2402
|
+
}
|
|
2149
2403
|
|
|
2150
2404
|
// src/translate/gemini-client.ts
|
|
2151
2405
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
2152
2406
|
function extractJson(text) {
|
|
2153
|
-
const
|
|
2407
|
+
const trimmed = text.trim();
|
|
2408
|
+
const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
|
2154
2409
|
if (fenced?.[1]) return fenced[1].trim();
|
|
2155
|
-
const start =
|
|
2156
|
-
const end =
|
|
2157
|
-
if (start >= 0 && end > start) return
|
|
2158
|
-
return
|
|
2410
|
+
const start = trimmed.indexOf("{");
|
|
2411
|
+
const end = trimmed.lastIndexOf("}");
|
|
2412
|
+
if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
|
|
2413
|
+
return trimmed;
|
|
2414
|
+
}
|
|
2415
|
+
function parseGeminiResponse(text) {
|
|
2416
|
+
let parsed;
|
|
2417
|
+
try {
|
|
2418
|
+
parsed = JSON.parse(text.trim());
|
|
2419
|
+
} catch {
|
|
2420
|
+
parsed = JSON.parse(extractJson(text));
|
|
2421
|
+
}
|
|
2422
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2423
|
+
parsed.frontmatter = {};
|
|
2424
|
+
}
|
|
2425
|
+
return parsed;
|
|
2426
|
+
}
|
|
2427
|
+
function buildGeminiRequestConfig(input) {
|
|
2428
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2429
|
+
return {
|
|
2430
|
+
responseMimeType: "application/json",
|
|
2431
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2432
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2433
|
+
};
|
|
2434
|
+
}
|
|
2435
|
+
function usageFromResponse(response) {
|
|
2436
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2437
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2438
|
+
return {
|
|
2439
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2440
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2441
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2442
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2443
|
+
thoughtsTokens,
|
|
2444
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2445
|
+
};
|
|
2159
2446
|
}
|
|
2160
2447
|
async function translatePageWithGemini(input) {
|
|
2161
2448
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2167,28 +2454,28 @@ async function translatePageWithGemini(input) {
|
|
|
2167
2454
|
);
|
|
2168
2455
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2169
2456
|
const ai = new GoogleGenAI({ apiKey });
|
|
2170
|
-
const response = await
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2457
|
+
const response = await withRetry(
|
|
2458
|
+
() => ai.models.generateContent({
|
|
2459
|
+
model: apiModel,
|
|
2460
|
+
contents: input.prompt,
|
|
2461
|
+
config: buildGeminiRequestConfig({
|
|
2462
|
+
apiModelId: apiModel,
|
|
2463
|
+
responseSchema: input.responseSchema
|
|
2464
|
+
})
|
|
2465
|
+
})
|
|
2466
|
+
);
|
|
2178
2467
|
const raw = response.text ?? "";
|
|
2179
|
-
const parsed =
|
|
2468
|
+
const parsed = parseGeminiResponse(raw);
|
|
2180
2469
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2181
2470
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2182
2471
|
}
|
|
2183
|
-
|
|
2184
|
-
const usage = {
|
|
2185
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2186
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2187
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2188
|
-
};
|
|
2189
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2472
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2190
2473
|
}
|
|
2191
2474
|
|
|
2475
|
+
// src/translate/translate-core.ts
|
|
2476
|
+
init_esm_shims();
|
|
2477
|
+
init_sqlite();
|
|
2478
|
+
|
|
2192
2479
|
// src/translate/gemini-pricing.ts
|
|
2193
2480
|
init_esm_shims();
|
|
2194
2481
|
var CONTEXT_TIER_TOKENS = 2e5;
|
|
@@ -2207,12 +2494,14 @@ function resolveModelPricing(model) {
|
|
|
2207
2494
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2208
2495
|
return match?.[1];
|
|
2209
2496
|
}
|
|
2210
|
-
|
|
2497
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2498
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2211
2499
|
const pricing = resolveModelPricing(model);
|
|
2212
2500
|
if (!pricing) return void 0;
|
|
2213
2501
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2214
2502
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2215
|
-
|
|
2503
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2504
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2216
2505
|
}
|
|
2217
2506
|
function formatTokenCount(tokens) {
|
|
2218
2507
|
if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
|
|
@@ -2240,6 +2529,7 @@ var LOCALE_NAMES = {
|
|
|
2240
2529
|
"zh-CN": "Simplified Chinese",
|
|
2241
2530
|
"zh-TW": "Traditional Chinese",
|
|
2242
2531
|
pt: "Portuguese",
|
|
2532
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2243
2533
|
ja: "Japanese",
|
|
2244
2534
|
ru: "Russian",
|
|
2245
2535
|
it: "Italian",
|
|
@@ -2247,29 +2537,29 @@ var LOCALE_NAMES = {
|
|
|
2247
2537
|
};
|
|
2248
2538
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2249
2539
|
return [
|
|
2250
|
-
`Localize the content
|
|
2540
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2251
2541
|
"Do not translate word-for-word.",
|
|
2252
2542
|
"Preserve the source tone and brand voice.",
|
|
2253
2543
|
"Write as if a native speaker authored it for the target market."
|
|
2254
2544
|
].join(" ");
|
|
2255
2545
|
}
|
|
2546
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2547
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2548
|
+
if (!hasFrontmatter) {
|
|
2549
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2550
|
+
}
|
|
2551
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2552
|
+
}
|
|
2256
2553
|
function buildPageTranslationPrompt(input) {
|
|
2257
2554
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2258
|
-
const
|
|
2259
|
-
const
|
|
2260
|
-
|
|
2555
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2556
|
+
const prefix = [
|
|
2557
|
+
TASK_FRAMING,
|
|
2261
2558
|
"",
|
|
2262
2559
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2263
2560
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2264
2561
|
"## Rules",
|
|
2265
2562
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2266
|
-
...input.slugStrategy === "localized" ? [
|
|
2267
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2268
|
-
] : [],
|
|
2269
|
-
"",
|
|
2270
|
-
"## Output format",
|
|
2271
|
-
"Return ONLY valid JSON with keys:",
|
|
2272
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2273
2563
|
"",
|
|
2274
2564
|
"## EN translatable frontmatter (JSON)",
|
|
2275
2565
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2277,7 +2567,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2277
2567
|
"## EN body (MDX)",
|
|
2278
2568
|
input.enBody
|
|
2279
2569
|
];
|
|
2280
|
-
|
|
2570
|
+
const suffix = [
|
|
2571
|
+
"",
|
|
2572
|
+
"## Target language",
|
|
2573
|
+
localizationPrompt,
|
|
2574
|
+
...input.slugStrategy === "localized" ? [
|
|
2575
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2576
|
+
] : [],
|
|
2577
|
+
"",
|
|
2578
|
+
"## Output format",
|
|
2579
|
+
"Return ONLY valid JSON with keys:",
|
|
2580
|
+
buildOutputFormatLine(
|
|
2581
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2582
|
+
input.slugStrategy
|
|
2583
|
+
)
|
|
2584
|
+
];
|
|
2585
|
+
return [...prefix, ...suffix].join("\n");
|
|
2281
2586
|
}
|
|
2282
2587
|
|
|
2283
2588
|
// src/translate/response-schema.ts
|
|
@@ -2328,9 +2633,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2328
2633
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2329
2634
|
try {
|
|
2330
2635
|
const translatable = buildTranslatableSubschema(schema);
|
|
2331
|
-
if (!translatable) return null;
|
|
2332
2636
|
const responseShape = {
|
|
2333
|
-
frontmatter: translatable,
|
|
2637
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2334
2638
|
body: z.string()
|
|
2335
2639
|
};
|
|
2336
2640
|
if (slugStrategy === "localized") {
|
|
@@ -2411,7 +2715,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2411
2715
|
return { ok: true, frontmatter };
|
|
2412
2716
|
}
|
|
2413
2717
|
|
|
2414
|
-
// src/translate/
|
|
2718
|
+
// src/translate/translate-core.ts
|
|
2719
|
+
function translationItemKey(item) {
|
|
2720
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2721
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2722
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2723
|
+
}
|
|
2415
2724
|
function summarizeResults(results, durationMs) {
|
|
2416
2725
|
return results.reduce(
|
|
2417
2726
|
(totals, result) => {
|
|
@@ -2447,17 +2756,6 @@ function formatTranslateError(error) {
|
|
|
2447
2756
|
}
|
|
2448
2757
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2449
2758
|
}
|
|
2450
|
-
async function runPool(items, concurrency, worker) {
|
|
2451
|
-
if (items.length === 0) return;
|
|
2452
|
-
let nextIndex = 0;
|
|
2453
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2454
|
-
while (nextIndex < items.length) {
|
|
2455
|
-
const index = nextIndex++;
|
|
2456
|
-
await worker(items[index], index);
|
|
2457
|
-
}
|
|
2458
|
-
});
|
|
2459
|
-
await Promise.all(runners);
|
|
2460
|
-
}
|
|
2461
2759
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2462
2760
|
const fm = enDoc.frontmatter;
|
|
2463
2761
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2466,73 +2764,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2466
2764
|
}
|
|
2467
2765
|
return enSlug;
|
|
2468
2766
|
}
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
const base = {
|
|
2767
|
+
function baseForItem(item) {
|
|
2768
|
+
return {
|
|
2472
2769
|
contentType: item.contentType,
|
|
2473
2770
|
enSlug: item.enSlug,
|
|
2474
2771
|
locale: item.locale
|
|
2475
2772
|
};
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2773
|
+
}
|
|
2774
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
2775
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
2776
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
2777
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
2778
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
2779
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
2780
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
2781
|
+
const db = openStore(config, "readonly");
|
|
2782
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
2783
|
+
db.close();
|
|
2784
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
2785
|
+
return {
|
|
2786
|
+
status: "done",
|
|
2787
|
+
result: {
|
|
2788
|
+
...baseForItem(item),
|
|
2489
2789
|
skipped: true,
|
|
2490
2790
|
reason: "fresh",
|
|
2491
2791
|
durationMs: Date.now() - startedAt
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2792
|
+
}
|
|
2793
|
+
};
|
|
2794
|
+
}
|
|
2795
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
2796
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
2797
|
+
const prompt = buildPageTranslationPrompt({
|
|
2798
|
+
resolved: resolvedTranslate,
|
|
2799
|
+
targetLocale: item.locale,
|
|
2800
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
2801
|
+
translatableFrontmatter: payload.frontmatter,
|
|
2802
|
+
enBody: payload.body,
|
|
2803
|
+
slugStrategy: type.slugStrategy
|
|
2804
|
+
});
|
|
2805
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
2806
|
+
return {
|
|
2807
|
+
status: "ready",
|
|
2808
|
+
prepared: {
|
|
2809
|
+
item,
|
|
2810
|
+
type,
|
|
2811
|
+
enDoc,
|
|
2812
|
+
payload,
|
|
2813
|
+
currentEnHash,
|
|
2814
|
+
existingSlug: existing?.slug,
|
|
2515
2815
|
model,
|
|
2816
|
+
prompt,
|
|
2516
2817
|
responseSchema: responseSchema ?? void 0
|
|
2517
|
-
}
|
|
2518
|
-
|
|
2818
|
+
}
|
|
2819
|
+
};
|
|
2820
|
+
}
|
|
2821
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
2822
|
+
const { item, type, enDoc, payload } = prepared;
|
|
2823
|
+
const base = baseForItem(item);
|
|
2824
|
+
try {
|
|
2825
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2519
2826
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2520
2827
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2521
2828
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2522
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
2829
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2523
2830
|
if (!validated.ok) {
|
|
2524
2831
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2525
2832
|
}
|
|
2526
2833
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2527
|
-
|
|
2834
|
+
output.parsed.body
|
|
2528
2835
|
);
|
|
2529
2836
|
const writeDb = openStore(config, "readwrite");
|
|
2530
|
-
const snapshotId = recordEnSnapshot(
|
|
2837
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2531
2838
|
config,
|
|
2532
2839
|
{
|
|
2533
2840
|
contentType: type.id,
|
|
2534
2841
|
enSlug: item.enSlug,
|
|
2535
|
-
enHash: currentEnHash,
|
|
2842
|
+
enHash: prepared.currentEnHash,
|
|
2536
2843
|
frontmatter: payload.frontmatter,
|
|
2537
2844
|
body: payload.body
|
|
2538
2845
|
},
|
|
@@ -2545,24 +2852,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2545
2852
|
slug,
|
|
2546
2853
|
frontmatter: validated.frontmatter,
|
|
2547
2854
|
body: translatedBody,
|
|
2548
|
-
enHash: currentEnHash,
|
|
2855
|
+
enHash: prepared.currentEnHash,
|
|
2549
2856
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2550
|
-
model:
|
|
2857
|
+
model: output.model,
|
|
2551
2858
|
snapshotId
|
|
2552
2859
|
});
|
|
2553
2860
|
writeDb.close();
|
|
2554
2861
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2555
|
-
normalizeGeminiDisplayName(
|
|
2556
|
-
|
|
2557
|
-
|
|
2862
|
+
normalizeGeminiDisplayName(output.model),
|
|
2863
|
+
output.usage.inputTokens,
|
|
2864
|
+
output.usage.outputTokens,
|
|
2865
|
+
options.costMode
|
|
2558
2866
|
);
|
|
2559
2867
|
return {
|
|
2560
2868
|
...base,
|
|
2561
2869
|
skipped: false,
|
|
2562
|
-
model:
|
|
2563
|
-
usage:
|
|
2870
|
+
model: output.model,
|
|
2871
|
+
usage: output.usage,
|
|
2564
2872
|
estimatedCostUsd,
|
|
2565
|
-
durationMs: Date.now() - startedAt,
|
|
2873
|
+
durationMs: Date.now() - options.startedAt,
|
|
2566
2874
|
slugAdjusted,
|
|
2567
2875
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2568
2876
|
};
|
|
@@ -2572,33 +2880,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2572
2880
|
skipped: false,
|
|
2573
2881
|
failed: true,
|
|
2574
2882
|
error: formatTranslateError(error),
|
|
2883
|
+
durationMs: Date.now() - options.startedAt
|
|
2884
|
+
};
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
function displayModelFor(prepared) {
|
|
2888
|
+
return normalizeGeminiDisplayName(
|
|
2889
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
2890
|
+
);
|
|
2891
|
+
}
|
|
2892
|
+
|
|
2893
|
+
// src/translate/batch-worklist.ts
|
|
2894
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
2895
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
2896
|
+
var INITIAL_POLL_MS = 5e3;
|
|
2897
|
+
var MAX_POLL_MS = 3e4;
|
|
2898
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
2899
|
+
function sleep(ms) {
|
|
2900
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2901
|
+
}
|
|
2902
|
+
function planBatchJobs(entries) {
|
|
2903
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
2904
|
+
for (const entry of entries) {
|
|
2905
|
+
const group = byModel.get(entry.apiModel);
|
|
2906
|
+
if (group) group.push(entry);
|
|
2907
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
2908
|
+
}
|
|
2909
|
+
const plans = [];
|
|
2910
|
+
for (const [apiModel, group] of byModel) {
|
|
2911
|
+
let current = [];
|
|
2912
|
+
let currentBytes = 0;
|
|
2913
|
+
for (const entry of group) {
|
|
2914
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
2915
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
2916
|
+
plans.push({ apiModel, entries: current });
|
|
2917
|
+
current = [];
|
|
2918
|
+
currentBytes = 0;
|
|
2919
|
+
}
|
|
2920
|
+
current.push(entry);
|
|
2921
|
+
currentBytes += size;
|
|
2922
|
+
}
|
|
2923
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
2924
|
+
}
|
|
2925
|
+
return plans;
|
|
2926
|
+
}
|
|
2927
|
+
function readPendingBatchWork(config) {
|
|
2928
|
+
const db = openStore(config, "readwrite");
|
|
2929
|
+
const jobs = listPendingBatchJobs(db);
|
|
2930
|
+
const pendingItems = listPendingBatchItems(db);
|
|
2931
|
+
db.close();
|
|
2932
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
2933
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
2934
|
+
}
|
|
2935
|
+
async function submitBatchJobPlan(config, input) {
|
|
2936
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
2937
|
+
const job = await createGeminiBatchJob({
|
|
2938
|
+
model: plan.apiModel,
|
|
2939
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
2940
|
+
contents: prepared.prompt,
|
|
2941
|
+
config: buildGeminiRequestConfig({
|
|
2942
|
+
apiModelId: plan.apiModel,
|
|
2943
|
+
responseSchema: prepared.responseSchema
|
|
2944
|
+
})
|
|
2945
|
+
})),
|
|
2946
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
2947
|
+
});
|
|
2948
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2949
|
+
const db = openStore(config, "readwrite");
|
|
2950
|
+
const jobId = insertBatchJob(db, {
|
|
2951
|
+
jobName: job.name,
|
|
2952
|
+
model: plan.apiModel,
|
|
2953
|
+
displayModel,
|
|
2954
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2955
|
+
createdAt
|
|
2956
|
+
});
|
|
2957
|
+
insertBatchItems(
|
|
2958
|
+
db,
|
|
2959
|
+
jobId,
|
|
2960
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
2961
|
+
requestIndex,
|
|
2962
|
+
contentType: prepared.item.contentType,
|
|
2963
|
+
enSlug: prepared.item.enSlug,
|
|
2964
|
+
locale: prepared.item.locale,
|
|
2965
|
+
enHash: prepared.currentEnHash,
|
|
2966
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
2967
|
+
// on the EN files still matching (or existing) on disk.
|
|
2968
|
+
snapshotId: recordEnSnapshot(
|
|
2969
|
+
config,
|
|
2970
|
+
{
|
|
2971
|
+
contentType: prepared.item.contentType,
|
|
2972
|
+
enSlug: prepared.item.enSlug,
|
|
2973
|
+
enHash: prepared.currentEnHash,
|
|
2974
|
+
frontmatter: prepared.payload.frontmatter,
|
|
2975
|
+
body: prepared.payload.body
|
|
2976
|
+
},
|
|
2977
|
+
db
|
|
2978
|
+
)
|
|
2979
|
+
}))
|
|
2980
|
+
);
|
|
2981
|
+
const row = {
|
|
2982
|
+
id: jobId,
|
|
2983
|
+
job_name: job.name,
|
|
2984
|
+
model: plan.apiModel,
|
|
2985
|
+
display_model: displayModel,
|
|
2986
|
+
created_at: createdAt,
|
|
2987
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
2988
|
+
completed_at: null
|
|
2989
|
+
};
|
|
2990
|
+
db.close();
|
|
2991
|
+
input.onProgress?.({
|
|
2992
|
+
type: "batch-submitted",
|
|
2993
|
+
name: job.name,
|
|
2994
|
+
count: plan.entries.length,
|
|
2995
|
+
model: displayModel,
|
|
2996
|
+
jobIndex,
|
|
2997
|
+
jobCount,
|
|
2998
|
+
createdAt
|
|
2999
|
+
});
|
|
3000
|
+
return row;
|
|
3001
|
+
}
|
|
3002
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3003
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3004
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3005
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3006
|
+
if (!enDoc) {
|
|
3007
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3008
|
+
if (!snapshot) {
|
|
3009
|
+
throw new Error(
|
|
3010
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3011
|
+
);
|
|
3012
|
+
}
|
|
3013
|
+
enDoc = {
|
|
3014
|
+
slug: itemRow.en_slug,
|
|
3015
|
+
enSlug: itemRow.en_slug,
|
|
3016
|
+
locale: config.defaultLocale,
|
|
3017
|
+
noindex: false,
|
|
3018
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3019
|
+
content: snapshot.body
|
|
3020
|
+
};
|
|
3021
|
+
}
|
|
3022
|
+
const item = {
|
|
3023
|
+
contentType: itemRow.content_type,
|
|
3024
|
+
enSlug: itemRow.en_slug,
|
|
3025
|
+
locale: itemRow.locale,
|
|
3026
|
+
reason: "missing",
|
|
3027
|
+
currentEnHash: itemRow.en_hash
|
|
3028
|
+
};
|
|
3029
|
+
return {
|
|
3030
|
+
item,
|
|
3031
|
+
type,
|
|
3032
|
+
enDoc,
|
|
3033
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3034
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3035
|
+
currentEnHash: itemRow.en_hash,
|
|
3036
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3037
|
+
model: jobRow.display_model,
|
|
3038
|
+
prompt: "",
|
|
3039
|
+
responseSchema: void 0
|
|
3040
|
+
};
|
|
3041
|
+
}
|
|
3042
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3043
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3044
|
+
const db = openStore(config, "readwrite");
|
|
3045
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3046
|
+
db.close();
|
|
3047
|
+
return null;
|
|
3048
|
+
}
|
|
3049
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3050
|
+
const results = [];
|
|
3051
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3052
|
+
const result = {
|
|
3053
|
+
contentType: itemRow.content_type,
|
|
3054
|
+
enSlug: itemRow.en_slug,
|
|
3055
|
+
locale: itemRow.locale,
|
|
3056
|
+
skipped: false,
|
|
3057
|
+
failed: true,
|
|
3058
|
+
error: formatTranslateError(new Error(message)),
|
|
2575
3059
|
durationMs: Date.now() - startedAt
|
|
2576
3060
|
};
|
|
3061
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3062
|
+
results.push(result);
|
|
3063
|
+
onResult?.(result);
|
|
3064
|
+
};
|
|
3065
|
+
if (isSuccessfulBatchState(state)) {
|
|
3066
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3067
|
+
for (const itemRow of pendingItems) {
|
|
3068
|
+
const startedAt = Date.now();
|
|
3069
|
+
const inlined = responses[itemRow.request_index];
|
|
3070
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3071
|
+
failItem(
|
|
3072
|
+
itemRow,
|
|
3073
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3074
|
+
startedAt
|
|
3075
|
+
);
|
|
3076
|
+
continue;
|
|
3077
|
+
}
|
|
3078
|
+
let result;
|
|
3079
|
+
try {
|
|
3080
|
+
const response = inlined.response;
|
|
3081
|
+
const raw = textFromBatchResponse(response);
|
|
3082
|
+
const parsed = parseGeminiResponse(raw);
|
|
3083
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3084
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3085
|
+
}
|
|
3086
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3087
|
+
result = finalizeTranslation(
|
|
3088
|
+
config,
|
|
3089
|
+
prepared,
|
|
3090
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3091
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3092
|
+
);
|
|
3093
|
+
} catch (error) {
|
|
3094
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3095
|
+
continue;
|
|
3096
|
+
}
|
|
3097
|
+
updateBatchItemStatus(
|
|
3098
|
+
db,
|
|
3099
|
+
jobRow.id,
|
|
3100
|
+
itemRow.request_index,
|
|
3101
|
+
result.failed ? "failed" : "done",
|
|
3102
|
+
result.error
|
|
3103
|
+
);
|
|
3104
|
+
results.push(result);
|
|
3105
|
+
onResult?.(result);
|
|
3106
|
+
}
|
|
3107
|
+
} else {
|
|
3108
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3109
|
+
const startedAt = Date.now();
|
|
3110
|
+
for (const itemRow of pendingItems) {
|
|
3111
|
+
failItem(itemRow, message, startedAt);
|
|
3112
|
+
}
|
|
2577
3113
|
}
|
|
3114
|
+
db.close();
|
|
3115
|
+
return results;
|
|
3116
|
+
}
|
|
3117
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3118
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3119
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3120
|
+
const loopStartedAt = Date.now();
|
|
3121
|
+
let delay = INITIAL_POLL_MS;
|
|
3122
|
+
let firstRound = true;
|
|
3123
|
+
const elapsedFor = (row) => {
|
|
3124
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3125
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3126
|
+
};
|
|
3127
|
+
while (active.length > 0) {
|
|
3128
|
+
if (!firstRound) {
|
|
3129
|
+
await sleep(delay);
|
|
3130
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3131
|
+
}
|
|
3132
|
+
firstRound = false;
|
|
3133
|
+
for (const tracked of [...active]) {
|
|
3134
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3135
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3136
|
+
options.onProgress?.({
|
|
3137
|
+
type: "batch-polling",
|
|
3138
|
+
name: tracked.row.job_name,
|
|
3139
|
+
state,
|
|
3140
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3141
|
+
jobIndex: tracked.jobIndex,
|
|
3142
|
+
jobCount
|
|
3143
|
+
});
|
|
3144
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3145
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3146
|
+
const results = ingestBatchJob(
|
|
3147
|
+
config,
|
|
3148
|
+
tracked.row,
|
|
3149
|
+
batchJob,
|
|
3150
|
+
options.onResult
|
|
3151
|
+
);
|
|
3152
|
+
if (results === null) {
|
|
3153
|
+
options.onProgress?.({
|
|
3154
|
+
type: "batch-done",
|
|
3155
|
+
name: tracked.row.job_name,
|
|
3156
|
+
state,
|
|
3157
|
+
model: tracked.row.display_model,
|
|
3158
|
+
count: 0,
|
|
3159
|
+
jobIndex: tracked.jobIndex,
|
|
3160
|
+
jobCount,
|
|
3161
|
+
translated: 0,
|
|
3162
|
+
failed: 0,
|
|
3163
|
+
inputTokens: 0,
|
|
3164
|
+
outputTokens: 0,
|
|
3165
|
+
estimatedCostUsd: 0,
|
|
3166
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3167
|
+
alreadyIngested: true
|
|
3168
|
+
});
|
|
3169
|
+
continue;
|
|
3170
|
+
}
|
|
3171
|
+
options.onProgress?.({
|
|
3172
|
+
type: "batch-done",
|
|
3173
|
+
name: tracked.row.job_name,
|
|
3174
|
+
state,
|
|
3175
|
+
model: tracked.row.display_model,
|
|
3176
|
+
count: results.length,
|
|
3177
|
+
jobIndex: tracked.jobIndex,
|
|
3178
|
+
jobCount,
|
|
3179
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3180
|
+
failed: results.filter((r) => r.failed).length,
|
|
3181
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3182
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3183
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3184
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3185
|
+
});
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
}
|
|
3190
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3191
|
+
return {
|
|
3192
|
+
...baseForItem(item),
|
|
3193
|
+
skipped: false,
|
|
3194
|
+
failed: true,
|
|
3195
|
+
error: formatTranslateError(error),
|
|
3196
|
+
durationMs: Date.now() - startedAt
|
|
3197
|
+
};
|
|
3198
|
+
}
|
|
3199
|
+
|
|
3200
|
+
// src/translate/page-translator.ts
|
|
3201
|
+
async function runPool(items, concurrency, worker) {
|
|
3202
|
+
if (items.length === 0) return;
|
|
3203
|
+
let nextIndex = 0;
|
|
3204
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3205
|
+
while (nextIndex < items.length) {
|
|
3206
|
+
const index = nextIndex++;
|
|
3207
|
+
await worker(items[index], index);
|
|
3208
|
+
}
|
|
3209
|
+
});
|
|
3210
|
+
await Promise.all(runners);
|
|
2578
3211
|
}
|
|
2579
3212
|
function labelForItem(item) {
|
|
2580
3213
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2581
3214
|
}
|
|
3215
|
+
async function translatePage(config, item, options = {}) {
|
|
3216
|
+
const startedAt = Date.now();
|
|
3217
|
+
let outcome;
|
|
3218
|
+
try {
|
|
3219
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3220
|
+
} catch (error) {
|
|
3221
|
+
return failedResultForItem(item, error, startedAt);
|
|
3222
|
+
}
|
|
3223
|
+
if (outcome.status === "done") return outcome.result;
|
|
3224
|
+
const prepared = outcome.prepared;
|
|
3225
|
+
if (options.dryRun) {
|
|
3226
|
+
return {
|
|
3227
|
+
...baseForItem(item),
|
|
3228
|
+
skipped: false,
|
|
3229
|
+
model: prepared.model,
|
|
3230
|
+
durationMs: Date.now() - startedAt
|
|
3231
|
+
};
|
|
3232
|
+
}
|
|
3233
|
+
try {
|
|
3234
|
+
const result = await translatePageWithGemini({
|
|
3235
|
+
prompt: prepared.prompt,
|
|
3236
|
+
model: prepared.model,
|
|
3237
|
+
responseSchema: prepared.responseSchema
|
|
3238
|
+
});
|
|
3239
|
+
return finalizeTranslation(
|
|
3240
|
+
config,
|
|
3241
|
+
prepared,
|
|
3242
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3243
|
+
{ costMode: "interactive", startedAt }
|
|
3244
|
+
);
|
|
3245
|
+
} catch (error) {
|
|
3246
|
+
return failedResultForItem(item, error, startedAt);
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3249
|
+
function createResultCollector(items, onProgress) {
|
|
3250
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3251
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3252
|
+
const slots = new Array(items.length);
|
|
3253
|
+
const extras = [];
|
|
3254
|
+
return {
|
|
3255
|
+
emit(result) {
|
|
3256
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3257
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3258
|
+
else extras.push(result);
|
|
3259
|
+
onProgress?.({ type: "item-done", result });
|
|
3260
|
+
},
|
|
3261
|
+
assemble() {
|
|
3262
|
+
return [
|
|
3263
|
+
...slots.filter((result) => result !== void 0),
|
|
3264
|
+
...extras
|
|
3265
|
+
];
|
|
3266
|
+
}
|
|
3267
|
+
};
|
|
3268
|
+
}
|
|
2582
3269
|
async function translateWorklist(config, items, options = {}) {
|
|
3270
|
+
const mode = options.mode ?? "batch";
|
|
2583
3271
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2584
3272
|
const startedAt = Date.now();
|
|
2585
|
-
|
|
2586
|
-
|
|
3273
|
+
if (options.dryRun) {
|
|
3274
|
+
options.onProgress?.({
|
|
3275
|
+
type: "start",
|
|
3276
|
+
total: items.length,
|
|
3277
|
+
concurrency,
|
|
3278
|
+
dryRun: true,
|
|
3279
|
+
model: options.model,
|
|
3280
|
+
mode
|
|
3281
|
+
});
|
|
3282
|
+
const results2 = items.map((item) => {
|
|
3283
|
+
const itemStartedAt = Date.now();
|
|
3284
|
+
try {
|
|
3285
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3286
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3287
|
+
...baseForItem(item),
|
|
3288
|
+
skipped: false,
|
|
3289
|
+
model: outcome.prepared.model,
|
|
3290
|
+
durationMs: Date.now() - itemStartedAt
|
|
3291
|
+
};
|
|
3292
|
+
} catch (error) {
|
|
3293
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3294
|
+
}
|
|
3295
|
+
});
|
|
3296
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3297
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3298
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3299
|
+
return results2;
|
|
3300
|
+
}
|
|
3301
|
+
const pending = readPendingBatchWork(config);
|
|
3302
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3303
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3304
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3305
|
+
).length;
|
|
2587
3306
|
options.onProgress?.({
|
|
2588
3307
|
type: "start",
|
|
2589
|
-
total: items.length,
|
|
3308
|
+
total: items.length + resumedExtraCount,
|
|
2590
3309
|
concurrency,
|
|
2591
|
-
dryRun:
|
|
2592
|
-
model: options.model
|
|
3310
|
+
dryRun: false,
|
|
3311
|
+
model: options.model,
|
|
3312
|
+
mode
|
|
3313
|
+
});
|
|
3314
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3315
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3316
|
+
if (mode === "direct") {
|
|
3317
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3318
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3319
|
+
const active = /* @__PURE__ */ new Set();
|
|
3320
|
+
await Promise.all([
|
|
3321
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3322
|
+
const label = labelForItem(item);
|
|
3323
|
+
active.add(label);
|
|
3324
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3325
|
+
const result = await translatePage(config, item, options);
|
|
3326
|
+
active.delete(label);
|
|
3327
|
+
collector.emit(result);
|
|
3328
|
+
}),
|
|
3329
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3330
|
+
onProgress: options.onProgress,
|
|
3331
|
+
onResult: collector.emit
|
|
3332
|
+
}) : Promise.resolve()
|
|
3333
|
+
]);
|
|
3334
|
+
} else {
|
|
3335
|
+
const entries = [];
|
|
3336
|
+
for (const item of workItems) {
|
|
3337
|
+
const itemStartedAt = Date.now();
|
|
3338
|
+
try {
|
|
3339
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3340
|
+
if (outcome.status === "done") {
|
|
3341
|
+
collector.emit(outcome.result);
|
|
3342
|
+
} else {
|
|
3343
|
+
entries.push({
|
|
3344
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3345
|
+
prompt: outcome.prepared.prompt,
|
|
3346
|
+
prepared: outcome.prepared
|
|
3347
|
+
});
|
|
3348
|
+
}
|
|
3349
|
+
} catch (error) {
|
|
3350
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3351
|
+
}
|
|
3352
|
+
}
|
|
3353
|
+
const plans = planBatchJobs(entries);
|
|
3354
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3355
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3356
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3357
|
+
const submitted = await Promise.all(
|
|
3358
|
+
plans.map(async (plan, planIndex) => {
|
|
3359
|
+
try {
|
|
3360
|
+
return await submitBatchJobPlan(config, {
|
|
3361
|
+
plan,
|
|
3362
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3363
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3364
|
+
jobCount,
|
|
3365
|
+
onProgress: options.onProgress
|
|
3366
|
+
});
|
|
3367
|
+
} catch (error) {
|
|
3368
|
+
for (const entry of plan.entries) {
|
|
3369
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3370
|
+
}
|
|
3371
|
+
return void 0;
|
|
3372
|
+
}
|
|
3373
|
+
})
|
|
3374
|
+
);
|
|
3375
|
+
const jobsToPoll = [
|
|
3376
|
+
...pending.jobs,
|
|
3377
|
+
...submitted.filter((row) => row !== void 0)
|
|
3378
|
+
];
|
|
3379
|
+
if (jobsToPoll.length > 0) {
|
|
3380
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3381
|
+
jobCount,
|
|
3382
|
+
onProgress: options.onProgress,
|
|
3383
|
+
onResult: collector.emit
|
|
3384
|
+
});
|
|
3385
|
+
}
|
|
3386
|
+
}
|
|
3387
|
+
const results = collector.assemble();
|
|
3388
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3389
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3390
|
+
return results;
|
|
3391
|
+
}
|
|
3392
|
+
function countPendingItems(config, jobs) {
|
|
3393
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3394
|
+
const db = openStore(config, "readonly");
|
|
3395
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3396
|
+
for (const job of jobs) {
|
|
3397
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3398
|
+
}
|
|
3399
|
+
db.close();
|
|
3400
|
+
return counts;
|
|
3401
|
+
}
|
|
3402
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3403
|
+
jobs.forEach((job, jobIndex) => {
|
|
3404
|
+
onProgress?.({
|
|
3405
|
+
type: "batch-submitted",
|
|
3406
|
+
name: job.job_name,
|
|
3407
|
+
count: counts.get(job.id) ?? 0,
|
|
3408
|
+
model: job.display_model,
|
|
3409
|
+
jobIndex,
|
|
3410
|
+
jobCount,
|
|
3411
|
+
resumed: true,
|
|
3412
|
+
createdAt: job.created_at
|
|
3413
|
+
});
|
|
3414
|
+
});
|
|
3415
|
+
}
|
|
3416
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3417
|
+
const startedAt = Date.now();
|
|
3418
|
+
const pending = readPendingBatchWork(config);
|
|
3419
|
+
if (pending.jobs.length === 0) return null;
|
|
3420
|
+
options.onProgress?.({
|
|
3421
|
+
type: "start",
|
|
3422
|
+
total: pending.pendingItems.length,
|
|
3423
|
+
concurrency: 1,
|
|
3424
|
+
dryRun: false,
|
|
3425
|
+
mode: "batch"
|
|
2593
3426
|
});
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
3427
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3428
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3429
|
+
const results = [];
|
|
3430
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3431
|
+
onProgress: options.onProgress,
|
|
3432
|
+
onResult: (result) => {
|
|
3433
|
+
results.push(result);
|
|
3434
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3435
|
+
}
|
|
2602
3436
|
});
|
|
2603
3437
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2604
3438
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3216,35 +4050,43 @@ async function promptContentType(config) {
|
|
|
3216
4050
|
})
|
|
3217
4051
|
);
|
|
3218
4052
|
}
|
|
4053
|
+
var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
|
|
4054
|
+
var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
|
|
4055
|
+
async function promptManualLocales(config) {
|
|
4056
|
+
const locales = nonDefaultLocales(config);
|
|
4057
|
+
if (locales.length <= 1) return {};
|
|
4058
|
+
const picked = await runPrompt(
|
|
4059
|
+
checkbox({
|
|
4060
|
+
message: "Locales to translate",
|
|
4061
|
+
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
4062
|
+
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
4063
|
+
})
|
|
4064
|
+
);
|
|
4065
|
+
return picked.length === locales.length ? {} : { locale: picked };
|
|
4066
|
+
}
|
|
3219
4067
|
async function promptLocalePreset(config) {
|
|
3220
4068
|
const presets = Object.entries(config.localePresets ?? {}).filter(
|
|
3221
4069
|
(entry) => Array.isArray(entry[1])
|
|
3222
4070
|
);
|
|
3223
4071
|
if (presets.length === 0) {
|
|
3224
|
-
|
|
3225
|
-
if (locales.length <= 1) return {};
|
|
3226
|
-
const picked = await runPrompt(
|
|
3227
|
-
checkbox({
|
|
3228
|
-
message: "Locales to translate",
|
|
3229
|
-
choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
|
|
3230
|
-
validate: (values) => values.length > 0 || "Pick at least one locale"
|
|
3231
|
-
})
|
|
3232
|
-
);
|
|
3233
|
-
return picked.length === locales.length ? {} : { locale: picked };
|
|
4072
|
+
return promptManualLocales(config);
|
|
3234
4073
|
}
|
|
3235
4074
|
const choice = await runPrompt(
|
|
3236
4075
|
select({
|
|
3237
4076
|
message: "Locale preset",
|
|
3238
4077
|
choices: [
|
|
3239
|
-
{ name: "All locales", value:
|
|
4078
|
+
{ name: "All locales", value: ALL_LOCALES },
|
|
3240
4079
|
...presets.map(([name, locales]) => ({
|
|
3241
4080
|
name: `${name} (${locales.join(", ")})`,
|
|
3242
4081
|
value: name
|
|
3243
|
-
}))
|
|
4082
|
+
})),
|
|
4083
|
+
{ name: "Choose locales manually\u2026", value: PICK_LOCALES }
|
|
3244
4084
|
]
|
|
3245
4085
|
})
|
|
3246
4086
|
);
|
|
3247
|
-
|
|
4087
|
+
if (choice === PICK_LOCALES) return promptManualLocales(config);
|
|
4088
|
+
if (choice === ALL_LOCALES) return {};
|
|
4089
|
+
return { preset: choice };
|
|
3248
4090
|
}
|
|
3249
4091
|
async function promptStrategy() {
|
|
3250
4092
|
return runPrompt(
|
|
@@ -3258,12 +4100,25 @@ async function promptStrategy() {
|
|
|
3258
4100
|
})
|
|
3259
4101
|
);
|
|
3260
4102
|
}
|
|
4103
|
+
async function promptMode() {
|
|
4104
|
+
return runPrompt(
|
|
4105
|
+
select({
|
|
4106
|
+
message: "Translation mode",
|
|
4107
|
+
choices: [
|
|
4108
|
+
{ name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
|
|
4109
|
+
{ name: "Direct \u2014 immediate results, full price", value: "direct" }
|
|
4110
|
+
],
|
|
4111
|
+
default: "batch"
|
|
4112
|
+
})
|
|
4113
|
+
);
|
|
4114
|
+
}
|
|
3261
4115
|
async function promptTranslateSelection(config, flags) {
|
|
3262
4116
|
const selection = {
|
|
3263
4117
|
contentType: flags.type,
|
|
3264
4118
|
preset: flags.preset,
|
|
3265
4119
|
locale: flags.locale,
|
|
3266
|
-
strategy: flags.strategy
|
|
4120
|
+
strategy: flags.strategy,
|
|
4121
|
+
mode: flags.mode
|
|
3267
4122
|
};
|
|
3268
4123
|
if (!isInteractive()) return selection;
|
|
3269
4124
|
if (!selection.contentType) {
|
|
@@ -3275,6 +4130,9 @@ async function promptTranslateSelection(config, flags) {
|
|
|
3275
4130
|
if (!selection.strategy) {
|
|
3276
4131
|
selection.strategy = await promptStrategy();
|
|
3277
4132
|
}
|
|
4133
|
+
if (!selection.mode) {
|
|
4134
|
+
selection.mode = await promptMode();
|
|
4135
|
+
}
|
|
3278
4136
|
return selection;
|
|
3279
4137
|
}
|
|
3280
4138
|
|
|
@@ -3296,6 +4154,34 @@ function progressBar(ratio, width = 28) {
|
|
|
3296
4154
|
function labelForResult(result) {
|
|
3297
4155
|
return `${result.contentType}/${result.enSlug}@${result.locale}`;
|
|
3298
4156
|
}
|
|
4157
|
+
function shortJobId(name) {
|
|
4158
|
+
const segments = name.split("/");
|
|
4159
|
+
return segments[segments.length - 1] || name;
|
|
4160
|
+
}
|
|
4161
|
+
function formatElapsed(elapsedMs) {
|
|
4162
|
+
return `${Math.round(elapsedMs / 1e3)}s`;
|
|
4163
|
+
}
|
|
4164
|
+
function batchDoneParts(event) {
|
|
4165
|
+
if (event.alreadyIngested) {
|
|
4166
|
+
return [
|
|
4167
|
+
`job ${shortJobId(event.name)}`,
|
|
4168
|
+
event.model ?? "?",
|
|
4169
|
+
"already ingested by another scribe run (results saved there)",
|
|
4170
|
+
formatElapsed(event.elapsedMs)
|
|
4171
|
+
].join(" \xB7 ");
|
|
4172
|
+
}
|
|
4173
|
+
const parts = [
|
|
4174
|
+
`job ${shortJobId(event.name)}`,
|
|
4175
|
+
event.model ?? "?",
|
|
4176
|
+
`${event.count} request${event.count === 1 ? "" : "s"}`,
|
|
4177
|
+
`${event.translated} translated`
|
|
4178
|
+
];
|
|
4179
|
+
if (event.failed > 0) parts.push(`${event.failed} failed`);
|
|
4180
|
+
parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
|
|
4181
|
+
parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
|
|
4182
|
+
parts.push(formatElapsed(event.elapsedMs));
|
|
4183
|
+
return parts.join(" \xB7 ");
|
|
4184
|
+
}
|
|
3299
4185
|
function statusForResult(result, dryRun) {
|
|
3300
4186
|
if (result.failed) return red("failed");
|
|
3301
4187
|
if (result.skipped) return dim("skipped");
|
|
@@ -3341,6 +4227,25 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3341
4227
|
if (!enabled) {
|
|
3342
4228
|
return {
|
|
3343
4229
|
onEvent(event) {
|
|
4230
|
+
if (event.type === "batch-submitted") {
|
|
4231
|
+
const verb = event.resumed ? "Resuming" : "Submitted";
|
|
4232
|
+
console.log(
|
|
4233
|
+
`${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
|
|
4234
|
+
);
|
|
4235
|
+
return;
|
|
4236
|
+
}
|
|
4237
|
+
if (event.type === "batch-polling") {
|
|
4238
|
+
console.log(
|
|
4239
|
+
`batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
|
|
4240
|
+
);
|
|
4241
|
+
return;
|
|
4242
|
+
}
|
|
4243
|
+
if (event.type === "batch-done") {
|
|
4244
|
+
const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
|
|
4245
|
+
if (event.failed > 0 && event.translated === 0) console.error(line);
|
|
4246
|
+
else console.log(line);
|
|
4247
|
+
return;
|
|
4248
|
+
}
|
|
3344
4249
|
if (event.type === "item-done") {
|
|
3345
4250
|
const label = labelForResult(event.result);
|
|
3346
4251
|
const status = statusForResult(event.result, dryRun);
|
|
@@ -3371,6 +4276,9 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3371
4276
|
let outputTokens = 0;
|
|
3372
4277
|
let estimatedCostUsd = 0;
|
|
3373
4278
|
let active = [];
|
|
4279
|
+
let mode;
|
|
4280
|
+
const batchJobs = /* @__PURE__ */ new Map();
|
|
4281
|
+
let batchElapsedMs = 0;
|
|
3374
4282
|
const recent = [];
|
|
3375
4283
|
let renderedLines = 0;
|
|
3376
4284
|
let cursorHidden = false;
|
|
@@ -3386,6 +4294,15 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3386
4294
|
cursorHidden = false;
|
|
3387
4295
|
}
|
|
3388
4296
|
}
|
|
4297
|
+
function printPersistent(lines) {
|
|
4298
|
+
if (renderedLines > 0) {
|
|
4299
|
+
process.stdout.write(`\x1B[${renderedLines}A`);
|
|
4300
|
+
}
|
|
4301
|
+
for (const line of lines) {
|
|
4302
|
+
process.stdout.write("\x1B[2K" + line + "\n");
|
|
4303
|
+
}
|
|
4304
|
+
renderedLines = 0;
|
|
4305
|
+
}
|
|
3389
4306
|
function render() {
|
|
3390
4307
|
hideCursor();
|
|
3391
4308
|
if (renderedLines > 0) {
|
|
@@ -3393,12 +4310,34 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3393
4310
|
}
|
|
3394
4311
|
const lines = [];
|
|
3395
4312
|
const title = dryRun ? "Dry run" : "Translating";
|
|
3396
|
-
|
|
4313
|
+
const isBatch = mode === "batch" || batchJobs.size > 0;
|
|
4314
|
+
lines.push(
|
|
4315
|
+
cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
|
|
4316
|
+
);
|
|
3397
4317
|
lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
|
|
3398
4318
|
lines.push(
|
|
3399
4319
|
dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
|
|
3400
4320
|
);
|
|
3401
|
-
if (
|
|
4321
|
+
if (batchJobs.size > 0 && done < total) {
|
|
4322
|
+
const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
|
|
4323
|
+
let running = 0;
|
|
4324
|
+
let finished = 0;
|
|
4325
|
+
let requests = 0;
|
|
4326
|
+
for (const job of batchJobs.values()) {
|
|
4327
|
+
if (terminal.test(job.state)) finished += 1;
|
|
4328
|
+
else running += 1;
|
|
4329
|
+
requests += job.count;
|
|
4330
|
+
}
|
|
4331
|
+
lines.push(
|
|
4332
|
+
dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
|
|
4333
|
+
);
|
|
4334
|
+
for (const [name, job] of batchJobs) {
|
|
4335
|
+
if (terminal.test(job.state)) continue;
|
|
4336
|
+
lines.push(
|
|
4337
|
+
dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
|
|
4338
|
+
);
|
|
4339
|
+
}
|
|
4340
|
+
} else if (active.length > 0) {
|
|
3402
4341
|
lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
|
|
3403
4342
|
} else if (done < total) {
|
|
3404
4343
|
lines.push(dim("Active ") + "starting\u2026");
|
|
@@ -3426,12 +4365,58 @@ function createTranslateProgressReporter(options = {}) {
|
|
|
3426
4365
|
total = event.total;
|
|
3427
4366
|
concurrency = event.concurrency;
|
|
3428
4367
|
model = event.model;
|
|
4368
|
+
mode = event.mode;
|
|
3429
4369
|
render();
|
|
3430
4370
|
break;
|
|
3431
4371
|
case "item-start":
|
|
3432
4372
|
active = event.active;
|
|
3433
4373
|
render();
|
|
3434
4374
|
break;
|
|
4375
|
+
case "batch-submitted": {
|
|
4376
|
+
const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
|
|
4377
|
+
const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
|
|
4378
|
+
batchJobs.set(event.name, {
|
|
4379
|
+
state: event.resumed ? "RESUMED" : "SUBMITTED",
|
|
4380
|
+
count: event.count,
|
|
4381
|
+
model: event.model,
|
|
4382
|
+
createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
|
|
4383
|
+
elapsedMs
|
|
4384
|
+
});
|
|
4385
|
+
const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
|
|
4386
|
+
const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
|
|
4387
|
+
printPersistent([
|
|
4388
|
+
dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
|
|
4389
|
+
]);
|
|
4390
|
+
render();
|
|
4391
|
+
break;
|
|
4392
|
+
}
|
|
4393
|
+
case "batch-polling": {
|
|
4394
|
+
const job = batchJobs.get(event.name);
|
|
4395
|
+
batchJobs.set(event.name, {
|
|
4396
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4397
|
+
count: job?.count ?? 0,
|
|
4398
|
+
model: job?.model,
|
|
4399
|
+
createdAtMs: job?.createdAtMs,
|
|
4400
|
+
elapsedMs: event.elapsedMs
|
|
4401
|
+
});
|
|
4402
|
+
batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
|
|
4403
|
+
render();
|
|
4404
|
+
break;
|
|
4405
|
+
}
|
|
4406
|
+
case "batch-done": {
|
|
4407
|
+
const job = batchJobs.get(event.name);
|
|
4408
|
+
batchJobs.set(event.name, {
|
|
4409
|
+
state: event.state.replace(/^JOB_STATE_/, ""),
|
|
4410
|
+
count: job?.count ?? event.count,
|
|
4411
|
+
model: job?.model ?? event.model,
|
|
4412
|
+
createdAtMs: job?.createdAtMs,
|
|
4413
|
+
elapsedMs: event.elapsedMs
|
|
4414
|
+
});
|
|
4415
|
+
const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
|
|
4416
|
+
printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
|
|
4417
|
+
render();
|
|
4418
|
+
break;
|
|
4419
|
+
}
|
|
3435
4420
|
case "item-done": {
|
|
3436
4421
|
done += 1;
|
|
3437
4422
|
active = active.filter((label) => label !== labelForResult(event.result));
|
|
@@ -3554,6 +4539,21 @@ function parseArgs(argv) {
|
|
|
3554
4539
|
i++;
|
|
3555
4540
|
continue;
|
|
3556
4541
|
}
|
|
4542
|
+
if (arg === "--batch") {
|
|
4543
|
+
options.batch = true;
|
|
4544
|
+
i++;
|
|
4545
|
+
continue;
|
|
4546
|
+
}
|
|
4547
|
+
if (arg === "--direct") {
|
|
4548
|
+
options.direct = true;
|
|
4549
|
+
i++;
|
|
4550
|
+
continue;
|
|
4551
|
+
}
|
|
4552
|
+
if (arg === "--resume") {
|
|
4553
|
+
options.resume = true;
|
|
4554
|
+
i++;
|
|
4555
|
+
continue;
|
|
4556
|
+
}
|
|
3557
4557
|
if (arg === "--strategy") {
|
|
3558
4558
|
const value = argv[++i];
|
|
3559
4559
|
if (value !== "all" && value !== "missing-only") {
|
|
@@ -3622,11 +4622,19 @@ Translate flags:
|
|
|
3622
4622
|
--locale <code>... Target locale(s); overrides --preset
|
|
3623
4623
|
--slug <en-slug> Single English document
|
|
3624
4624
|
--model <id> Gemini model override
|
|
3625
|
-
--
|
|
4625
|
+
--batch Force Gemini Batch API mode (50% token cost; default)
|
|
4626
|
+
--direct Force direct interactive calls (immediate, full price)
|
|
4627
|
+
--resume Only poll/ingest pending batch jobs; submit nothing new
|
|
4628
|
+
--concurrency <n> Parallel translations in --direct mode (default: 3)
|
|
3626
4629
|
--dry-run List work without writing
|
|
3627
4630
|
--force Re-translate even when hashes match
|
|
3628
4631
|
--strategy <mode> all (default) or missing-only
|
|
3629
4632
|
--no-progress Plain line logging instead of live progress
|
|
4633
|
+
|
|
4634
|
+
Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
|
|
4635
|
+
during polling is safe: the next run (or --resume) picks the jobs back up and
|
|
4636
|
+
ingests their results. In a TTY without --batch/--direct, a prompt asks which
|
|
4637
|
+
mode to use.
|
|
3630
4638
|
`);
|
|
3631
4639
|
return;
|
|
3632
4640
|
}
|
|
@@ -3672,11 +4680,29 @@ Translate flags:
|
|
|
3672
4680
|
break;
|
|
3673
4681
|
}
|
|
3674
4682
|
case "translate": {
|
|
4683
|
+
if (options.batch && options.direct) {
|
|
4684
|
+
throw new Error("--batch and --direct are mutually exclusive");
|
|
4685
|
+
}
|
|
4686
|
+
if (options.resume) {
|
|
4687
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4688
|
+
enabled: !options.noProgress,
|
|
4689
|
+
dryRun: false
|
|
4690
|
+
});
|
|
4691
|
+
const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4692
|
+
reporter2.finish();
|
|
4693
|
+
if (results2 === null) {
|
|
4694
|
+
console.log("No pending translation batch jobs.");
|
|
4695
|
+
break;
|
|
4696
|
+
}
|
|
4697
|
+
if (results2.some((result) => result.failed)) process.exitCode = 1;
|
|
4698
|
+
break;
|
|
4699
|
+
}
|
|
3675
4700
|
const selection = await promptTranslateSelection(config, {
|
|
3676
4701
|
type: options.type,
|
|
3677
4702
|
preset: options.preset,
|
|
3678
4703
|
locale: options.locale,
|
|
3679
|
-
strategy: options.strategy
|
|
4704
|
+
strategy: options.strategy,
|
|
4705
|
+
mode: options.batch ? "batch" : options.direct ? "direct" : void 0
|
|
3680
4706
|
});
|
|
3681
4707
|
const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
|
|
3682
4708
|
const worklist = buildWorklist(config, {
|
|
@@ -3686,6 +4712,18 @@ Translate flags:
|
|
|
3686
4712
|
strategy: selection.strategy ?? "all"
|
|
3687
4713
|
});
|
|
3688
4714
|
if (worklist.length === 0) {
|
|
4715
|
+
if (!options.dryRun) {
|
|
4716
|
+
const reporter2 = createTranslateProgressReporter({
|
|
4717
|
+
enabled: !options.noProgress,
|
|
4718
|
+
dryRun: false
|
|
4719
|
+
});
|
|
4720
|
+
const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
|
|
4721
|
+
reporter2.finish();
|
|
4722
|
+
if (resumed !== null) {
|
|
4723
|
+
if (resumed.some((result) => result.failed)) process.exitCode = 1;
|
|
4724
|
+
break;
|
|
4725
|
+
}
|
|
4726
|
+
}
|
|
3689
4727
|
console.log("Nothing to translate.");
|
|
3690
4728
|
break;
|
|
3691
4729
|
}
|
|
@@ -3698,6 +4736,7 @@ Translate flags:
|
|
|
3698
4736
|
dryRun: options.dryRun,
|
|
3699
4737
|
force: options.force,
|
|
3700
4738
|
concurrency: options.concurrency,
|
|
4739
|
+
mode: selection.mode,
|
|
3701
4740
|
onProgress: reporter.onEvent
|
|
3702
4741
|
});
|
|
3703
4742
|
reporter.finish();
|