scribe-cms 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1258 -219
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1259 -220
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +1020 -181
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1021 -183
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +149 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +149 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +23 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/src/validate/validate-project.d.ts.map +1 -1
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -257,6 +257,39 @@ function resolveConfig(input, baseDir) {
|
|
|
257
257
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
258
258
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
259
259
|
}
|
|
260
|
+
const localeFallbacks = {};
|
|
261
|
+
for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
|
|
262
|
+
if (!raw.locales.includes(locale)) {
|
|
263
|
+
throw new Error(
|
|
264
|
+
`scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
const seen = /* @__PURE__ */ new Set();
|
|
268
|
+
for (const fallback of chain) {
|
|
269
|
+
if (!raw.locales.includes(fallback)) {
|
|
270
|
+
throw new Error(
|
|
271
|
+
`scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
if (fallback === locale) {
|
|
275
|
+
throw new Error(
|
|
276
|
+
`scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
if (fallback === defaultLocale) {
|
|
280
|
+
throw new Error(
|
|
281
|
+
`scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
if (seen.has(fallback)) {
|
|
285
|
+
throw new Error(
|
|
286
|
+
`scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
seen.add(fallback);
|
|
290
|
+
}
|
|
291
|
+
localeFallbacks[locale] = [...chain];
|
|
292
|
+
}
|
|
260
293
|
const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
261
294
|
const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
|
|
262
295
|
const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -286,6 +319,7 @@ function resolveConfig(input, baseDir) {
|
|
|
286
319
|
defaultLocale,
|
|
287
320
|
localeRouting,
|
|
288
321
|
localePresets: raw.localePresets,
|
|
322
|
+
localeFallbacks,
|
|
289
323
|
translate: raw.translate,
|
|
290
324
|
types
|
|
291
325
|
};
|
|
@@ -297,6 +331,12 @@ function resolveConfig(input, baseDir) {
|
|
|
297
331
|
}
|
|
298
332
|
|
|
299
333
|
// src/core/introspect-schema.ts
|
|
334
|
+
function getArrayElement(schema) {
|
|
335
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
336
|
+
return schema.element;
|
|
337
|
+
}
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
300
340
|
function introspectSchema(schema, prefix = []) {
|
|
301
341
|
const relation = getRelationTarget(schema);
|
|
302
342
|
if (relation && prefix.length > 0) {
|
|
@@ -310,7 +350,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
310
350
|
}
|
|
311
351
|
];
|
|
312
352
|
}
|
|
313
|
-
|
|
353
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
354
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
355
|
+
}
|
|
356
|
+
const base = peelOptionalWrappers(schema);
|
|
314
357
|
if (base instanceof Object && "shape" in base) {
|
|
315
358
|
const shape = base.shape;
|
|
316
359
|
const fields = [];
|
|
@@ -319,9 +362,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
319
362
|
}
|
|
320
363
|
return fields;
|
|
321
364
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
const elementBase =
|
|
365
|
+
const element = getArrayElement(base);
|
|
366
|
+
if (element) {
|
|
367
|
+
const elementBase = peelOptionalWrappers(element);
|
|
325
368
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
326
369
|
const shape = elementBase.shape;
|
|
327
370
|
const fields = [];
|
|
@@ -333,15 +376,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
333
376
|
}
|
|
334
377
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
335
378
|
}
|
|
336
|
-
function
|
|
337
|
-
const
|
|
379
|
+
function buildPathTrie(paths) {
|
|
380
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
338
381
|
for (const path13 of paths) {
|
|
339
|
-
|
|
382
|
+
let node = root;
|
|
383
|
+
for (const segment of path13) {
|
|
384
|
+
let child = node.children.get(segment);
|
|
385
|
+
if (!child) {
|
|
386
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
387
|
+
node.children.set(segment, child);
|
|
388
|
+
}
|
|
389
|
+
node = child;
|
|
390
|
+
}
|
|
391
|
+
node.leaf = true;
|
|
392
|
+
}
|
|
393
|
+
return root;
|
|
394
|
+
}
|
|
395
|
+
function extractNode(data, trie) {
|
|
396
|
+
if (trie.leaf) return data;
|
|
397
|
+
const star = trie.children.get("*");
|
|
398
|
+
if (star) {
|
|
399
|
+
if (!Array.isArray(data)) return void 0;
|
|
400
|
+
return data.map(
|
|
401
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
405
|
+
const record = data;
|
|
406
|
+
const out = {};
|
|
407
|
+
let any = false;
|
|
408
|
+
for (const [key, child] of trie.children) {
|
|
409
|
+
const value = extractNode(record[key], child);
|
|
340
410
|
if (value !== void 0) {
|
|
341
|
-
|
|
411
|
+
out[key] = value;
|
|
412
|
+
any = true;
|
|
342
413
|
}
|
|
343
414
|
}
|
|
344
|
-
return out;
|
|
415
|
+
return any ? out : void 0;
|
|
416
|
+
}
|
|
417
|
+
function extractByPaths(data, paths) {
|
|
418
|
+
if (paths.length === 0) return {};
|
|
419
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
420
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
345
421
|
}
|
|
346
422
|
function pickTranslatable(data, schema) {
|
|
347
423
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -356,32 +432,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
356
432
|
const translatable = pickTranslatable(localeData, schema);
|
|
357
433
|
return deepMerge(structural, translatable);
|
|
358
434
|
}
|
|
359
|
-
function getAtPath(obj, path13) {
|
|
360
|
-
let current = obj;
|
|
361
|
-
for (const segment of path13) {
|
|
362
|
-
if (segment === "*") {
|
|
363
|
-
if (!Array.isArray(current)) return void 0;
|
|
364
|
-
return current.map(
|
|
365
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path13.slice(path13.indexOf("*") + 1)) : void 0
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
369
|
-
current = current[segment];
|
|
370
|
-
}
|
|
371
|
-
return current;
|
|
372
|
-
}
|
|
373
|
-
function setAtPath(obj, path13, value) {
|
|
374
|
-
if (path13.length === 0) return;
|
|
375
|
-
if (path13.length === 1) {
|
|
376
|
-
obj[path13[0]] = value;
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
const [head, ...rest] = path13;
|
|
380
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
381
|
-
obj[head] = {};
|
|
382
|
-
}
|
|
383
|
-
setAtPath(obj[head], rest, value);
|
|
384
|
-
}
|
|
385
435
|
function deepMerge(base, overlay) {
|
|
386
436
|
const out = { ...base };
|
|
387
437
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -551,7 +601,7 @@ function seoFieldsFromEn(enDoc) {
|
|
|
551
601
|
canonicalPathOverride: enDoc.canonicalPathOverride
|
|
552
602
|
};
|
|
553
603
|
}
|
|
554
|
-
var SCHEMA_VERSION =
|
|
604
|
+
var SCHEMA_VERSION = 5;
|
|
555
605
|
var MIGRATIONS = [
|
|
556
606
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
557
607
|
key TEXT PRIMARY KEY,
|
|
@@ -599,7 +649,30 @@ var MIGRATIONS = [
|
|
|
599
649
|
UNIQUE (content_type, en_slug, en_hash)
|
|
600
650
|
)`,
|
|
601
651
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
602
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
652
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
653
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
654
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
655
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
656
|
+
model TEXT NOT NULL,
|
|
657
|
+
display_model TEXT NOT NULL,
|
|
658
|
+
created_at TEXT NOT NULL,
|
|
659
|
+
state TEXT NOT NULL,
|
|
660
|
+
completed_at TEXT
|
|
661
|
+
)`,
|
|
662
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
663
|
+
job_id INTEGER NOT NULL,
|
|
664
|
+
request_index INTEGER NOT NULL,
|
|
665
|
+
content_type TEXT NOT NULL,
|
|
666
|
+
en_slug TEXT NOT NULL,
|
|
667
|
+
locale TEXT NOT NULL,
|
|
668
|
+
en_hash TEXT NOT NULL,
|
|
669
|
+
snapshot_id INTEGER NOT NULL,
|
|
670
|
+
status TEXT NOT NULL,
|
|
671
|
+
error TEXT,
|
|
672
|
+
PRIMARY KEY (job_id, request_index)
|
|
673
|
+
)`,
|
|
674
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
675
|
+
ON translation_batch_items(status)`
|
|
603
676
|
];
|
|
604
677
|
function resolveStorePath(config) {
|
|
605
678
|
return config.storePath;
|
|
@@ -622,6 +695,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
622
695
|
}
|
|
623
696
|
}
|
|
624
697
|
function readSchemaVersion(db) {
|
|
698
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
699
|
+
if (!metaTable) return 0;
|
|
625
700
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
626
701
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
627
702
|
}
|
|
@@ -660,6 +735,9 @@ function getOrCreateEnSnapshot(db, input) {
|
|
|
660
735
|
).get(input.contentType, input.enSlug, input.enHash);
|
|
661
736
|
return row.id;
|
|
662
737
|
}
|
|
738
|
+
function getEnSnapshot(db, snapshotId) {
|
|
739
|
+
return db.prepare(`SELECT * FROM en_snapshots WHERE id = ?`).get(snapshotId);
|
|
740
|
+
}
|
|
663
741
|
function upsertTranslation(db, input) {
|
|
664
742
|
db.prepare(
|
|
665
743
|
`INSERT INTO translations (
|
|
@@ -740,7 +818,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
740
818
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
741
819
|
const attrStart = i;
|
|
742
820
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
743
|
-
body.slice(attrStart, i);
|
|
821
|
+
const attrName = body.slice(attrStart, i);
|
|
822
|
+
if (!attrName) break;
|
|
744
823
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
745
824
|
if (body[i] !== "=") {
|
|
746
825
|
tagOut += body.slice(attrStart, i);
|
|
@@ -749,10 +828,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
749
828
|
tagOut += body.slice(attrStart, i);
|
|
750
829
|
tagOut += "=";
|
|
751
830
|
i += 1;
|
|
831
|
+
const wsStart = i;
|
|
752
832
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
833
|
+
tagOut += body.slice(wsStart, i);
|
|
753
834
|
const quote = body[i];
|
|
754
835
|
if (quote !== '"' && quote !== "'") {
|
|
755
|
-
|
|
836
|
+
break;
|
|
756
837
|
}
|
|
757
838
|
if (quote === "'") {
|
|
758
839
|
const valStart2 = i + 1;
|
|
@@ -1072,7 +1153,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1072
1153
|
}
|
|
1073
1154
|
|
|
1074
1155
|
// src/i18n/resolve-document.ts
|
|
1075
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1156
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1076
1157
|
const urlBuilder = createUrlBuilder({
|
|
1077
1158
|
locales: [defaultLocale, locale],
|
|
1078
1159
|
defaultLocale,
|
|
@@ -1086,15 +1167,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1086
1167
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1087
1168
|
const found = docIdx.bySlug.get(slug);
|
|
1088
1169
|
if (!found) continue;
|
|
1089
|
-
const
|
|
1090
|
-
|
|
1170
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1171
|
+
let target;
|
|
1172
|
+
let targetLocale = locale;
|
|
1173
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1174
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1175
|
+
if (cand) {
|
|
1176
|
+
target = cand;
|
|
1177
|
+
targetLocale = candidateLocale;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
if (target) {
|
|
1182
|
+
if (target.slug === slug) {
|
|
1183
|
+
return { document: target, actualLocale: targetLocale };
|
|
1184
|
+
}
|
|
1091
1185
|
if (!type.path) {
|
|
1092
1186
|
return { document: null, actualLocale: locale };
|
|
1093
1187
|
}
|
|
1094
1188
|
return {
|
|
1095
1189
|
document: null,
|
|
1096
1190
|
actualLocale: locale,
|
|
1097
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1191
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1098
1192
|
};
|
|
1099
1193
|
}
|
|
1100
1194
|
if (docLocale === defaultLocale) {
|
|
@@ -1125,13 +1219,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1125
1219
|
}
|
|
1126
1220
|
return { document: null, actualLocale: locale };
|
|
1127
1221
|
}
|
|
1128
|
-
function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1129
|
-
if (sourceLocale === targetLocale) return document2.slug;
|
|
1130
|
-
const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
|
|
1131
|
-
if (!englishSlug) return null;
|
|
1132
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1133
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1134
|
-
}
|
|
1135
1222
|
|
|
1136
1223
|
// src/create-project.ts
|
|
1137
1224
|
function comparatorFor(orderBy) {
|
|
@@ -1184,7 +1271,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1184
1271
|
config.defaultLocale,
|
|
1185
1272
|
load(),
|
|
1186
1273
|
type,
|
|
1187
|
-
config.localeRouting
|
|
1274
|
+
config.localeRouting,
|
|
1275
|
+
config.localeFallbacks[locale] ?? []
|
|
1188
1276
|
);
|
|
1189
1277
|
if (result.document && type.path) {
|
|
1190
1278
|
return {
|
|
@@ -1206,8 +1294,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1206
1294
|
const params = [];
|
|
1207
1295
|
for (const locale of options.locales ?? config.locales) {
|
|
1208
1296
|
const localeIdx = all.get(locale);
|
|
1297
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1209
1298
|
for (const doc of enIdx.bySlug.values()) {
|
|
1210
|
-
|
|
1299
|
+
let slug;
|
|
1300
|
+
if (locale === config.defaultLocale) {
|
|
1301
|
+
slug = doc.slug;
|
|
1302
|
+
} else {
|
|
1303
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1304
|
+
}
|
|
1211
1305
|
params.push({ locale, slug });
|
|
1212
1306
|
}
|
|
1213
1307
|
}
|
|
@@ -2026,7 +2120,7 @@ function validateTypeRedirects(project) {
|
|
|
2026
2120
|
}
|
|
2027
2121
|
return issues;
|
|
2028
2122
|
}
|
|
2029
|
-
function
|
|
2123
|
+
function getAtPath(obj, fieldPath) {
|
|
2030
2124
|
let current = obj;
|
|
2031
2125
|
for (const segment of fieldPath) {
|
|
2032
2126
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -2071,7 +2165,7 @@ function validateRelations(project) {
|
|
|
2071
2165
|
});
|
|
2072
2166
|
continue;
|
|
2073
2167
|
}
|
|
2074
|
-
const value =
|
|
2168
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
2075
2169
|
if (value === void 0 || value === null || value === "") continue;
|
|
2076
2170
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
2077
2171
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2220,6 +2314,20 @@ function listEnSlugs3(rootDir, contentDir) {
|
|
|
2220
2314
|
if (!fs2__default.default.existsSync(dir)) return [];
|
|
2221
2315
|
return fs2__default.default.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
|
|
2222
2316
|
}
|
|
2317
|
+
function validateDocumentMdxBody(issues, input) {
|
|
2318
|
+
const preparedBody = prepareTranslatedMdxBody(input.body).body;
|
|
2319
|
+
const mdxValidation = validateMdxBody(preparedBody);
|
|
2320
|
+
if (!mdxValidation.ok) {
|
|
2321
|
+
issues.push({
|
|
2322
|
+
level: "error",
|
|
2323
|
+
contentType: input.contentType,
|
|
2324
|
+
enSlug: input.enSlug,
|
|
2325
|
+
locale: input.locale,
|
|
2326
|
+
field: "body",
|
|
2327
|
+
message: `Invalid MDX: ${mdxValidation.error}`
|
|
2328
|
+
});
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2223
2331
|
function validateProject(config) {
|
|
2224
2332
|
const issues = [];
|
|
2225
2333
|
const project = createProject(config);
|
|
@@ -2278,6 +2386,12 @@ function validateProject(config) {
|
|
|
2278
2386
|
})) {
|
|
2279
2387
|
issues.push(issue);
|
|
2280
2388
|
}
|
|
2389
|
+
validateDocumentMdxBody(issues, {
|
|
2390
|
+
contentType: type.id,
|
|
2391
|
+
enSlug,
|
|
2392
|
+
locale: config.defaultLocale,
|
|
2393
|
+
body: enDoc.content
|
|
2394
|
+
});
|
|
2281
2395
|
for (const locale of config.locales) {
|
|
2282
2396
|
if (locale === config.defaultLocale) continue;
|
|
2283
2397
|
const row = getTranslation(db, type.id, enSlug, locale);
|
|
@@ -2303,17 +2417,12 @@ function validateProject(config) {
|
|
|
2303
2417
|
})) {
|
|
2304
2418
|
issues.push(issue);
|
|
2305
2419
|
}
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
locale,
|
|
2313
|
-
field: "body",
|
|
2314
|
-
message: `Invalid MDX: ${mdxValidation.error}`
|
|
2315
|
-
});
|
|
2316
|
-
}
|
|
2420
|
+
validateDocumentMdxBody(issues, {
|
|
2421
|
+
contentType: type.id,
|
|
2422
|
+
enSlug,
|
|
2423
|
+
locale,
|
|
2424
|
+
body: row.body
|
|
2425
|
+
});
|
|
2317
2426
|
}
|
|
2318
2427
|
}
|
|
2319
2428
|
try {
|
|
@@ -2423,6 +2532,59 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2423
2532
|
return config.locales.filter((l) => l !== config.defaultLocale);
|
|
2424
2533
|
}
|
|
2425
2534
|
|
|
2535
|
+
// src/storage/batch-jobs.ts
|
|
2536
|
+
function insertBatchJob(db, input) {
|
|
2537
|
+
const info = db.prepare(
|
|
2538
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2539
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2540
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2541
|
+
return Number(info.lastInsertRowid);
|
|
2542
|
+
}
|
|
2543
|
+
function insertBatchItems(db, jobId, items) {
|
|
2544
|
+
const stmt = db.prepare(
|
|
2545
|
+
`INSERT INTO translation_batch_items (
|
|
2546
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2547
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2548
|
+
);
|
|
2549
|
+
for (const item of items) {
|
|
2550
|
+
stmt.run(
|
|
2551
|
+
jobId,
|
|
2552
|
+
item.requestIndex,
|
|
2553
|
+
item.contentType,
|
|
2554
|
+
item.enSlug,
|
|
2555
|
+
item.locale,
|
|
2556
|
+
item.enHash,
|
|
2557
|
+
item.snapshotId
|
|
2558
|
+
);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
function listPendingBatchJobs(db) {
|
|
2562
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2563
|
+
}
|
|
2564
|
+
function listBatchItems(db, jobId) {
|
|
2565
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2566
|
+
}
|
|
2567
|
+
function listPendingBatchItems(db) {
|
|
2568
|
+
return db.prepare(
|
|
2569
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2570
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2571
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2572
|
+
ORDER BY i.job_id, i.request_index`
|
|
2573
|
+
).all();
|
|
2574
|
+
}
|
|
2575
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2576
|
+
const info = db.prepare(
|
|
2577
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2578
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2579
|
+
).run(state, completedAt, jobId);
|
|
2580
|
+
return info.changes > 0;
|
|
2581
|
+
}
|
|
2582
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2583
|
+
db.prepare(
|
|
2584
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2585
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2586
|
+
}
|
|
2587
|
+
|
|
2426
2588
|
// src/history/record-snapshot.ts
|
|
2427
2589
|
function recordEnSnapshot(config, input, db) {
|
|
2428
2590
|
const ownDb = db ?? openStore(config, "readwrite");
|
|
@@ -2437,8 +2599,102 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2437
2599
|
if (!db) ownDb.close();
|
|
2438
2600
|
return id;
|
|
2439
2601
|
}
|
|
2602
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2603
|
+
function statusFromError(error) {
|
|
2604
|
+
if (error instanceof genai.ApiError && typeof error.status === "number") {
|
|
2605
|
+
return error.status;
|
|
2606
|
+
}
|
|
2607
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2608
|
+
const status = error.status;
|
|
2609
|
+
if (typeof status === "number") return status;
|
|
2610
|
+
}
|
|
2611
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2612
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2613
|
+
if (match) return Number(match[1]);
|
|
2614
|
+
return void 0;
|
|
2615
|
+
}
|
|
2616
|
+
function isNetworkError(error) {
|
|
2617
|
+
const err = error;
|
|
2618
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2619
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2620
|
+
return true;
|
|
2621
|
+
}
|
|
2622
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2623
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2624
|
+
}
|
|
2625
|
+
function isTransientError(error) {
|
|
2626
|
+
const status = statusFromError(error);
|
|
2627
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2628
|
+
return isNetworkError(error);
|
|
2629
|
+
}
|
|
2630
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2631
|
+
async function withRetry(fn, options = {}) {
|
|
2632
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2633
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2634
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2635
|
+
const random = options.random ?? Math.random;
|
|
2636
|
+
let lastError;
|
|
2637
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2638
|
+
try {
|
|
2639
|
+
return await fn();
|
|
2640
|
+
} catch (error) {
|
|
2641
|
+
lastError = error;
|
|
2642
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2643
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2644
|
+
const delay = backoff + random() * backoff;
|
|
2645
|
+
await sleep2(delay);
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
throw lastError;
|
|
2649
|
+
}
|
|
2440
2650
|
|
|
2441
|
-
// src/translate/gemini-
|
|
2651
|
+
// src/translate/gemini-batch.ts
|
|
2652
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2653
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2654
|
+
"SUCCEEDED",
|
|
2655
|
+
"FAILED",
|
|
2656
|
+
"CANCELLED",
|
|
2657
|
+
"EXPIRED",
|
|
2658
|
+
"PARTIALLY_SUCCEEDED"
|
|
2659
|
+
]);
|
|
2660
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2661
|
+
function normalizeBatchState(state) {
|
|
2662
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2663
|
+
}
|
|
2664
|
+
function isTerminalBatchState(state) {
|
|
2665
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2666
|
+
}
|
|
2667
|
+
function isSuccessfulBatchState(state) {
|
|
2668
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2669
|
+
}
|
|
2670
|
+
function makeClient(apiKey) {
|
|
2671
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2672
|
+
if (!key) {
|
|
2673
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2674
|
+
}
|
|
2675
|
+
return new genai.GoogleGenAI({ apiKey: key });
|
|
2676
|
+
}
|
|
2677
|
+
function textFromBatchResponse(response) {
|
|
2678
|
+
if (typeof response.text === "string") return response.text;
|
|
2679
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2680
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2681
|
+
}
|
|
2682
|
+
async function createGeminiBatchJob(input) {
|
|
2683
|
+
const ai = makeClient(input.apiKey);
|
|
2684
|
+
const job = await withRetry(
|
|
2685
|
+
() => ai.batches.create({
|
|
2686
|
+
model: input.model,
|
|
2687
|
+
src: input.requests,
|
|
2688
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2689
|
+
})
|
|
2690
|
+
);
|
|
2691
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2692
|
+
return job;
|
|
2693
|
+
}
|
|
2694
|
+
async function getGeminiBatchJob(input) {
|
|
2695
|
+
const ai = makeClient(input.apiKey);
|
|
2696
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2697
|
+
}
|
|
2442
2698
|
var GEMINI_MODEL_IDS = {
|
|
2443
2699
|
"gemini-2.5-pro": "gemini-2.5-pro",
|
|
2444
2700
|
"gemini-3.1-pro": "gemini-3.1-pro-preview"
|
|
@@ -2458,16 +2714,55 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2458
2714
|
if (alias) return alias[0];
|
|
2459
2715
|
return name;
|
|
2460
2716
|
}
|
|
2717
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2718
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2719
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
|
|
2720
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2721
|
+
return void 0;
|
|
2722
|
+
}
|
|
2461
2723
|
|
|
2462
2724
|
// src/translate/gemini-client.ts
|
|
2463
2725
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
2464
2726
|
function extractJson(text) {
|
|
2465
|
-
const
|
|
2727
|
+
const trimmed = text.trim();
|
|
2728
|
+
const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
|
2466
2729
|
if (fenced?.[1]) return fenced[1].trim();
|
|
2467
|
-
const start =
|
|
2468
|
-
const end =
|
|
2469
|
-
if (start >= 0 && end > start) return
|
|
2470
|
-
return
|
|
2730
|
+
const start = trimmed.indexOf("{");
|
|
2731
|
+
const end = trimmed.lastIndexOf("}");
|
|
2732
|
+
if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
|
|
2733
|
+
return trimmed;
|
|
2734
|
+
}
|
|
2735
|
+
function parseGeminiResponse(text) {
|
|
2736
|
+
let parsed;
|
|
2737
|
+
try {
|
|
2738
|
+
parsed = JSON.parse(text.trim());
|
|
2739
|
+
} catch {
|
|
2740
|
+
parsed = JSON.parse(extractJson(text));
|
|
2741
|
+
}
|
|
2742
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2743
|
+
parsed.frontmatter = {};
|
|
2744
|
+
}
|
|
2745
|
+
return parsed;
|
|
2746
|
+
}
|
|
2747
|
+
function buildGeminiRequestConfig(input) {
|
|
2748
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2749
|
+
return {
|
|
2750
|
+
responseMimeType: "application/json",
|
|
2751
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2752
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2753
|
+
};
|
|
2754
|
+
}
|
|
2755
|
+
function usageFromResponse(response) {
|
|
2756
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2757
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2758
|
+
return {
|
|
2759
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2760
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2761
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2762
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2763
|
+
thoughtsTokens,
|
|
2764
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2765
|
+
};
|
|
2471
2766
|
}
|
|
2472
2767
|
async function translatePageWithGemini(input) {
|
|
2473
2768
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2479,26 +2774,22 @@ async function translatePageWithGemini(input) {
|
|
|
2479
2774
|
);
|
|
2480
2775
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2481
2776
|
const ai = new genai.GoogleGenAI({ apiKey });
|
|
2482
|
-
const response = await
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2777
|
+
const response = await withRetry(
|
|
2778
|
+
() => ai.models.generateContent({
|
|
2779
|
+
model: apiModel,
|
|
2780
|
+
contents: input.prompt,
|
|
2781
|
+
config: buildGeminiRequestConfig({
|
|
2782
|
+
apiModelId: apiModel,
|
|
2783
|
+
responseSchema: input.responseSchema
|
|
2784
|
+
})
|
|
2785
|
+
})
|
|
2786
|
+
);
|
|
2490
2787
|
const raw = response.text ?? "";
|
|
2491
|
-
const parsed =
|
|
2788
|
+
const parsed = parseGeminiResponse(raw);
|
|
2492
2789
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2493
2790
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2494
2791
|
}
|
|
2495
|
-
|
|
2496
|
-
const usage = {
|
|
2497
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2498
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2499
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2500
|
-
};
|
|
2501
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2792
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2502
2793
|
}
|
|
2503
2794
|
|
|
2504
2795
|
// src/translate/gemini-pricing.ts
|
|
@@ -2518,12 +2809,14 @@ function resolveModelPricing(model) {
|
|
|
2518
2809
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2519
2810
|
return match?.[1];
|
|
2520
2811
|
}
|
|
2521
|
-
|
|
2812
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2813
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2522
2814
|
const pricing = resolveModelPricing(model);
|
|
2523
2815
|
if (!pricing) return void 0;
|
|
2524
2816
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2525
2817
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2526
|
-
|
|
2818
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2819
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2527
2820
|
}
|
|
2528
2821
|
|
|
2529
2822
|
// src/translate/prompts/translation-prompt.ts
|
|
@@ -2539,6 +2832,7 @@ var LOCALE_NAMES = {
|
|
|
2539
2832
|
"zh-CN": "Simplified Chinese",
|
|
2540
2833
|
"zh-TW": "Traditional Chinese",
|
|
2541
2834
|
pt: "Portuguese",
|
|
2835
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2542
2836
|
ja: "Japanese",
|
|
2543
2837
|
ru: "Russian",
|
|
2544
2838
|
it: "Italian",
|
|
@@ -2546,29 +2840,29 @@ var LOCALE_NAMES = {
|
|
|
2546
2840
|
};
|
|
2547
2841
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2548
2842
|
return [
|
|
2549
|
-
`Localize the content
|
|
2843
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2550
2844
|
"Do not translate word-for-word.",
|
|
2551
2845
|
"Preserve the source tone and brand voice.",
|
|
2552
2846
|
"Write as if a native speaker authored it for the target market."
|
|
2553
2847
|
].join(" ");
|
|
2554
2848
|
}
|
|
2849
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2850
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2851
|
+
if (!hasFrontmatter) {
|
|
2852
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2853
|
+
}
|
|
2854
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2855
|
+
}
|
|
2555
2856
|
function buildPageTranslationPrompt(input) {
|
|
2556
2857
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2557
|
-
const
|
|
2558
|
-
const
|
|
2559
|
-
|
|
2858
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2859
|
+
const prefix = [
|
|
2860
|
+
TASK_FRAMING,
|
|
2560
2861
|
"",
|
|
2561
2862
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2562
2863
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2563
2864
|
"## Rules",
|
|
2564
2865
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2565
|
-
...input.slugStrategy === "localized" ? [
|
|
2566
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2567
|
-
] : [],
|
|
2568
|
-
"",
|
|
2569
|
-
"## Output format",
|
|
2570
|
-
"Return ONLY valid JSON with keys:",
|
|
2571
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2572
2866
|
"",
|
|
2573
2867
|
"## EN translatable frontmatter (JSON)",
|
|
2574
2868
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2576,7 +2870,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2576
2870
|
"## EN body (MDX)",
|
|
2577
2871
|
input.enBody
|
|
2578
2872
|
];
|
|
2579
|
-
|
|
2873
|
+
const suffix = [
|
|
2874
|
+
"",
|
|
2875
|
+
"## Target language",
|
|
2876
|
+
localizationPrompt,
|
|
2877
|
+
...input.slugStrategy === "localized" ? [
|
|
2878
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2879
|
+
] : [],
|
|
2880
|
+
"",
|
|
2881
|
+
"## Output format",
|
|
2882
|
+
"Return ONLY valid JSON with keys:",
|
|
2883
|
+
buildOutputFormatLine(
|
|
2884
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2885
|
+
input.slugStrategy
|
|
2886
|
+
)
|
|
2887
|
+
];
|
|
2888
|
+
return [...prefix, ...suffix].join("\n");
|
|
2580
2889
|
}
|
|
2581
2890
|
function getObjectShape(schema) {
|
|
2582
2891
|
const base = peelOptionalWrappers(schema);
|
|
@@ -2624,9 +2933,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2624
2933
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2625
2934
|
try {
|
|
2626
2935
|
const translatable = buildTranslatableSubschema(schema);
|
|
2627
|
-
if (!translatable) return null;
|
|
2628
2936
|
const responseShape = {
|
|
2629
|
-
frontmatter: translatable,
|
|
2937
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2630
2938
|
body: zod.z.string()
|
|
2631
2939
|
};
|
|
2632
2940
|
if (slugStrategy === "localized") {
|
|
@@ -2705,7 +3013,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2705
3013
|
return { ok: true, frontmatter };
|
|
2706
3014
|
}
|
|
2707
3015
|
|
|
2708
|
-
// src/translate/
|
|
3016
|
+
// src/translate/translate-core.ts
|
|
3017
|
+
function translationItemKey(item) {
|
|
3018
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
3019
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
3020
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
3021
|
+
}
|
|
2709
3022
|
function summarizeResults(results, durationMs) {
|
|
2710
3023
|
return results.reduce(
|
|
2711
3024
|
(totals, result) => {
|
|
@@ -2741,17 +3054,6 @@ function formatTranslateError(error) {
|
|
|
2741
3054
|
}
|
|
2742
3055
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2743
3056
|
}
|
|
2744
|
-
async function runPool(items, concurrency, worker) {
|
|
2745
|
-
if (items.length === 0) return;
|
|
2746
|
-
let nextIndex = 0;
|
|
2747
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2748
|
-
while (nextIndex < items.length) {
|
|
2749
|
-
const index = nextIndex++;
|
|
2750
|
-
await worker(items[index], index);
|
|
2751
|
-
}
|
|
2752
|
-
});
|
|
2753
|
-
await Promise.all(runners);
|
|
2754
|
-
}
|
|
2755
3057
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2756
3058
|
const fm = enDoc.frontmatter;
|
|
2757
3059
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2760,73 +3062,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2760
3062
|
}
|
|
2761
3063
|
return enSlug;
|
|
2762
3064
|
}
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
const base = {
|
|
3065
|
+
function baseForItem(item) {
|
|
3066
|
+
return {
|
|
2766
3067
|
contentType: item.contentType,
|
|
2767
3068
|
enSlug: item.enSlug,
|
|
2768
3069
|
locale: item.locale
|
|
2769
3070
|
};
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
3071
|
+
}
|
|
3072
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
3073
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
3074
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
3075
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
3076
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
3077
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
3078
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
3079
|
+
const db = openStore(config, "readonly");
|
|
3080
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
3081
|
+
db.close();
|
|
3082
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
3083
|
+
return {
|
|
3084
|
+
status: "done",
|
|
3085
|
+
result: {
|
|
3086
|
+
...baseForItem(item),
|
|
2783
3087
|
skipped: true,
|
|
2784
3088
|
reason: "fresh",
|
|
2785
3089
|
durationMs: Date.now() - startedAt
|
|
2786
|
-
}
|
|
2787
|
-
}
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
3090
|
+
}
|
|
3091
|
+
};
|
|
3092
|
+
}
|
|
3093
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
3094
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
3095
|
+
const prompt = buildPageTranslationPrompt({
|
|
3096
|
+
resolved: resolvedTranslate,
|
|
3097
|
+
targetLocale: item.locale,
|
|
3098
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
3099
|
+
translatableFrontmatter: payload.frontmatter,
|
|
3100
|
+
enBody: payload.body,
|
|
3101
|
+
slugStrategy: type.slugStrategy
|
|
3102
|
+
});
|
|
3103
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
3104
|
+
return {
|
|
3105
|
+
status: "ready",
|
|
3106
|
+
prepared: {
|
|
3107
|
+
item,
|
|
3108
|
+
type,
|
|
3109
|
+
enDoc,
|
|
3110
|
+
payload,
|
|
3111
|
+
currentEnHash,
|
|
3112
|
+
existingSlug: existing?.slug,
|
|
2809
3113
|
model,
|
|
3114
|
+
prompt,
|
|
2810
3115
|
responseSchema: responseSchema ?? void 0
|
|
2811
|
-
}
|
|
2812
|
-
|
|
3116
|
+
}
|
|
3117
|
+
};
|
|
3118
|
+
}
|
|
3119
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
3120
|
+
const { item, type, enDoc, payload } = prepared;
|
|
3121
|
+
const base = baseForItem(item);
|
|
3122
|
+
try {
|
|
3123
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2813
3124
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2814
3125
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2815
3126
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2816
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
3127
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2817
3128
|
if (!validated.ok) {
|
|
2818
3129
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2819
3130
|
}
|
|
2820
3131
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2821
|
-
|
|
3132
|
+
output.parsed.body
|
|
2822
3133
|
);
|
|
2823
3134
|
const writeDb = openStore(config, "readwrite");
|
|
2824
|
-
const snapshotId = recordEnSnapshot(
|
|
3135
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2825
3136
|
config,
|
|
2826
3137
|
{
|
|
2827
3138
|
contentType: type.id,
|
|
2828
3139
|
enSlug: item.enSlug,
|
|
2829
|
-
enHash: currentEnHash,
|
|
3140
|
+
enHash: prepared.currentEnHash,
|
|
2830
3141
|
frontmatter: payload.frontmatter,
|
|
2831
3142
|
body: payload.body
|
|
2832
3143
|
},
|
|
@@ -2839,24 +3150,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2839
3150
|
slug,
|
|
2840
3151
|
frontmatter: validated.frontmatter,
|
|
2841
3152
|
body: translatedBody,
|
|
2842
|
-
enHash: currentEnHash,
|
|
3153
|
+
enHash: prepared.currentEnHash,
|
|
2843
3154
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2844
|
-
model:
|
|
3155
|
+
model: output.model,
|
|
2845
3156
|
snapshotId
|
|
2846
3157
|
});
|
|
2847
3158
|
writeDb.close();
|
|
2848
3159
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2849
|
-
normalizeGeminiDisplayName(
|
|
2850
|
-
|
|
2851
|
-
|
|
3160
|
+
normalizeGeminiDisplayName(output.model),
|
|
3161
|
+
output.usage.inputTokens,
|
|
3162
|
+
output.usage.outputTokens,
|
|
3163
|
+
options.costMode
|
|
2852
3164
|
);
|
|
2853
3165
|
return {
|
|
2854
3166
|
...base,
|
|
2855
3167
|
skipped: false,
|
|
2856
|
-
model:
|
|
2857
|
-
usage:
|
|
3168
|
+
model: output.model,
|
|
3169
|
+
usage: output.usage,
|
|
2858
3170
|
estimatedCostUsd,
|
|
2859
|
-
durationMs: Date.now() - startedAt,
|
|
3171
|
+
durationMs: Date.now() - options.startedAt,
|
|
2860
3172
|
slugAdjusted,
|
|
2861
3173
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2862
3174
|
};
|
|
@@ -2866,33 +3178,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2866
3178
|
skipped: false,
|
|
2867
3179
|
failed: true,
|
|
2868
3180
|
error: formatTranslateError(error),
|
|
3181
|
+
durationMs: Date.now() - options.startedAt
|
|
3182
|
+
};
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
function displayModelFor(prepared) {
|
|
3186
|
+
return normalizeGeminiDisplayName(
|
|
3187
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
3188
|
+
);
|
|
3189
|
+
}
|
|
3190
|
+
|
|
3191
|
+
// src/translate/batch-worklist.ts
|
|
3192
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
3193
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
3194
|
+
var INITIAL_POLL_MS = 5e3;
|
|
3195
|
+
var MAX_POLL_MS = 3e4;
|
|
3196
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
3197
|
+
function sleep(ms) {
|
|
3198
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3199
|
+
}
|
|
3200
|
+
function planBatchJobs(entries) {
|
|
3201
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
3202
|
+
for (const entry of entries) {
|
|
3203
|
+
const group = byModel.get(entry.apiModel);
|
|
3204
|
+
if (group) group.push(entry);
|
|
3205
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
3206
|
+
}
|
|
3207
|
+
const plans = [];
|
|
3208
|
+
for (const [apiModel, group] of byModel) {
|
|
3209
|
+
let current = [];
|
|
3210
|
+
let currentBytes = 0;
|
|
3211
|
+
for (const entry of group) {
|
|
3212
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
3213
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
3214
|
+
plans.push({ apiModel, entries: current });
|
|
3215
|
+
current = [];
|
|
3216
|
+
currentBytes = 0;
|
|
3217
|
+
}
|
|
3218
|
+
current.push(entry);
|
|
3219
|
+
currentBytes += size;
|
|
3220
|
+
}
|
|
3221
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
3222
|
+
}
|
|
3223
|
+
return plans;
|
|
3224
|
+
}
|
|
3225
|
+
function readPendingBatchWork(config) {
|
|
3226
|
+
const db = openStore(config, "readwrite");
|
|
3227
|
+
const jobs = listPendingBatchJobs(db);
|
|
3228
|
+
const pendingItems = listPendingBatchItems(db);
|
|
3229
|
+
db.close();
|
|
3230
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
3231
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
3232
|
+
}
|
|
3233
|
+
async function submitBatchJobPlan(config, input) {
|
|
3234
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
3235
|
+
const job = await createGeminiBatchJob({
|
|
3236
|
+
model: plan.apiModel,
|
|
3237
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
3238
|
+
contents: prepared.prompt,
|
|
3239
|
+
config: buildGeminiRequestConfig({
|
|
3240
|
+
apiModelId: plan.apiModel,
|
|
3241
|
+
responseSchema: prepared.responseSchema
|
|
3242
|
+
})
|
|
3243
|
+
})),
|
|
3244
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
3245
|
+
});
|
|
3246
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3247
|
+
const db = openStore(config, "readwrite");
|
|
3248
|
+
const jobId = insertBatchJob(db, {
|
|
3249
|
+
jobName: job.name,
|
|
3250
|
+
model: plan.apiModel,
|
|
3251
|
+
displayModel,
|
|
3252
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3253
|
+
createdAt
|
|
3254
|
+
});
|
|
3255
|
+
insertBatchItems(
|
|
3256
|
+
db,
|
|
3257
|
+
jobId,
|
|
3258
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
3259
|
+
requestIndex,
|
|
3260
|
+
contentType: prepared.item.contentType,
|
|
3261
|
+
enSlug: prepared.item.enSlug,
|
|
3262
|
+
locale: prepared.item.locale,
|
|
3263
|
+
enHash: prepared.currentEnHash,
|
|
3264
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
3265
|
+
// on the EN files still matching (or existing) on disk.
|
|
3266
|
+
snapshotId: recordEnSnapshot(
|
|
3267
|
+
config,
|
|
3268
|
+
{
|
|
3269
|
+
contentType: prepared.item.contentType,
|
|
3270
|
+
enSlug: prepared.item.enSlug,
|
|
3271
|
+
enHash: prepared.currentEnHash,
|
|
3272
|
+
frontmatter: prepared.payload.frontmatter,
|
|
3273
|
+
body: prepared.payload.body
|
|
3274
|
+
},
|
|
3275
|
+
db
|
|
3276
|
+
)
|
|
3277
|
+
}))
|
|
3278
|
+
);
|
|
3279
|
+
const row = {
|
|
3280
|
+
id: jobId,
|
|
3281
|
+
job_name: job.name,
|
|
3282
|
+
model: plan.apiModel,
|
|
3283
|
+
display_model: displayModel,
|
|
3284
|
+
created_at: createdAt,
|
|
3285
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3286
|
+
completed_at: null
|
|
3287
|
+
};
|
|
3288
|
+
db.close();
|
|
3289
|
+
input.onProgress?.({
|
|
3290
|
+
type: "batch-submitted",
|
|
3291
|
+
name: job.name,
|
|
3292
|
+
count: plan.entries.length,
|
|
3293
|
+
model: displayModel,
|
|
3294
|
+
jobIndex,
|
|
3295
|
+
jobCount,
|
|
3296
|
+
createdAt
|
|
3297
|
+
});
|
|
3298
|
+
return row;
|
|
3299
|
+
}
|
|
3300
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3301
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3302
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3303
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3304
|
+
if (!enDoc) {
|
|
3305
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3306
|
+
if (!snapshot) {
|
|
3307
|
+
throw new Error(
|
|
3308
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3309
|
+
);
|
|
3310
|
+
}
|
|
3311
|
+
enDoc = {
|
|
3312
|
+
slug: itemRow.en_slug,
|
|
3313
|
+
enSlug: itemRow.en_slug,
|
|
3314
|
+
locale: config.defaultLocale,
|
|
3315
|
+
noindex: false,
|
|
3316
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3317
|
+
content: snapshot.body
|
|
3318
|
+
};
|
|
3319
|
+
}
|
|
3320
|
+
const item = {
|
|
3321
|
+
contentType: itemRow.content_type,
|
|
3322
|
+
enSlug: itemRow.en_slug,
|
|
3323
|
+
locale: itemRow.locale,
|
|
3324
|
+
reason: "missing",
|
|
3325
|
+
currentEnHash: itemRow.en_hash
|
|
3326
|
+
};
|
|
3327
|
+
return {
|
|
3328
|
+
item,
|
|
3329
|
+
type,
|
|
3330
|
+
enDoc,
|
|
3331
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3332
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3333
|
+
currentEnHash: itemRow.en_hash,
|
|
3334
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3335
|
+
model: jobRow.display_model,
|
|
3336
|
+
prompt: "",
|
|
3337
|
+
responseSchema: void 0
|
|
3338
|
+
};
|
|
3339
|
+
}
|
|
3340
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3341
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3342
|
+
const db = openStore(config, "readwrite");
|
|
3343
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3344
|
+
db.close();
|
|
3345
|
+
return null;
|
|
3346
|
+
}
|
|
3347
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3348
|
+
const results = [];
|
|
3349
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3350
|
+
const result = {
|
|
3351
|
+
contentType: itemRow.content_type,
|
|
3352
|
+
enSlug: itemRow.en_slug,
|
|
3353
|
+
locale: itemRow.locale,
|
|
3354
|
+
skipped: false,
|
|
3355
|
+
failed: true,
|
|
3356
|
+
error: formatTranslateError(new Error(message)),
|
|
2869
3357
|
durationMs: Date.now() - startedAt
|
|
2870
3358
|
};
|
|
3359
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3360
|
+
results.push(result);
|
|
3361
|
+
onResult?.(result);
|
|
3362
|
+
};
|
|
3363
|
+
if (isSuccessfulBatchState(state)) {
|
|
3364
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3365
|
+
for (const itemRow of pendingItems) {
|
|
3366
|
+
const startedAt = Date.now();
|
|
3367
|
+
const inlined = responses[itemRow.request_index];
|
|
3368
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3369
|
+
failItem(
|
|
3370
|
+
itemRow,
|
|
3371
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3372
|
+
startedAt
|
|
3373
|
+
);
|
|
3374
|
+
continue;
|
|
3375
|
+
}
|
|
3376
|
+
let result;
|
|
3377
|
+
try {
|
|
3378
|
+
const response = inlined.response;
|
|
3379
|
+
const raw = textFromBatchResponse(response);
|
|
3380
|
+
const parsed = parseGeminiResponse(raw);
|
|
3381
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3382
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3383
|
+
}
|
|
3384
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3385
|
+
result = finalizeTranslation(
|
|
3386
|
+
config,
|
|
3387
|
+
prepared,
|
|
3388
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3389
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3390
|
+
);
|
|
3391
|
+
} catch (error) {
|
|
3392
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3393
|
+
continue;
|
|
3394
|
+
}
|
|
3395
|
+
updateBatchItemStatus(
|
|
3396
|
+
db,
|
|
3397
|
+
jobRow.id,
|
|
3398
|
+
itemRow.request_index,
|
|
3399
|
+
result.failed ? "failed" : "done",
|
|
3400
|
+
result.error
|
|
3401
|
+
);
|
|
3402
|
+
results.push(result);
|
|
3403
|
+
onResult?.(result);
|
|
3404
|
+
}
|
|
3405
|
+
} else {
|
|
3406
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3407
|
+
const startedAt = Date.now();
|
|
3408
|
+
for (const itemRow of pendingItems) {
|
|
3409
|
+
failItem(itemRow, message, startedAt);
|
|
3410
|
+
}
|
|
2871
3411
|
}
|
|
3412
|
+
db.close();
|
|
3413
|
+
return results;
|
|
3414
|
+
}
|
|
3415
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3416
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3417
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3418
|
+
const loopStartedAt = Date.now();
|
|
3419
|
+
let delay = INITIAL_POLL_MS;
|
|
3420
|
+
let firstRound = true;
|
|
3421
|
+
const elapsedFor = (row) => {
|
|
3422
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3423
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3424
|
+
};
|
|
3425
|
+
while (active.length > 0) {
|
|
3426
|
+
if (!firstRound) {
|
|
3427
|
+
await sleep(delay);
|
|
3428
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3429
|
+
}
|
|
3430
|
+
firstRound = false;
|
|
3431
|
+
for (const tracked of [...active]) {
|
|
3432
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3433
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3434
|
+
options.onProgress?.({
|
|
3435
|
+
type: "batch-polling",
|
|
3436
|
+
name: tracked.row.job_name,
|
|
3437
|
+
state,
|
|
3438
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3439
|
+
jobIndex: tracked.jobIndex,
|
|
3440
|
+
jobCount
|
|
3441
|
+
});
|
|
3442
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3443
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3444
|
+
const results = ingestBatchJob(
|
|
3445
|
+
config,
|
|
3446
|
+
tracked.row,
|
|
3447
|
+
batchJob,
|
|
3448
|
+
options.onResult
|
|
3449
|
+
);
|
|
3450
|
+
if (results === null) {
|
|
3451
|
+
options.onProgress?.({
|
|
3452
|
+
type: "batch-done",
|
|
3453
|
+
name: tracked.row.job_name,
|
|
3454
|
+
state,
|
|
3455
|
+
model: tracked.row.display_model,
|
|
3456
|
+
count: 0,
|
|
3457
|
+
jobIndex: tracked.jobIndex,
|
|
3458
|
+
jobCount,
|
|
3459
|
+
translated: 0,
|
|
3460
|
+
failed: 0,
|
|
3461
|
+
inputTokens: 0,
|
|
3462
|
+
outputTokens: 0,
|
|
3463
|
+
estimatedCostUsd: 0,
|
|
3464
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3465
|
+
alreadyIngested: true
|
|
3466
|
+
});
|
|
3467
|
+
continue;
|
|
3468
|
+
}
|
|
3469
|
+
options.onProgress?.({
|
|
3470
|
+
type: "batch-done",
|
|
3471
|
+
name: tracked.row.job_name,
|
|
3472
|
+
state,
|
|
3473
|
+
model: tracked.row.display_model,
|
|
3474
|
+
count: results.length,
|
|
3475
|
+
jobIndex: tracked.jobIndex,
|
|
3476
|
+
jobCount,
|
|
3477
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3478
|
+
failed: results.filter((r) => r.failed).length,
|
|
3479
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3480
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3481
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3482
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3483
|
+
});
|
|
3484
|
+
}
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
}
|
|
3488
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3489
|
+
return {
|
|
3490
|
+
...baseForItem(item),
|
|
3491
|
+
skipped: false,
|
|
3492
|
+
failed: true,
|
|
3493
|
+
error: formatTranslateError(error),
|
|
3494
|
+
durationMs: Date.now() - startedAt
|
|
3495
|
+
};
|
|
3496
|
+
}
|
|
3497
|
+
|
|
3498
|
+
// src/translate/page-translator.ts
|
|
3499
|
+
async function runPool(items, concurrency, worker) {
|
|
3500
|
+
if (items.length === 0) return;
|
|
3501
|
+
let nextIndex = 0;
|
|
3502
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3503
|
+
while (nextIndex < items.length) {
|
|
3504
|
+
const index = nextIndex++;
|
|
3505
|
+
await worker(items[index], index);
|
|
3506
|
+
}
|
|
3507
|
+
});
|
|
3508
|
+
await Promise.all(runners);
|
|
2872
3509
|
}
|
|
2873
3510
|
function labelForItem(item) {
|
|
2874
3511
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2875
3512
|
}
|
|
3513
|
+
async function translatePage(config, item, options = {}) {
|
|
3514
|
+
const startedAt = Date.now();
|
|
3515
|
+
let outcome;
|
|
3516
|
+
try {
|
|
3517
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3518
|
+
} catch (error) {
|
|
3519
|
+
return failedResultForItem(item, error, startedAt);
|
|
3520
|
+
}
|
|
3521
|
+
if (outcome.status === "done") return outcome.result;
|
|
3522
|
+
const prepared = outcome.prepared;
|
|
3523
|
+
if (options.dryRun) {
|
|
3524
|
+
return {
|
|
3525
|
+
...baseForItem(item),
|
|
3526
|
+
skipped: false,
|
|
3527
|
+
model: prepared.model,
|
|
3528
|
+
durationMs: Date.now() - startedAt
|
|
3529
|
+
};
|
|
3530
|
+
}
|
|
3531
|
+
try {
|
|
3532
|
+
const result = await translatePageWithGemini({
|
|
3533
|
+
prompt: prepared.prompt,
|
|
3534
|
+
model: prepared.model,
|
|
3535
|
+
responseSchema: prepared.responseSchema
|
|
3536
|
+
});
|
|
3537
|
+
return finalizeTranslation(
|
|
3538
|
+
config,
|
|
3539
|
+
prepared,
|
|
3540
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3541
|
+
{ costMode: "interactive", startedAt }
|
|
3542
|
+
);
|
|
3543
|
+
} catch (error) {
|
|
3544
|
+
return failedResultForItem(item, error, startedAt);
|
|
3545
|
+
}
|
|
3546
|
+
}
|
|
3547
|
+
function createResultCollector(items, onProgress) {
|
|
3548
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3549
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3550
|
+
const slots = new Array(items.length);
|
|
3551
|
+
const extras = [];
|
|
3552
|
+
return {
|
|
3553
|
+
emit(result) {
|
|
3554
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3555
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3556
|
+
else extras.push(result);
|
|
3557
|
+
onProgress?.({ type: "item-done", result });
|
|
3558
|
+
},
|
|
3559
|
+
assemble() {
|
|
3560
|
+
return [
|
|
3561
|
+
...slots.filter((result) => result !== void 0),
|
|
3562
|
+
...extras
|
|
3563
|
+
];
|
|
3564
|
+
}
|
|
3565
|
+
};
|
|
3566
|
+
}
|
|
2876
3567
|
async function translateWorklist(config, items, options = {}) {
|
|
3568
|
+
const mode = options.mode ?? "batch";
|
|
2877
3569
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2878
3570
|
const startedAt = Date.now();
|
|
2879
|
-
|
|
2880
|
-
|
|
3571
|
+
if (options.dryRun) {
|
|
3572
|
+
options.onProgress?.({
|
|
3573
|
+
type: "start",
|
|
3574
|
+
total: items.length,
|
|
3575
|
+
concurrency,
|
|
3576
|
+
dryRun: true,
|
|
3577
|
+
model: options.model,
|
|
3578
|
+
mode
|
|
3579
|
+
});
|
|
3580
|
+
const results2 = items.map((item) => {
|
|
3581
|
+
const itemStartedAt = Date.now();
|
|
3582
|
+
try {
|
|
3583
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3584
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3585
|
+
...baseForItem(item),
|
|
3586
|
+
skipped: false,
|
|
3587
|
+
model: outcome.prepared.model,
|
|
3588
|
+
durationMs: Date.now() - itemStartedAt
|
|
3589
|
+
};
|
|
3590
|
+
} catch (error) {
|
|
3591
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3592
|
+
}
|
|
3593
|
+
});
|
|
3594
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3595
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3596
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3597
|
+
return results2;
|
|
3598
|
+
}
|
|
3599
|
+
const pending = readPendingBatchWork(config);
|
|
3600
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3601
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3602
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3603
|
+
).length;
|
|
2881
3604
|
options.onProgress?.({
|
|
2882
3605
|
type: "start",
|
|
2883
|
-
total: items.length,
|
|
3606
|
+
total: items.length + resumedExtraCount,
|
|
2884
3607
|
concurrency,
|
|
2885
|
-
dryRun:
|
|
2886
|
-
model: options.model
|
|
3608
|
+
dryRun: false,
|
|
3609
|
+
model: options.model,
|
|
3610
|
+
mode
|
|
3611
|
+
});
|
|
3612
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3613
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3614
|
+
if (mode === "direct") {
|
|
3615
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3616
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3617
|
+
const active = /* @__PURE__ */ new Set();
|
|
3618
|
+
await Promise.all([
|
|
3619
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3620
|
+
const label = labelForItem(item);
|
|
3621
|
+
active.add(label);
|
|
3622
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3623
|
+
const result = await translatePage(config, item, options);
|
|
3624
|
+
active.delete(label);
|
|
3625
|
+
collector.emit(result);
|
|
3626
|
+
}),
|
|
3627
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3628
|
+
onProgress: options.onProgress,
|
|
3629
|
+
onResult: collector.emit
|
|
3630
|
+
}) : Promise.resolve()
|
|
3631
|
+
]);
|
|
3632
|
+
} else {
|
|
3633
|
+
const entries = [];
|
|
3634
|
+
for (const item of workItems) {
|
|
3635
|
+
const itemStartedAt = Date.now();
|
|
3636
|
+
try {
|
|
3637
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3638
|
+
if (outcome.status === "done") {
|
|
3639
|
+
collector.emit(outcome.result);
|
|
3640
|
+
} else {
|
|
3641
|
+
entries.push({
|
|
3642
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3643
|
+
prompt: outcome.prepared.prompt,
|
|
3644
|
+
prepared: outcome.prepared
|
|
3645
|
+
});
|
|
3646
|
+
}
|
|
3647
|
+
} catch (error) {
|
|
3648
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3649
|
+
}
|
|
3650
|
+
}
|
|
3651
|
+
const plans = planBatchJobs(entries);
|
|
3652
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3653
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3654
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3655
|
+
const submitted = await Promise.all(
|
|
3656
|
+
plans.map(async (plan, planIndex) => {
|
|
3657
|
+
try {
|
|
3658
|
+
return await submitBatchJobPlan(config, {
|
|
3659
|
+
plan,
|
|
3660
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3661
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3662
|
+
jobCount,
|
|
3663
|
+
onProgress: options.onProgress
|
|
3664
|
+
});
|
|
3665
|
+
} catch (error) {
|
|
3666
|
+
for (const entry of plan.entries) {
|
|
3667
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3668
|
+
}
|
|
3669
|
+
return void 0;
|
|
3670
|
+
}
|
|
3671
|
+
})
|
|
3672
|
+
);
|
|
3673
|
+
const jobsToPoll = [
|
|
3674
|
+
...pending.jobs,
|
|
3675
|
+
...submitted.filter((row) => row !== void 0)
|
|
3676
|
+
];
|
|
3677
|
+
if (jobsToPoll.length > 0) {
|
|
3678
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3679
|
+
jobCount,
|
|
3680
|
+
onProgress: options.onProgress,
|
|
3681
|
+
onResult: collector.emit
|
|
3682
|
+
});
|
|
3683
|
+
}
|
|
3684
|
+
}
|
|
3685
|
+
const results = collector.assemble();
|
|
3686
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3687
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3688
|
+
return results;
|
|
3689
|
+
}
|
|
3690
|
+
function countPendingItems(config, jobs) {
|
|
3691
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3692
|
+
const db = openStore(config, "readonly");
|
|
3693
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3694
|
+
for (const job of jobs) {
|
|
3695
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3696
|
+
}
|
|
3697
|
+
db.close();
|
|
3698
|
+
return counts;
|
|
3699
|
+
}
|
|
3700
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3701
|
+
jobs.forEach((job, jobIndex) => {
|
|
3702
|
+
onProgress?.({
|
|
3703
|
+
type: "batch-submitted",
|
|
3704
|
+
name: job.job_name,
|
|
3705
|
+
count: counts.get(job.id) ?? 0,
|
|
3706
|
+
model: job.display_model,
|
|
3707
|
+
jobIndex,
|
|
3708
|
+
jobCount,
|
|
3709
|
+
resumed: true,
|
|
3710
|
+
createdAt: job.created_at
|
|
3711
|
+
});
|
|
3712
|
+
});
|
|
3713
|
+
}
|
|
3714
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3715
|
+
const startedAt = Date.now();
|
|
3716
|
+
const pending = readPendingBatchWork(config);
|
|
3717
|
+
if (pending.jobs.length === 0) return null;
|
|
3718
|
+
options.onProgress?.({
|
|
3719
|
+
type: "start",
|
|
3720
|
+
total: pending.pendingItems.length,
|
|
3721
|
+
concurrency: 1,
|
|
3722
|
+
dryRun: false,
|
|
3723
|
+
mode: "batch"
|
|
2887
3724
|
});
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
3725
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3726
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3727
|
+
const results = [];
|
|
3728
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3729
|
+
onProgress: options.onProgress,
|
|
3730
|
+
onResult: (result) => {
|
|
3731
|
+
results.push(result);
|
|
3732
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3733
|
+
}
|
|
2896
3734
|
});
|
|
2897
3735
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2898
3736
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3015,6 +3853,7 @@ exports.isRoutableType = isRoutableType;
|
|
|
3015
3853
|
exports.loadConfigSync = loadConfigSync;
|
|
3016
3854
|
exports.resolveConfig = resolveConfig;
|
|
3017
3855
|
exports.resolveLocalesFromPreset = resolveLocalesFromPreset;
|
|
3856
|
+
exports.resumeTranslationJobs = resumeTranslationJobs;
|
|
3018
3857
|
exports.serializeMdx = serializeMdx;
|
|
3019
3858
|
exports.translatePage = translatePage;
|
|
3020
3859
|
exports.translateWorklist = translateWorklist;
|