scribe-cms 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1207 -204
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1208 -205
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +969 -166
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +970 -168
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +136 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +136 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +18 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -257,6 +257,7 @@ function resolveConfig(input, baseDir) {
|
|
|
257
257
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
258
258
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
259
259
|
}
|
|
260
|
+
const localeFallbacks = raw.localeFallbacks === false ? {} : deriveLocaleFallbacks(raw.locales, defaultLocale);
|
|
260
261
|
const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
261
262
|
const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
|
|
262
263
|
const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -286,6 +287,7 @@ function resolveConfig(input, baseDir) {
|
|
|
286
287
|
defaultLocale,
|
|
287
288
|
localeRouting,
|
|
288
289
|
localePresets: raw.localePresets,
|
|
290
|
+
localeFallbacks,
|
|
289
291
|
translate: raw.translate,
|
|
290
292
|
types
|
|
291
293
|
};
|
|
@@ -295,8 +297,33 @@ function resolveConfig(input, baseDir) {
|
|
|
295
297
|
});
|
|
296
298
|
return config;
|
|
297
299
|
}
|
|
300
|
+
function deriveLocaleFallbacks(locales, defaultLocale) {
|
|
301
|
+
const byLowercase = new Map(locales.map((l) => [l.toLowerCase(), l]));
|
|
302
|
+
const fallbacks = {};
|
|
303
|
+
for (const locale of locales) {
|
|
304
|
+
const subtags = locale.split("-");
|
|
305
|
+
if (subtags.length < 2) continue;
|
|
306
|
+
const chain = [];
|
|
307
|
+
for (let end = subtags.length - 1; end >= 1; end--) {
|
|
308
|
+
const prefix = byLowercase.get(subtags.slice(0, end).join("-").toLowerCase());
|
|
309
|
+
if (prefix && prefix !== locale && prefix !== defaultLocale) {
|
|
310
|
+
chain.push(prefix);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
if (chain.length > 0) {
|
|
314
|
+
fallbacks[locale] = chain;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return fallbacks;
|
|
318
|
+
}
|
|
298
319
|
|
|
299
320
|
// src/core/introspect-schema.ts
|
|
321
|
+
function getArrayElement(schema) {
|
|
322
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
323
|
+
return schema.element;
|
|
324
|
+
}
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
300
327
|
function introspectSchema(schema, prefix = []) {
|
|
301
328
|
const relation = getRelationTarget(schema);
|
|
302
329
|
if (relation && prefix.length > 0) {
|
|
@@ -310,7 +337,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
310
337
|
}
|
|
311
338
|
];
|
|
312
339
|
}
|
|
313
|
-
|
|
340
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
341
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
342
|
+
}
|
|
343
|
+
const base = peelOptionalWrappers(schema);
|
|
314
344
|
if (base instanceof Object && "shape" in base) {
|
|
315
345
|
const shape = base.shape;
|
|
316
346
|
const fields = [];
|
|
@@ -319,9 +349,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
319
349
|
}
|
|
320
350
|
return fields;
|
|
321
351
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
const elementBase =
|
|
352
|
+
const element = getArrayElement(base);
|
|
353
|
+
if (element) {
|
|
354
|
+
const elementBase = peelOptionalWrappers(element);
|
|
325
355
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
326
356
|
const shape = elementBase.shape;
|
|
327
357
|
const fields = [];
|
|
@@ -333,15 +363,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
333
363
|
}
|
|
334
364
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
335
365
|
}
|
|
336
|
-
function
|
|
337
|
-
const
|
|
366
|
+
function buildPathTrie(paths) {
|
|
367
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
338
368
|
for (const path13 of paths) {
|
|
339
|
-
|
|
369
|
+
let node = root;
|
|
370
|
+
for (const segment of path13) {
|
|
371
|
+
let child = node.children.get(segment);
|
|
372
|
+
if (!child) {
|
|
373
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
374
|
+
node.children.set(segment, child);
|
|
375
|
+
}
|
|
376
|
+
node = child;
|
|
377
|
+
}
|
|
378
|
+
node.leaf = true;
|
|
379
|
+
}
|
|
380
|
+
return root;
|
|
381
|
+
}
|
|
382
|
+
function extractNode(data, trie) {
|
|
383
|
+
if (trie.leaf) return data;
|
|
384
|
+
const star = trie.children.get("*");
|
|
385
|
+
if (star) {
|
|
386
|
+
if (!Array.isArray(data)) return void 0;
|
|
387
|
+
return data.map(
|
|
388
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
392
|
+
const record = data;
|
|
393
|
+
const out = {};
|
|
394
|
+
let any = false;
|
|
395
|
+
for (const [key, child] of trie.children) {
|
|
396
|
+
const value = extractNode(record[key], child);
|
|
340
397
|
if (value !== void 0) {
|
|
341
|
-
|
|
398
|
+
out[key] = value;
|
|
399
|
+
any = true;
|
|
342
400
|
}
|
|
343
401
|
}
|
|
344
|
-
return out;
|
|
402
|
+
return any ? out : void 0;
|
|
403
|
+
}
|
|
404
|
+
function extractByPaths(data, paths) {
|
|
405
|
+
if (paths.length === 0) return {};
|
|
406
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
407
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
345
408
|
}
|
|
346
409
|
function pickTranslatable(data, schema) {
|
|
347
410
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -356,32 +419,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
356
419
|
const translatable = pickTranslatable(localeData, schema);
|
|
357
420
|
return deepMerge(structural, translatable);
|
|
358
421
|
}
|
|
359
|
-
function getAtPath(obj, path13) {
|
|
360
|
-
let current = obj;
|
|
361
|
-
for (const segment of path13) {
|
|
362
|
-
if (segment === "*") {
|
|
363
|
-
if (!Array.isArray(current)) return void 0;
|
|
364
|
-
return current.map(
|
|
365
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path13.slice(path13.indexOf("*") + 1)) : void 0
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
369
|
-
current = current[segment];
|
|
370
|
-
}
|
|
371
|
-
return current;
|
|
372
|
-
}
|
|
373
|
-
function setAtPath(obj, path13, value) {
|
|
374
|
-
if (path13.length === 0) return;
|
|
375
|
-
if (path13.length === 1) {
|
|
376
|
-
obj[path13[0]] = value;
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
const [head, ...rest] = path13;
|
|
380
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
381
|
-
obj[head] = {};
|
|
382
|
-
}
|
|
383
|
-
setAtPath(obj[head], rest, value);
|
|
384
|
-
}
|
|
385
422
|
function deepMerge(base, overlay) {
|
|
386
423
|
const out = { ...base };
|
|
387
424
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -551,7 +588,7 @@ function seoFieldsFromEn(enDoc) {
|
|
|
551
588
|
canonicalPathOverride: enDoc.canonicalPathOverride
|
|
552
589
|
};
|
|
553
590
|
}
|
|
554
|
-
var SCHEMA_VERSION =
|
|
591
|
+
var SCHEMA_VERSION = 5;
|
|
555
592
|
var MIGRATIONS = [
|
|
556
593
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
557
594
|
key TEXT PRIMARY KEY,
|
|
@@ -599,7 +636,30 @@ var MIGRATIONS = [
|
|
|
599
636
|
UNIQUE (content_type, en_slug, en_hash)
|
|
600
637
|
)`,
|
|
601
638
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
602
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
639
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
640
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
641
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
642
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
643
|
+
model TEXT NOT NULL,
|
|
644
|
+
display_model TEXT NOT NULL,
|
|
645
|
+
created_at TEXT NOT NULL,
|
|
646
|
+
state TEXT NOT NULL,
|
|
647
|
+
completed_at TEXT
|
|
648
|
+
)`,
|
|
649
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
650
|
+
job_id INTEGER NOT NULL,
|
|
651
|
+
request_index INTEGER NOT NULL,
|
|
652
|
+
content_type TEXT NOT NULL,
|
|
653
|
+
en_slug TEXT NOT NULL,
|
|
654
|
+
locale TEXT NOT NULL,
|
|
655
|
+
en_hash TEXT NOT NULL,
|
|
656
|
+
snapshot_id INTEGER NOT NULL,
|
|
657
|
+
status TEXT NOT NULL,
|
|
658
|
+
error TEXT,
|
|
659
|
+
PRIMARY KEY (job_id, request_index)
|
|
660
|
+
)`,
|
|
661
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
662
|
+
ON translation_batch_items(status)`
|
|
603
663
|
];
|
|
604
664
|
function resolveStorePath(config) {
|
|
605
665
|
return config.storePath;
|
|
@@ -622,6 +682,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
622
682
|
}
|
|
623
683
|
}
|
|
624
684
|
function readSchemaVersion(db) {
|
|
685
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
686
|
+
if (!metaTable) return 0;
|
|
625
687
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
626
688
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
627
689
|
}
|
|
@@ -660,6 +722,9 @@ function getOrCreateEnSnapshot(db, input) {
|
|
|
660
722
|
).get(input.contentType, input.enSlug, input.enHash);
|
|
661
723
|
return row.id;
|
|
662
724
|
}
|
|
725
|
+
function getEnSnapshot(db, snapshotId) {
|
|
726
|
+
return db.prepare(`SELECT * FROM en_snapshots WHERE id = ?`).get(snapshotId);
|
|
727
|
+
}
|
|
663
728
|
function upsertTranslation(db, input) {
|
|
664
729
|
db.prepare(
|
|
665
730
|
`INSERT INTO translations (
|
|
@@ -740,7 +805,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
740
805
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
741
806
|
const attrStart = i;
|
|
742
807
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
743
|
-
body.slice(attrStart, i);
|
|
808
|
+
const attrName = body.slice(attrStart, i);
|
|
809
|
+
if (!attrName) break;
|
|
744
810
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
745
811
|
if (body[i] !== "=") {
|
|
746
812
|
tagOut += body.slice(attrStart, i);
|
|
@@ -749,10 +815,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
749
815
|
tagOut += body.slice(attrStart, i);
|
|
750
816
|
tagOut += "=";
|
|
751
817
|
i += 1;
|
|
818
|
+
const wsStart = i;
|
|
752
819
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
820
|
+
tagOut += body.slice(wsStart, i);
|
|
753
821
|
const quote = body[i];
|
|
754
822
|
if (quote !== '"' && quote !== "'") {
|
|
755
|
-
|
|
823
|
+
break;
|
|
756
824
|
}
|
|
757
825
|
if (quote === "'") {
|
|
758
826
|
const valStart2 = i + 1;
|
|
@@ -1072,7 +1140,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1072
1140
|
}
|
|
1073
1141
|
|
|
1074
1142
|
// src/i18n/resolve-document.ts
|
|
1075
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1143
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1076
1144
|
const urlBuilder = createUrlBuilder({
|
|
1077
1145
|
locales: [defaultLocale, locale],
|
|
1078
1146
|
defaultLocale,
|
|
@@ -1086,15 +1154,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1086
1154
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1087
1155
|
const found = docIdx.bySlug.get(slug);
|
|
1088
1156
|
if (!found) continue;
|
|
1089
|
-
const
|
|
1090
|
-
|
|
1157
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1158
|
+
let target;
|
|
1159
|
+
let targetLocale = locale;
|
|
1160
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1161
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1162
|
+
if (cand) {
|
|
1163
|
+
target = cand;
|
|
1164
|
+
targetLocale = candidateLocale;
|
|
1165
|
+
break;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
if (target) {
|
|
1169
|
+
if (target.slug === slug) {
|
|
1170
|
+
return { document: target, actualLocale: targetLocale };
|
|
1171
|
+
}
|
|
1091
1172
|
if (!type.path) {
|
|
1092
1173
|
return { document: null, actualLocale: locale };
|
|
1093
1174
|
}
|
|
1094
1175
|
return {
|
|
1095
1176
|
document: null,
|
|
1096
1177
|
actualLocale: locale,
|
|
1097
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1178
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1098
1179
|
};
|
|
1099
1180
|
}
|
|
1100
1181
|
if (docLocale === defaultLocale) {
|
|
@@ -1125,13 +1206,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1125
1206
|
}
|
|
1126
1207
|
return { document: null, actualLocale: locale };
|
|
1127
1208
|
}
|
|
1128
|
-
function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1129
|
-
if (sourceLocale === targetLocale) return document2.slug;
|
|
1130
|
-
const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
|
|
1131
|
-
if (!englishSlug) return null;
|
|
1132
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1133
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1134
|
-
}
|
|
1135
1209
|
|
|
1136
1210
|
// src/create-project.ts
|
|
1137
1211
|
function comparatorFor(orderBy) {
|
|
@@ -1184,7 +1258,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1184
1258
|
config.defaultLocale,
|
|
1185
1259
|
load(),
|
|
1186
1260
|
type,
|
|
1187
|
-
config.localeRouting
|
|
1261
|
+
config.localeRouting,
|
|
1262
|
+
config.localeFallbacks[locale] ?? []
|
|
1188
1263
|
);
|
|
1189
1264
|
if (result.document && type.path) {
|
|
1190
1265
|
return {
|
|
@@ -1206,8 +1281,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1206
1281
|
const params = [];
|
|
1207
1282
|
for (const locale of options.locales ?? config.locales) {
|
|
1208
1283
|
const localeIdx = all.get(locale);
|
|
1284
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1209
1285
|
for (const doc of enIdx.bySlug.values()) {
|
|
1210
|
-
|
|
1286
|
+
let slug;
|
|
1287
|
+
if (locale === config.defaultLocale) {
|
|
1288
|
+
slug = doc.slug;
|
|
1289
|
+
} else {
|
|
1290
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1291
|
+
}
|
|
1211
1292
|
params.push({ locale, slug });
|
|
1212
1293
|
}
|
|
1213
1294
|
}
|
|
@@ -2026,7 +2107,7 @@ function validateTypeRedirects(project) {
|
|
|
2026
2107
|
}
|
|
2027
2108
|
return issues;
|
|
2028
2109
|
}
|
|
2029
|
-
function
|
|
2110
|
+
function getAtPath(obj, fieldPath) {
|
|
2030
2111
|
let current = obj;
|
|
2031
2112
|
for (const segment of fieldPath) {
|
|
2032
2113
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -2071,7 +2152,7 @@ function validateRelations(project) {
|
|
|
2071
2152
|
});
|
|
2072
2153
|
continue;
|
|
2073
2154
|
}
|
|
2074
|
-
const value =
|
|
2155
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
2075
2156
|
if (value === void 0 || value === null || value === "") continue;
|
|
2076
2157
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
2077
2158
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2438,6 +2519,59 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2438
2519
|
return config.locales.filter((l) => l !== config.defaultLocale);
|
|
2439
2520
|
}
|
|
2440
2521
|
|
|
2522
|
+
// src/storage/batch-jobs.ts
|
|
2523
|
+
function insertBatchJob(db, input) {
|
|
2524
|
+
const info = db.prepare(
|
|
2525
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2526
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2527
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2528
|
+
return Number(info.lastInsertRowid);
|
|
2529
|
+
}
|
|
2530
|
+
function insertBatchItems(db, jobId, items) {
|
|
2531
|
+
const stmt = db.prepare(
|
|
2532
|
+
`INSERT INTO translation_batch_items (
|
|
2533
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2534
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2535
|
+
);
|
|
2536
|
+
for (const item of items) {
|
|
2537
|
+
stmt.run(
|
|
2538
|
+
jobId,
|
|
2539
|
+
item.requestIndex,
|
|
2540
|
+
item.contentType,
|
|
2541
|
+
item.enSlug,
|
|
2542
|
+
item.locale,
|
|
2543
|
+
item.enHash,
|
|
2544
|
+
item.snapshotId
|
|
2545
|
+
);
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
function listPendingBatchJobs(db) {
|
|
2549
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2550
|
+
}
|
|
2551
|
+
function listBatchItems(db, jobId) {
|
|
2552
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2553
|
+
}
|
|
2554
|
+
function listPendingBatchItems(db) {
|
|
2555
|
+
return db.prepare(
|
|
2556
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2557
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2558
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2559
|
+
ORDER BY i.job_id, i.request_index`
|
|
2560
|
+
).all();
|
|
2561
|
+
}
|
|
2562
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2563
|
+
const info = db.prepare(
|
|
2564
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2565
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2566
|
+
).run(state, completedAt, jobId);
|
|
2567
|
+
return info.changes > 0;
|
|
2568
|
+
}
|
|
2569
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2570
|
+
db.prepare(
|
|
2571
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2572
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2441
2575
|
// src/history/record-snapshot.ts
|
|
2442
2576
|
function recordEnSnapshot(config, input, db) {
|
|
2443
2577
|
const ownDb = db ?? openStore(config, "readwrite");
|
|
@@ -2452,8 +2586,102 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2452
2586
|
if (!db) ownDb.close();
|
|
2453
2587
|
return id;
|
|
2454
2588
|
}
|
|
2589
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2590
|
+
function statusFromError(error) {
|
|
2591
|
+
if (error instanceof genai.ApiError && typeof error.status === "number") {
|
|
2592
|
+
return error.status;
|
|
2593
|
+
}
|
|
2594
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2595
|
+
const status = error.status;
|
|
2596
|
+
if (typeof status === "number") return status;
|
|
2597
|
+
}
|
|
2598
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2599
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2600
|
+
if (match) return Number(match[1]);
|
|
2601
|
+
return void 0;
|
|
2602
|
+
}
|
|
2603
|
+
function isNetworkError(error) {
|
|
2604
|
+
const err = error;
|
|
2605
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2606
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2607
|
+
return true;
|
|
2608
|
+
}
|
|
2609
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2610
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2611
|
+
}
|
|
2612
|
+
function isTransientError(error) {
|
|
2613
|
+
const status = statusFromError(error);
|
|
2614
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2615
|
+
return isNetworkError(error);
|
|
2616
|
+
}
|
|
2617
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2618
|
+
async function withRetry(fn, options = {}) {
|
|
2619
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2620
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2621
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2622
|
+
const random = options.random ?? Math.random;
|
|
2623
|
+
let lastError;
|
|
2624
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2625
|
+
try {
|
|
2626
|
+
return await fn();
|
|
2627
|
+
} catch (error) {
|
|
2628
|
+
lastError = error;
|
|
2629
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2630
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2631
|
+
const delay = backoff + random() * backoff;
|
|
2632
|
+
await sleep2(delay);
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
throw lastError;
|
|
2636
|
+
}
|
|
2455
2637
|
|
|
2456
|
-
// src/translate/gemini-
|
|
2638
|
+
// src/translate/gemini-batch.ts
|
|
2639
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2640
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2641
|
+
"SUCCEEDED",
|
|
2642
|
+
"FAILED",
|
|
2643
|
+
"CANCELLED",
|
|
2644
|
+
"EXPIRED",
|
|
2645
|
+
"PARTIALLY_SUCCEEDED"
|
|
2646
|
+
]);
|
|
2647
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2648
|
+
function normalizeBatchState(state) {
|
|
2649
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2650
|
+
}
|
|
2651
|
+
function isTerminalBatchState(state) {
|
|
2652
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2653
|
+
}
|
|
2654
|
+
function isSuccessfulBatchState(state) {
|
|
2655
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2656
|
+
}
|
|
2657
|
+
function makeClient(apiKey) {
|
|
2658
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2659
|
+
if (!key) {
|
|
2660
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2661
|
+
}
|
|
2662
|
+
return new genai.GoogleGenAI({ apiKey: key });
|
|
2663
|
+
}
|
|
2664
|
+
function textFromBatchResponse(response) {
|
|
2665
|
+
if (typeof response.text === "string") return response.text;
|
|
2666
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2667
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2668
|
+
}
|
|
2669
|
+
async function createGeminiBatchJob(input) {
|
|
2670
|
+
const ai = makeClient(input.apiKey);
|
|
2671
|
+
const job = await withRetry(
|
|
2672
|
+
() => ai.batches.create({
|
|
2673
|
+
model: input.model,
|
|
2674
|
+
src: input.requests,
|
|
2675
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2676
|
+
})
|
|
2677
|
+
);
|
|
2678
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2679
|
+
return job;
|
|
2680
|
+
}
|
|
2681
|
+
async function getGeminiBatchJob(input) {
|
|
2682
|
+
const ai = makeClient(input.apiKey);
|
|
2683
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2684
|
+
}
|
|
2457
2685
|
var GEMINI_MODEL_IDS = {
|
|
2458
2686
|
"gemini-2.5-pro": "gemini-2.5-pro",
|
|
2459
2687
|
"gemini-3.1-pro": "gemini-3.1-pro-preview"
|
|
@@ -2473,6 +2701,12 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2473
2701
|
if (alias) return alias[0];
|
|
2474
2702
|
return name;
|
|
2475
2703
|
}
|
|
2704
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2705
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2706
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
|
|
2707
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2708
|
+
return void 0;
|
|
2709
|
+
}
|
|
2476
2710
|
|
|
2477
2711
|
// src/translate/gemini-client.ts
|
|
2478
2712
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
@@ -2486,11 +2720,36 @@ function extractJson(text) {
|
|
|
2486
2720
|
return trimmed;
|
|
2487
2721
|
}
|
|
2488
2722
|
function parseGeminiResponse(text) {
|
|
2723
|
+
let parsed;
|
|
2489
2724
|
try {
|
|
2490
|
-
|
|
2725
|
+
parsed = JSON.parse(text.trim());
|
|
2491
2726
|
} catch {
|
|
2492
|
-
|
|
2727
|
+
parsed = JSON.parse(extractJson(text));
|
|
2728
|
+
}
|
|
2729
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2730
|
+
parsed.frontmatter = {};
|
|
2493
2731
|
}
|
|
2732
|
+
return parsed;
|
|
2733
|
+
}
|
|
2734
|
+
function buildGeminiRequestConfig(input) {
|
|
2735
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2736
|
+
return {
|
|
2737
|
+
responseMimeType: "application/json",
|
|
2738
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2739
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2740
|
+
};
|
|
2741
|
+
}
|
|
2742
|
+
function usageFromResponse(response) {
|
|
2743
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2744
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2745
|
+
return {
|
|
2746
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2747
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2748
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2749
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2750
|
+
thoughtsTokens,
|
|
2751
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2752
|
+
};
|
|
2494
2753
|
}
|
|
2495
2754
|
async function translatePageWithGemini(input) {
|
|
2496
2755
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2502,26 +2761,22 @@ async function translatePageWithGemini(input) {
|
|
|
2502
2761
|
);
|
|
2503
2762
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2504
2763
|
const ai = new genai.GoogleGenAI({ apiKey });
|
|
2505
|
-
const response = await
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2764
|
+
const response = await withRetry(
|
|
2765
|
+
() => ai.models.generateContent({
|
|
2766
|
+
model: apiModel,
|
|
2767
|
+
contents: input.prompt,
|
|
2768
|
+
config: buildGeminiRequestConfig({
|
|
2769
|
+
apiModelId: apiModel,
|
|
2770
|
+
responseSchema: input.responseSchema
|
|
2771
|
+
})
|
|
2772
|
+
})
|
|
2773
|
+
);
|
|
2513
2774
|
const raw = response.text ?? "";
|
|
2514
2775
|
const parsed = parseGeminiResponse(raw);
|
|
2515
2776
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2516
2777
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2517
2778
|
}
|
|
2518
|
-
|
|
2519
|
-
const usage = {
|
|
2520
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2521
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2522
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2523
|
-
};
|
|
2524
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2779
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2525
2780
|
}
|
|
2526
2781
|
|
|
2527
2782
|
// src/translate/gemini-pricing.ts
|
|
@@ -2541,12 +2796,14 @@ function resolveModelPricing(model) {
|
|
|
2541
2796
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2542
2797
|
return match?.[1];
|
|
2543
2798
|
}
|
|
2544
|
-
|
|
2799
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2800
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2545
2801
|
const pricing = resolveModelPricing(model);
|
|
2546
2802
|
if (!pricing) return void 0;
|
|
2547
2803
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2548
2804
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2549
|
-
|
|
2805
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2806
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2550
2807
|
}
|
|
2551
2808
|
|
|
2552
2809
|
// src/translate/prompts/translation-prompt.ts
|
|
@@ -2562,6 +2819,7 @@ var LOCALE_NAMES = {
|
|
|
2562
2819
|
"zh-CN": "Simplified Chinese",
|
|
2563
2820
|
"zh-TW": "Traditional Chinese",
|
|
2564
2821
|
pt: "Portuguese",
|
|
2822
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2565
2823
|
ja: "Japanese",
|
|
2566
2824
|
ru: "Russian",
|
|
2567
2825
|
it: "Italian",
|
|
@@ -2569,29 +2827,29 @@ var LOCALE_NAMES = {
|
|
|
2569
2827
|
};
|
|
2570
2828
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2571
2829
|
return [
|
|
2572
|
-
`Localize the content
|
|
2830
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2573
2831
|
"Do not translate word-for-word.",
|
|
2574
2832
|
"Preserve the source tone and brand voice.",
|
|
2575
2833
|
"Write as if a native speaker authored it for the target market."
|
|
2576
2834
|
].join(" ");
|
|
2577
2835
|
}
|
|
2836
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2837
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2838
|
+
if (!hasFrontmatter) {
|
|
2839
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2840
|
+
}
|
|
2841
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2842
|
+
}
|
|
2578
2843
|
function buildPageTranslationPrompt(input) {
|
|
2579
2844
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2580
|
-
const
|
|
2581
|
-
const
|
|
2582
|
-
|
|
2845
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2846
|
+
const prefix = [
|
|
2847
|
+
TASK_FRAMING,
|
|
2583
2848
|
"",
|
|
2584
2849
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2585
2850
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2586
2851
|
"## Rules",
|
|
2587
2852
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2588
|
-
...input.slugStrategy === "localized" ? [
|
|
2589
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2590
|
-
] : [],
|
|
2591
|
-
"",
|
|
2592
|
-
"## Output format",
|
|
2593
|
-
"Return ONLY valid JSON with keys:",
|
|
2594
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2595
2853
|
"",
|
|
2596
2854
|
"## EN translatable frontmatter (JSON)",
|
|
2597
2855
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2599,7 +2857,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2599
2857
|
"## EN body (MDX)",
|
|
2600
2858
|
input.enBody
|
|
2601
2859
|
];
|
|
2602
|
-
|
|
2860
|
+
const suffix = [
|
|
2861
|
+
"",
|
|
2862
|
+
"## Target language",
|
|
2863
|
+
localizationPrompt,
|
|
2864
|
+
...input.slugStrategy === "localized" ? [
|
|
2865
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2866
|
+
] : [],
|
|
2867
|
+
"",
|
|
2868
|
+
"## Output format",
|
|
2869
|
+
"Return ONLY valid JSON with keys:",
|
|
2870
|
+
buildOutputFormatLine(
|
|
2871
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2872
|
+
input.slugStrategy
|
|
2873
|
+
)
|
|
2874
|
+
];
|
|
2875
|
+
return [...prefix, ...suffix].join("\n");
|
|
2603
2876
|
}
|
|
2604
2877
|
function getObjectShape(schema) {
|
|
2605
2878
|
const base = peelOptionalWrappers(schema);
|
|
@@ -2647,9 +2920,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2647
2920
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2648
2921
|
try {
|
|
2649
2922
|
const translatable = buildTranslatableSubschema(schema);
|
|
2650
|
-
if (!translatable) return null;
|
|
2651
2923
|
const responseShape = {
|
|
2652
|
-
frontmatter: translatable,
|
|
2924
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2653
2925
|
body: zod.z.string()
|
|
2654
2926
|
};
|
|
2655
2927
|
if (slugStrategy === "localized") {
|
|
@@ -2728,7 +3000,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2728
3000
|
return { ok: true, frontmatter };
|
|
2729
3001
|
}
|
|
2730
3002
|
|
|
2731
|
-
// src/translate/
|
|
3003
|
+
// src/translate/translate-core.ts
|
|
3004
|
+
function translationItemKey(item) {
|
|
3005
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
3006
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
3007
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
3008
|
+
}
|
|
2732
3009
|
function summarizeResults(results, durationMs) {
|
|
2733
3010
|
return results.reduce(
|
|
2734
3011
|
(totals, result) => {
|
|
@@ -2764,17 +3041,6 @@ function formatTranslateError(error) {
|
|
|
2764
3041
|
}
|
|
2765
3042
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2766
3043
|
}
|
|
2767
|
-
async function runPool(items, concurrency, worker) {
|
|
2768
|
-
if (items.length === 0) return;
|
|
2769
|
-
let nextIndex = 0;
|
|
2770
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2771
|
-
while (nextIndex < items.length) {
|
|
2772
|
-
const index = nextIndex++;
|
|
2773
|
-
await worker(items[index], index);
|
|
2774
|
-
}
|
|
2775
|
-
});
|
|
2776
|
-
await Promise.all(runners);
|
|
2777
|
-
}
|
|
2778
3044
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2779
3045
|
const fm = enDoc.frontmatter;
|
|
2780
3046
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2783,73 +3049,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2783
3049
|
}
|
|
2784
3050
|
return enSlug;
|
|
2785
3051
|
}
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
const base = {
|
|
3052
|
+
function baseForItem(item) {
|
|
3053
|
+
return {
|
|
2789
3054
|
contentType: item.contentType,
|
|
2790
3055
|
enSlug: item.enSlug,
|
|
2791
3056
|
locale: item.locale
|
|
2792
3057
|
};
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
3058
|
+
}
|
|
3059
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
3060
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
3061
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
3062
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
3063
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
3064
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
3065
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
3066
|
+
const db = openStore(config, "readonly");
|
|
3067
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
3068
|
+
db.close();
|
|
3069
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
3070
|
+
return {
|
|
3071
|
+
status: "done",
|
|
3072
|
+
result: {
|
|
3073
|
+
...baseForItem(item),
|
|
2806
3074
|
skipped: true,
|
|
2807
3075
|
reason: "fresh",
|
|
2808
3076
|
durationMs: Date.now() - startedAt
|
|
2809
|
-
}
|
|
2810
|
-
}
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
3077
|
+
}
|
|
3078
|
+
};
|
|
3079
|
+
}
|
|
3080
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
3081
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
3082
|
+
const prompt = buildPageTranslationPrompt({
|
|
3083
|
+
resolved: resolvedTranslate,
|
|
3084
|
+
targetLocale: item.locale,
|
|
3085
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
3086
|
+
translatableFrontmatter: payload.frontmatter,
|
|
3087
|
+
enBody: payload.body,
|
|
3088
|
+
slugStrategy: type.slugStrategy
|
|
3089
|
+
});
|
|
3090
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
3091
|
+
return {
|
|
3092
|
+
status: "ready",
|
|
3093
|
+
prepared: {
|
|
3094
|
+
item,
|
|
3095
|
+
type,
|
|
3096
|
+
enDoc,
|
|
3097
|
+
payload,
|
|
3098
|
+
currentEnHash,
|
|
3099
|
+
existingSlug: existing?.slug,
|
|
2832
3100
|
model,
|
|
3101
|
+
prompt,
|
|
2833
3102
|
responseSchema: responseSchema ?? void 0
|
|
2834
|
-
}
|
|
2835
|
-
|
|
3103
|
+
}
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
3107
|
+
const { item, type, enDoc, payload } = prepared;
|
|
3108
|
+
const base = baseForItem(item);
|
|
3109
|
+
try {
|
|
3110
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2836
3111
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2837
3112
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2838
3113
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2839
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
3114
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2840
3115
|
if (!validated.ok) {
|
|
2841
3116
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2842
3117
|
}
|
|
2843
3118
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2844
|
-
|
|
3119
|
+
output.parsed.body
|
|
2845
3120
|
);
|
|
2846
3121
|
const writeDb = openStore(config, "readwrite");
|
|
2847
|
-
const snapshotId = recordEnSnapshot(
|
|
3122
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2848
3123
|
config,
|
|
2849
3124
|
{
|
|
2850
3125
|
contentType: type.id,
|
|
2851
3126
|
enSlug: item.enSlug,
|
|
2852
|
-
enHash: currentEnHash,
|
|
3127
|
+
enHash: prepared.currentEnHash,
|
|
2853
3128
|
frontmatter: payload.frontmatter,
|
|
2854
3129
|
body: payload.body
|
|
2855
3130
|
},
|
|
@@ -2862,24 +3137,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2862
3137
|
slug,
|
|
2863
3138
|
frontmatter: validated.frontmatter,
|
|
2864
3139
|
body: translatedBody,
|
|
2865
|
-
enHash: currentEnHash,
|
|
3140
|
+
enHash: prepared.currentEnHash,
|
|
2866
3141
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2867
|
-
model:
|
|
3142
|
+
model: output.model,
|
|
2868
3143
|
snapshotId
|
|
2869
3144
|
});
|
|
2870
3145
|
writeDb.close();
|
|
2871
3146
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2872
|
-
normalizeGeminiDisplayName(
|
|
2873
|
-
|
|
2874
|
-
|
|
3147
|
+
normalizeGeminiDisplayName(output.model),
|
|
3148
|
+
output.usage.inputTokens,
|
|
3149
|
+
output.usage.outputTokens,
|
|
3150
|
+
options.costMode
|
|
2875
3151
|
);
|
|
2876
3152
|
return {
|
|
2877
3153
|
...base,
|
|
2878
3154
|
skipped: false,
|
|
2879
|
-
model:
|
|
2880
|
-
usage:
|
|
3155
|
+
model: output.model,
|
|
3156
|
+
usage: output.usage,
|
|
2881
3157
|
estimatedCostUsd,
|
|
2882
|
-
durationMs: Date.now() - startedAt,
|
|
3158
|
+
durationMs: Date.now() - options.startedAt,
|
|
2883
3159
|
slugAdjusted,
|
|
2884
3160
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2885
3161
|
};
|
|
@@ -2889,33 +3165,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2889
3165
|
skipped: false,
|
|
2890
3166
|
failed: true,
|
|
2891
3167
|
error: formatTranslateError(error),
|
|
3168
|
+
durationMs: Date.now() - options.startedAt
|
|
3169
|
+
};
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
function displayModelFor(prepared) {
|
|
3173
|
+
return normalizeGeminiDisplayName(
|
|
3174
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
3175
|
+
);
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
// src/translate/batch-worklist.ts
|
|
3179
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
3180
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
3181
|
+
var INITIAL_POLL_MS = 5e3;
|
|
3182
|
+
var MAX_POLL_MS = 3e4;
|
|
3183
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
3184
|
+
function sleep(ms) {
|
|
3185
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3186
|
+
}
|
|
3187
|
+
function planBatchJobs(entries) {
|
|
3188
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
3189
|
+
for (const entry of entries) {
|
|
3190
|
+
const group = byModel.get(entry.apiModel);
|
|
3191
|
+
if (group) group.push(entry);
|
|
3192
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
3193
|
+
}
|
|
3194
|
+
const plans = [];
|
|
3195
|
+
for (const [apiModel, group] of byModel) {
|
|
3196
|
+
let current = [];
|
|
3197
|
+
let currentBytes = 0;
|
|
3198
|
+
for (const entry of group) {
|
|
3199
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
3200
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
3201
|
+
plans.push({ apiModel, entries: current });
|
|
3202
|
+
current = [];
|
|
3203
|
+
currentBytes = 0;
|
|
3204
|
+
}
|
|
3205
|
+
current.push(entry);
|
|
3206
|
+
currentBytes += size;
|
|
3207
|
+
}
|
|
3208
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
3209
|
+
}
|
|
3210
|
+
return plans;
|
|
3211
|
+
}
|
|
3212
|
+
function readPendingBatchWork(config) {
|
|
3213
|
+
const db = openStore(config, "readwrite");
|
|
3214
|
+
const jobs = listPendingBatchJobs(db);
|
|
3215
|
+
const pendingItems = listPendingBatchItems(db);
|
|
3216
|
+
db.close();
|
|
3217
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
3218
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
3219
|
+
}
|
|
3220
|
+
async function submitBatchJobPlan(config, input) {
|
|
3221
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
3222
|
+
const job = await createGeminiBatchJob({
|
|
3223
|
+
model: plan.apiModel,
|
|
3224
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
3225
|
+
contents: prepared.prompt,
|
|
3226
|
+
config: buildGeminiRequestConfig({
|
|
3227
|
+
apiModelId: plan.apiModel,
|
|
3228
|
+
responseSchema: prepared.responseSchema
|
|
3229
|
+
})
|
|
3230
|
+
})),
|
|
3231
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
3232
|
+
});
|
|
3233
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3234
|
+
const db = openStore(config, "readwrite");
|
|
3235
|
+
const jobId = insertBatchJob(db, {
|
|
3236
|
+
jobName: job.name,
|
|
3237
|
+
model: plan.apiModel,
|
|
3238
|
+
displayModel,
|
|
3239
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3240
|
+
createdAt
|
|
3241
|
+
});
|
|
3242
|
+
insertBatchItems(
|
|
3243
|
+
db,
|
|
3244
|
+
jobId,
|
|
3245
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
3246
|
+
requestIndex,
|
|
3247
|
+
contentType: prepared.item.contentType,
|
|
3248
|
+
enSlug: prepared.item.enSlug,
|
|
3249
|
+
locale: prepared.item.locale,
|
|
3250
|
+
enHash: prepared.currentEnHash,
|
|
3251
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
3252
|
+
// on the EN files still matching (or existing) on disk.
|
|
3253
|
+
snapshotId: recordEnSnapshot(
|
|
3254
|
+
config,
|
|
3255
|
+
{
|
|
3256
|
+
contentType: prepared.item.contentType,
|
|
3257
|
+
enSlug: prepared.item.enSlug,
|
|
3258
|
+
enHash: prepared.currentEnHash,
|
|
3259
|
+
frontmatter: prepared.payload.frontmatter,
|
|
3260
|
+
body: prepared.payload.body
|
|
3261
|
+
},
|
|
3262
|
+
db
|
|
3263
|
+
)
|
|
3264
|
+
}))
|
|
3265
|
+
);
|
|
3266
|
+
const row = {
|
|
3267
|
+
id: jobId,
|
|
3268
|
+
job_name: job.name,
|
|
3269
|
+
model: plan.apiModel,
|
|
3270
|
+
display_model: displayModel,
|
|
3271
|
+
created_at: createdAt,
|
|
3272
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3273
|
+
completed_at: null
|
|
3274
|
+
};
|
|
3275
|
+
db.close();
|
|
3276
|
+
input.onProgress?.({
|
|
3277
|
+
type: "batch-submitted",
|
|
3278
|
+
name: job.name,
|
|
3279
|
+
count: plan.entries.length,
|
|
3280
|
+
model: displayModel,
|
|
3281
|
+
jobIndex,
|
|
3282
|
+
jobCount,
|
|
3283
|
+
createdAt
|
|
3284
|
+
});
|
|
3285
|
+
return row;
|
|
3286
|
+
}
|
|
3287
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3288
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3289
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3290
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3291
|
+
if (!enDoc) {
|
|
3292
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3293
|
+
if (!snapshot) {
|
|
3294
|
+
throw new Error(
|
|
3295
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3296
|
+
);
|
|
3297
|
+
}
|
|
3298
|
+
enDoc = {
|
|
3299
|
+
slug: itemRow.en_slug,
|
|
3300
|
+
enSlug: itemRow.en_slug,
|
|
3301
|
+
locale: config.defaultLocale,
|
|
3302
|
+
noindex: false,
|
|
3303
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3304
|
+
content: snapshot.body
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
const item = {
|
|
3308
|
+
contentType: itemRow.content_type,
|
|
3309
|
+
enSlug: itemRow.en_slug,
|
|
3310
|
+
locale: itemRow.locale,
|
|
3311
|
+
reason: "missing",
|
|
3312
|
+
currentEnHash: itemRow.en_hash
|
|
3313
|
+
};
|
|
3314
|
+
return {
|
|
3315
|
+
item,
|
|
3316
|
+
type,
|
|
3317
|
+
enDoc,
|
|
3318
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3319
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3320
|
+
currentEnHash: itemRow.en_hash,
|
|
3321
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3322
|
+
model: jobRow.display_model,
|
|
3323
|
+
prompt: "",
|
|
3324
|
+
responseSchema: void 0
|
|
3325
|
+
};
|
|
3326
|
+
}
|
|
3327
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3328
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3329
|
+
const db = openStore(config, "readwrite");
|
|
3330
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3331
|
+
db.close();
|
|
3332
|
+
return null;
|
|
3333
|
+
}
|
|
3334
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3335
|
+
const results = [];
|
|
3336
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3337
|
+
const result = {
|
|
3338
|
+
contentType: itemRow.content_type,
|
|
3339
|
+
enSlug: itemRow.en_slug,
|
|
3340
|
+
locale: itemRow.locale,
|
|
3341
|
+
skipped: false,
|
|
3342
|
+
failed: true,
|
|
3343
|
+
error: formatTranslateError(new Error(message)),
|
|
2892
3344
|
durationMs: Date.now() - startedAt
|
|
2893
3345
|
};
|
|
3346
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3347
|
+
results.push(result);
|
|
3348
|
+
onResult?.(result);
|
|
3349
|
+
};
|
|
3350
|
+
if (isSuccessfulBatchState(state)) {
|
|
3351
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3352
|
+
for (const itemRow of pendingItems) {
|
|
3353
|
+
const startedAt = Date.now();
|
|
3354
|
+
const inlined = responses[itemRow.request_index];
|
|
3355
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3356
|
+
failItem(
|
|
3357
|
+
itemRow,
|
|
3358
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3359
|
+
startedAt
|
|
3360
|
+
);
|
|
3361
|
+
continue;
|
|
3362
|
+
}
|
|
3363
|
+
let result;
|
|
3364
|
+
try {
|
|
3365
|
+
const response = inlined.response;
|
|
3366
|
+
const raw = textFromBatchResponse(response);
|
|
3367
|
+
const parsed = parseGeminiResponse(raw);
|
|
3368
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3369
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3370
|
+
}
|
|
3371
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3372
|
+
result = finalizeTranslation(
|
|
3373
|
+
config,
|
|
3374
|
+
prepared,
|
|
3375
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3376
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3377
|
+
);
|
|
3378
|
+
} catch (error) {
|
|
3379
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3380
|
+
continue;
|
|
3381
|
+
}
|
|
3382
|
+
updateBatchItemStatus(
|
|
3383
|
+
db,
|
|
3384
|
+
jobRow.id,
|
|
3385
|
+
itemRow.request_index,
|
|
3386
|
+
result.failed ? "failed" : "done",
|
|
3387
|
+
result.error
|
|
3388
|
+
);
|
|
3389
|
+
results.push(result);
|
|
3390
|
+
onResult?.(result);
|
|
3391
|
+
}
|
|
3392
|
+
} else {
|
|
3393
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3394
|
+
const startedAt = Date.now();
|
|
3395
|
+
for (const itemRow of pendingItems) {
|
|
3396
|
+
failItem(itemRow, message, startedAt);
|
|
3397
|
+
}
|
|
3398
|
+
}
|
|
3399
|
+
db.close();
|
|
3400
|
+
return results;
|
|
3401
|
+
}
|
|
3402
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3403
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3404
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3405
|
+
const loopStartedAt = Date.now();
|
|
3406
|
+
let delay = INITIAL_POLL_MS;
|
|
3407
|
+
let firstRound = true;
|
|
3408
|
+
const elapsedFor = (row) => {
|
|
3409
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3410
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3411
|
+
};
|
|
3412
|
+
while (active.length > 0) {
|
|
3413
|
+
if (!firstRound) {
|
|
3414
|
+
await sleep(delay);
|
|
3415
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3416
|
+
}
|
|
3417
|
+
firstRound = false;
|
|
3418
|
+
for (const tracked of [...active]) {
|
|
3419
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3420
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3421
|
+
options.onProgress?.({
|
|
3422
|
+
type: "batch-polling",
|
|
3423
|
+
name: tracked.row.job_name,
|
|
3424
|
+
state,
|
|
3425
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3426
|
+
jobIndex: tracked.jobIndex,
|
|
3427
|
+
jobCount
|
|
3428
|
+
});
|
|
3429
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3430
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3431
|
+
const results = ingestBatchJob(
|
|
3432
|
+
config,
|
|
3433
|
+
tracked.row,
|
|
3434
|
+
batchJob,
|
|
3435
|
+
options.onResult
|
|
3436
|
+
);
|
|
3437
|
+
if (results === null) {
|
|
3438
|
+
options.onProgress?.({
|
|
3439
|
+
type: "batch-done",
|
|
3440
|
+
name: tracked.row.job_name,
|
|
3441
|
+
state,
|
|
3442
|
+
model: tracked.row.display_model,
|
|
3443
|
+
count: 0,
|
|
3444
|
+
jobIndex: tracked.jobIndex,
|
|
3445
|
+
jobCount,
|
|
3446
|
+
translated: 0,
|
|
3447
|
+
failed: 0,
|
|
3448
|
+
inputTokens: 0,
|
|
3449
|
+
outputTokens: 0,
|
|
3450
|
+
estimatedCostUsd: 0,
|
|
3451
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3452
|
+
alreadyIngested: true
|
|
3453
|
+
});
|
|
3454
|
+
continue;
|
|
3455
|
+
}
|
|
3456
|
+
options.onProgress?.({
|
|
3457
|
+
type: "batch-done",
|
|
3458
|
+
name: tracked.row.job_name,
|
|
3459
|
+
state,
|
|
3460
|
+
model: tracked.row.display_model,
|
|
3461
|
+
count: results.length,
|
|
3462
|
+
jobIndex: tracked.jobIndex,
|
|
3463
|
+
jobCount,
|
|
3464
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3465
|
+
failed: results.filter((r) => r.failed).length,
|
|
3466
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3467
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3468
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3469
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3470
|
+
});
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
2894
3473
|
}
|
|
2895
3474
|
}
|
|
3475
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3476
|
+
return {
|
|
3477
|
+
...baseForItem(item),
|
|
3478
|
+
skipped: false,
|
|
3479
|
+
failed: true,
|
|
3480
|
+
error: formatTranslateError(error),
|
|
3481
|
+
durationMs: Date.now() - startedAt
|
|
3482
|
+
};
|
|
3483
|
+
}
|
|
3484
|
+
|
|
3485
|
+
// src/translate/page-translator.ts
|
|
3486
|
+
async function runPool(items, concurrency, worker) {
|
|
3487
|
+
if (items.length === 0) return;
|
|
3488
|
+
let nextIndex = 0;
|
|
3489
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3490
|
+
while (nextIndex < items.length) {
|
|
3491
|
+
const index = nextIndex++;
|
|
3492
|
+
await worker(items[index], index);
|
|
3493
|
+
}
|
|
3494
|
+
});
|
|
3495
|
+
await Promise.all(runners);
|
|
3496
|
+
}
|
|
2896
3497
|
function labelForItem(item) {
|
|
2897
3498
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2898
3499
|
}
|
|
3500
|
+
async function translatePage(config, item, options = {}) {
|
|
3501
|
+
const startedAt = Date.now();
|
|
3502
|
+
let outcome;
|
|
3503
|
+
try {
|
|
3504
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3505
|
+
} catch (error) {
|
|
3506
|
+
return failedResultForItem(item, error, startedAt);
|
|
3507
|
+
}
|
|
3508
|
+
if (outcome.status === "done") return outcome.result;
|
|
3509
|
+
const prepared = outcome.prepared;
|
|
3510
|
+
if (options.dryRun) {
|
|
3511
|
+
return {
|
|
3512
|
+
...baseForItem(item),
|
|
3513
|
+
skipped: false,
|
|
3514
|
+
model: prepared.model,
|
|
3515
|
+
durationMs: Date.now() - startedAt
|
|
3516
|
+
};
|
|
3517
|
+
}
|
|
3518
|
+
try {
|
|
3519
|
+
const result = await translatePageWithGemini({
|
|
3520
|
+
prompt: prepared.prompt,
|
|
3521
|
+
model: prepared.model,
|
|
3522
|
+
responseSchema: prepared.responseSchema
|
|
3523
|
+
});
|
|
3524
|
+
return finalizeTranslation(
|
|
3525
|
+
config,
|
|
3526
|
+
prepared,
|
|
3527
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3528
|
+
{ costMode: "interactive", startedAt }
|
|
3529
|
+
);
|
|
3530
|
+
} catch (error) {
|
|
3531
|
+
return failedResultForItem(item, error, startedAt);
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
function createResultCollector(items, onProgress) {
|
|
3535
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3536
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3537
|
+
const slots = new Array(items.length);
|
|
3538
|
+
const extras = [];
|
|
3539
|
+
return {
|
|
3540
|
+
emit(result) {
|
|
3541
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3542
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3543
|
+
else extras.push(result);
|
|
3544
|
+
onProgress?.({ type: "item-done", result });
|
|
3545
|
+
},
|
|
3546
|
+
assemble() {
|
|
3547
|
+
return [
|
|
3548
|
+
...slots.filter((result) => result !== void 0),
|
|
3549
|
+
...extras
|
|
3550
|
+
];
|
|
3551
|
+
}
|
|
3552
|
+
};
|
|
3553
|
+
}
|
|
2899
3554
|
async function translateWorklist(config, items, options = {}) {
|
|
3555
|
+
const mode = options.mode ?? "batch";
|
|
2900
3556
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2901
3557
|
const startedAt = Date.now();
|
|
2902
|
-
|
|
2903
|
-
|
|
3558
|
+
if (options.dryRun) {
|
|
3559
|
+
options.onProgress?.({
|
|
3560
|
+
type: "start",
|
|
3561
|
+
total: items.length,
|
|
3562
|
+
concurrency,
|
|
3563
|
+
dryRun: true,
|
|
3564
|
+
model: options.model,
|
|
3565
|
+
mode
|
|
3566
|
+
});
|
|
3567
|
+
const results2 = items.map((item) => {
|
|
3568
|
+
const itemStartedAt = Date.now();
|
|
3569
|
+
try {
|
|
3570
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3571
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3572
|
+
...baseForItem(item),
|
|
3573
|
+
skipped: false,
|
|
3574
|
+
model: outcome.prepared.model,
|
|
3575
|
+
durationMs: Date.now() - itemStartedAt
|
|
3576
|
+
};
|
|
3577
|
+
} catch (error) {
|
|
3578
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
3581
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3582
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3583
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3584
|
+
return results2;
|
|
3585
|
+
}
|
|
3586
|
+
const pending = readPendingBatchWork(config);
|
|
3587
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3588
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3589
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3590
|
+
).length;
|
|
2904
3591
|
options.onProgress?.({
|
|
2905
3592
|
type: "start",
|
|
2906
|
-
total: items.length,
|
|
3593
|
+
total: items.length + resumedExtraCount,
|
|
2907
3594
|
concurrency,
|
|
2908
|
-
dryRun:
|
|
2909
|
-
model: options.model
|
|
3595
|
+
dryRun: false,
|
|
3596
|
+
model: options.model,
|
|
3597
|
+
mode
|
|
3598
|
+
});
|
|
3599
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3600
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3601
|
+
if (mode === "direct") {
|
|
3602
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3603
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3604
|
+
const active = /* @__PURE__ */ new Set();
|
|
3605
|
+
await Promise.all([
|
|
3606
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3607
|
+
const label = labelForItem(item);
|
|
3608
|
+
active.add(label);
|
|
3609
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3610
|
+
const result = await translatePage(config, item, options);
|
|
3611
|
+
active.delete(label);
|
|
3612
|
+
collector.emit(result);
|
|
3613
|
+
}),
|
|
3614
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3615
|
+
onProgress: options.onProgress,
|
|
3616
|
+
onResult: collector.emit
|
|
3617
|
+
}) : Promise.resolve()
|
|
3618
|
+
]);
|
|
3619
|
+
} else {
|
|
3620
|
+
const entries = [];
|
|
3621
|
+
for (const item of workItems) {
|
|
3622
|
+
const itemStartedAt = Date.now();
|
|
3623
|
+
try {
|
|
3624
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3625
|
+
if (outcome.status === "done") {
|
|
3626
|
+
collector.emit(outcome.result);
|
|
3627
|
+
} else {
|
|
3628
|
+
entries.push({
|
|
3629
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3630
|
+
prompt: outcome.prepared.prompt,
|
|
3631
|
+
prepared: outcome.prepared
|
|
3632
|
+
});
|
|
3633
|
+
}
|
|
3634
|
+
} catch (error) {
|
|
3635
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
const plans = planBatchJobs(entries);
|
|
3639
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3640
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3641
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3642
|
+
const submitted = await Promise.all(
|
|
3643
|
+
plans.map(async (plan, planIndex) => {
|
|
3644
|
+
try {
|
|
3645
|
+
return await submitBatchJobPlan(config, {
|
|
3646
|
+
plan,
|
|
3647
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3648
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3649
|
+
jobCount,
|
|
3650
|
+
onProgress: options.onProgress
|
|
3651
|
+
});
|
|
3652
|
+
} catch (error) {
|
|
3653
|
+
for (const entry of plan.entries) {
|
|
3654
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3655
|
+
}
|
|
3656
|
+
return void 0;
|
|
3657
|
+
}
|
|
3658
|
+
})
|
|
3659
|
+
);
|
|
3660
|
+
const jobsToPoll = [
|
|
3661
|
+
...pending.jobs,
|
|
3662
|
+
...submitted.filter((row) => row !== void 0)
|
|
3663
|
+
];
|
|
3664
|
+
if (jobsToPoll.length > 0) {
|
|
3665
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3666
|
+
jobCount,
|
|
3667
|
+
onProgress: options.onProgress,
|
|
3668
|
+
onResult: collector.emit
|
|
3669
|
+
});
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
const results = collector.assemble();
|
|
3673
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3674
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3675
|
+
return results;
|
|
3676
|
+
}
|
|
3677
|
+
function countPendingItems(config, jobs) {
|
|
3678
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3679
|
+
const db = openStore(config, "readonly");
|
|
3680
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3681
|
+
for (const job of jobs) {
|
|
3682
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3683
|
+
}
|
|
3684
|
+
db.close();
|
|
3685
|
+
return counts;
|
|
3686
|
+
}
|
|
3687
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3688
|
+
jobs.forEach((job, jobIndex) => {
|
|
3689
|
+
onProgress?.({
|
|
3690
|
+
type: "batch-submitted",
|
|
3691
|
+
name: job.job_name,
|
|
3692
|
+
count: counts.get(job.id) ?? 0,
|
|
3693
|
+
model: job.display_model,
|
|
3694
|
+
jobIndex,
|
|
3695
|
+
jobCount,
|
|
3696
|
+
resumed: true,
|
|
3697
|
+
createdAt: job.created_at
|
|
3698
|
+
});
|
|
2910
3699
|
});
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
3700
|
+
}
|
|
3701
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3702
|
+
const startedAt = Date.now();
|
|
3703
|
+
const pending = readPendingBatchWork(config);
|
|
3704
|
+
if (pending.jobs.length === 0) return null;
|
|
3705
|
+
options.onProgress?.({
|
|
3706
|
+
type: "start",
|
|
3707
|
+
total: pending.pendingItems.length,
|
|
3708
|
+
concurrency: 1,
|
|
3709
|
+
dryRun: false,
|
|
3710
|
+
mode: "batch"
|
|
3711
|
+
});
|
|
3712
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3713
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3714
|
+
const results = [];
|
|
3715
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3716
|
+
onProgress: options.onProgress,
|
|
3717
|
+
onResult: (result) => {
|
|
3718
|
+
results.push(result);
|
|
3719
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3720
|
+
}
|
|
2919
3721
|
});
|
|
2920
3722
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2921
3723
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3038,6 +3840,7 @@ exports.isRoutableType = isRoutableType;
|
|
|
3038
3840
|
exports.loadConfigSync = loadConfigSync;
|
|
3039
3841
|
exports.resolveConfig = resolveConfig;
|
|
3040
3842
|
exports.resolveLocalesFromPreset = resolveLocalesFromPreset;
|
|
3843
|
+
exports.resumeTranslationJobs = resumeTranslationJobs;
|
|
3041
3844
|
exports.serializeMdx = serializeMdx;
|
|
3042
3845
|
exports.translatePage = translatePage;
|
|
3043
3846
|
exports.translateWorklist = translateWorklist;
|