scribe-cms 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1207 -204
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1208 -205
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +969 -166
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +970 -168
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +136 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +136 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +18 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import remarkParse from 'remark-parse';
|
|
|
8
8
|
import { unified } from 'unified';
|
|
9
9
|
import { createJiti } from 'jiti';
|
|
10
10
|
import { createHash } from 'crypto';
|
|
11
|
-
import { GoogleGenAI } from '@google/genai';
|
|
11
|
+
import { GoogleGenAI, ThinkingLevel, ApiError } from '@google/genai';
|
|
12
12
|
|
|
13
13
|
// src/core/field.ts
|
|
14
14
|
var FIELD_KIND = /* @__PURE__ */ Symbol.for("@genlook/scribe/fieldKind");
|
|
@@ -244,6 +244,7 @@ function resolveConfig(input, baseDir) {
|
|
|
244
244
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
245
245
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
246
246
|
}
|
|
247
|
+
const localeFallbacks = raw.localeFallbacks === false ? {} : deriveLocaleFallbacks(raw.locales, defaultLocale);
|
|
247
248
|
const projectRoot = path3.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
248
249
|
const contentRoot = path3.resolve(projectRoot, raw.contentDir ?? "content");
|
|
249
250
|
const storePath = path3.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -273,6 +274,7 @@ function resolveConfig(input, baseDir) {
|
|
|
273
274
|
defaultLocale,
|
|
274
275
|
localeRouting,
|
|
275
276
|
localePresets: raw.localePresets,
|
|
277
|
+
localeFallbacks,
|
|
276
278
|
translate: raw.translate,
|
|
277
279
|
types
|
|
278
280
|
};
|
|
@@ -282,8 +284,33 @@ function resolveConfig(input, baseDir) {
|
|
|
282
284
|
});
|
|
283
285
|
return config;
|
|
284
286
|
}
|
|
287
|
+
function deriveLocaleFallbacks(locales, defaultLocale) {
|
|
288
|
+
const byLowercase = new Map(locales.map((l) => [l.toLowerCase(), l]));
|
|
289
|
+
const fallbacks = {};
|
|
290
|
+
for (const locale of locales) {
|
|
291
|
+
const subtags = locale.split("-");
|
|
292
|
+
if (subtags.length < 2) continue;
|
|
293
|
+
const chain = [];
|
|
294
|
+
for (let end = subtags.length - 1; end >= 1; end--) {
|
|
295
|
+
const prefix = byLowercase.get(subtags.slice(0, end).join("-").toLowerCase());
|
|
296
|
+
if (prefix && prefix !== locale && prefix !== defaultLocale) {
|
|
297
|
+
chain.push(prefix);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
if (chain.length > 0) {
|
|
301
|
+
fallbacks[locale] = chain;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return fallbacks;
|
|
305
|
+
}
|
|
285
306
|
|
|
286
307
|
// src/core/introspect-schema.ts
|
|
308
|
+
function getArrayElement(schema) {
|
|
309
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
310
|
+
return schema.element;
|
|
311
|
+
}
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
287
314
|
function introspectSchema(schema, prefix = []) {
|
|
288
315
|
const relation = getRelationTarget(schema);
|
|
289
316
|
if (relation && prefix.length > 0) {
|
|
@@ -297,7 +324,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
297
324
|
}
|
|
298
325
|
];
|
|
299
326
|
}
|
|
300
|
-
|
|
327
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
328
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
329
|
+
}
|
|
330
|
+
const base = peelOptionalWrappers(schema);
|
|
301
331
|
if (base instanceof Object && "shape" in base) {
|
|
302
332
|
const shape = base.shape;
|
|
303
333
|
const fields = [];
|
|
@@ -306,9 +336,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
306
336
|
}
|
|
307
337
|
return fields;
|
|
308
338
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
const elementBase =
|
|
339
|
+
const element = getArrayElement(base);
|
|
340
|
+
if (element) {
|
|
341
|
+
const elementBase = peelOptionalWrappers(element);
|
|
312
342
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
313
343
|
const shape = elementBase.shape;
|
|
314
344
|
const fields = [];
|
|
@@ -320,15 +350,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
320
350
|
}
|
|
321
351
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
322
352
|
}
|
|
323
|
-
function
|
|
324
|
-
const
|
|
353
|
+
function buildPathTrie(paths) {
|
|
354
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
325
355
|
for (const path13 of paths) {
|
|
326
|
-
|
|
356
|
+
let node = root;
|
|
357
|
+
for (const segment of path13) {
|
|
358
|
+
let child = node.children.get(segment);
|
|
359
|
+
if (!child) {
|
|
360
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
361
|
+
node.children.set(segment, child);
|
|
362
|
+
}
|
|
363
|
+
node = child;
|
|
364
|
+
}
|
|
365
|
+
node.leaf = true;
|
|
366
|
+
}
|
|
367
|
+
return root;
|
|
368
|
+
}
|
|
369
|
+
function extractNode(data, trie) {
|
|
370
|
+
if (trie.leaf) return data;
|
|
371
|
+
const star = trie.children.get("*");
|
|
372
|
+
if (star) {
|
|
373
|
+
if (!Array.isArray(data)) return void 0;
|
|
374
|
+
return data.map(
|
|
375
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
379
|
+
const record = data;
|
|
380
|
+
const out = {};
|
|
381
|
+
let any = false;
|
|
382
|
+
for (const [key, child] of trie.children) {
|
|
383
|
+
const value = extractNode(record[key], child);
|
|
327
384
|
if (value !== void 0) {
|
|
328
|
-
|
|
385
|
+
out[key] = value;
|
|
386
|
+
any = true;
|
|
329
387
|
}
|
|
330
388
|
}
|
|
331
|
-
return out;
|
|
389
|
+
return any ? out : void 0;
|
|
390
|
+
}
|
|
391
|
+
function extractByPaths(data, paths) {
|
|
392
|
+
if (paths.length === 0) return {};
|
|
393
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
394
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
332
395
|
}
|
|
333
396
|
function pickTranslatable(data, schema) {
|
|
334
397
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -343,32 +406,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
343
406
|
const translatable = pickTranslatable(localeData, schema);
|
|
344
407
|
return deepMerge(structural, translatable);
|
|
345
408
|
}
|
|
346
|
-
function getAtPath(obj, path13) {
|
|
347
|
-
let current = obj;
|
|
348
|
-
for (const segment of path13) {
|
|
349
|
-
if (segment === "*") {
|
|
350
|
-
if (!Array.isArray(current)) return void 0;
|
|
351
|
-
return current.map(
|
|
352
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path13.slice(path13.indexOf("*") + 1)) : void 0
|
|
353
|
-
);
|
|
354
|
-
}
|
|
355
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
356
|
-
current = current[segment];
|
|
357
|
-
}
|
|
358
|
-
return current;
|
|
359
|
-
}
|
|
360
|
-
function setAtPath(obj, path13, value) {
|
|
361
|
-
if (path13.length === 0) return;
|
|
362
|
-
if (path13.length === 1) {
|
|
363
|
-
obj[path13[0]] = value;
|
|
364
|
-
return;
|
|
365
|
-
}
|
|
366
|
-
const [head, ...rest] = path13;
|
|
367
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
368
|
-
obj[head] = {};
|
|
369
|
-
}
|
|
370
|
-
setAtPath(obj[head], rest, value);
|
|
371
|
-
}
|
|
372
409
|
function deepMerge(base, overlay) {
|
|
373
410
|
const out = { ...base };
|
|
374
411
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -538,7 +575,7 @@ function seoFieldsFromEn(enDoc) {
|
|
|
538
575
|
canonicalPathOverride: enDoc.canonicalPathOverride
|
|
539
576
|
};
|
|
540
577
|
}
|
|
541
|
-
var SCHEMA_VERSION =
|
|
578
|
+
var SCHEMA_VERSION = 5;
|
|
542
579
|
var MIGRATIONS = [
|
|
543
580
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
544
581
|
key TEXT PRIMARY KEY,
|
|
@@ -586,7 +623,30 @@ var MIGRATIONS = [
|
|
|
586
623
|
UNIQUE (content_type, en_slug, en_hash)
|
|
587
624
|
)`,
|
|
588
625
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
589
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
626
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
627
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
628
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
629
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
630
|
+
model TEXT NOT NULL,
|
|
631
|
+
display_model TEXT NOT NULL,
|
|
632
|
+
created_at TEXT NOT NULL,
|
|
633
|
+
state TEXT NOT NULL,
|
|
634
|
+
completed_at TEXT
|
|
635
|
+
)`,
|
|
636
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
637
|
+
job_id INTEGER NOT NULL,
|
|
638
|
+
request_index INTEGER NOT NULL,
|
|
639
|
+
content_type TEXT NOT NULL,
|
|
640
|
+
en_slug TEXT NOT NULL,
|
|
641
|
+
locale TEXT NOT NULL,
|
|
642
|
+
en_hash TEXT NOT NULL,
|
|
643
|
+
snapshot_id INTEGER NOT NULL,
|
|
644
|
+
status TEXT NOT NULL,
|
|
645
|
+
error TEXT,
|
|
646
|
+
PRIMARY KEY (job_id, request_index)
|
|
647
|
+
)`,
|
|
648
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
649
|
+
ON translation_batch_items(status)`
|
|
590
650
|
];
|
|
591
651
|
function resolveStorePath(config) {
|
|
592
652
|
return config.storePath;
|
|
@@ -609,6 +669,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
609
669
|
}
|
|
610
670
|
}
|
|
611
671
|
function readSchemaVersion(db) {
|
|
672
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
673
|
+
if (!metaTable) return 0;
|
|
612
674
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
613
675
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
614
676
|
}
|
|
@@ -647,6 +709,9 @@ function getOrCreateEnSnapshot(db, input) {
|
|
|
647
709
|
).get(input.contentType, input.enSlug, input.enHash);
|
|
648
710
|
return row.id;
|
|
649
711
|
}
|
|
712
|
+
function getEnSnapshot(db, snapshotId) {
|
|
713
|
+
return db.prepare(`SELECT * FROM en_snapshots WHERE id = ?`).get(snapshotId);
|
|
714
|
+
}
|
|
650
715
|
function upsertTranslation(db, input) {
|
|
651
716
|
db.prepare(
|
|
652
717
|
`INSERT INTO translations (
|
|
@@ -727,7 +792,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
727
792
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
728
793
|
const attrStart = i;
|
|
729
794
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
730
|
-
body.slice(attrStart, i);
|
|
795
|
+
const attrName = body.slice(attrStart, i);
|
|
796
|
+
if (!attrName) break;
|
|
731
797
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
732
798
|
if (body[i] !== "=") {
|
|
733
799
|
tagOut += body.slice(attrStart, i);
|
|
@@ -736,10 +802,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
736
802
|
tagOut += body.slice(attrStart, i);
|
|
737
803
|
tagOut += "=";
|
|
738
804
|
i += 1;
|
|
805
|
+
const wsStart = i;
|
|
739
806
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
807
|
+
tagOut += body.slice(wsStart, i);
|
|
740
808
|
const quote = body[i];
|
|
741
809
|
if (quote !== '"' && quote !== "'") {
|
|
742
|
-
|
|
810
|
+
break;
|
|
743
811
|
}
|
|
744
812
|
if (quote === "'") {
|
|
745
813
|
const valStart2 = i + 1;
|
|
@@ -1059,7 +1127,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1059
1127
|
}
|
|
1060
1128
|
|
|
1061
1129
|
// src/i18n/resolve-document.ts
|
|
1062
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1130
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1063
1131
|
const urlBuilder = createUrlBuilder({
|
|
1064
1132
|
locales: [defaultLocale, locale],
|
|
1065
1133
|
defaultLocale,
|
|
@@ -1073,15 +1141,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1073
1141
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1074
1142
|
const found = docIdx.bySlug.get(slug);
|
|
1075
1143
|
if (!found) continue;
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1144
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1145
|
+
let target;
|
|
1146
|
+
let targetLocale = locale;
|
|
1147
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1148
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1149
|
+
if (cand) {
|
|
1150
|
+
target = cand;
|
|
1151
|
+
targetLocale = candidateLocale;
|
|
1152
|
+
break;
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
if (target) {
|
|
1156
|
+
if (target.slug === slug) {
|
|
1157
|
+
return { document: target, actualLocale: targetLocale };
|
|
1158
|
+
}
|
|
1078
1159
|
if (!type.path) {
|
|
1079
1160
|
return { document: null, actualLocale: locale };
|
|
1080
1161
|
}
|
|
1081
1162
|
return {
|
|
1082
1163
|
document: null,
|
|
1083
1164
|
actualLocale: locale,
|
|
1084
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1165
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1085
1166
|
};
|
|
1086
1167
|
}
|
|
1087
1168
|
if (docLocale === defaultLocale) {
|
|
@@ -1112,13 +1193,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1112
1193
|
}
|
|
1113
1194
|
return { document: null, actualLocale: locale };
|
|
1114
1195
|
}
|
|
1115
|
-
function getSlugForLocale(document, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1116
|
-
if (sourceLocale === targetLocale) return document.slug;
|
|
1117
|
-
const englishSlug = sourceLocale === defaultLocale ? document.slug : document.enSlug;
|
|
1118
|
-
if (!englishSlug) return null;
|
|
1119
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1120
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1121
|
-
}
|
|
1122
1196
|
|
|
1123
1197
|
// src/create-project.ts
|
|
1124
1198
|
function comparatorFor(orderBy) {
|
|
@@ -1171,7 +1245,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1171
1245
|
config.defaultLocale,
|
|
1172
1246
|
load(),
|
|
1173
1247
|
type,
|
|
1174
|
-
config.localeRouting
|
|
1248
|
+
config.localeRouting,
|
|
1249
|
+
config.localeFallbacks[locale] ?? []
|
|
1175
1250
|
);
|
|
1176
1251
|
if (result.document && type.path) {
|
|
1177
1252
|
return {
|
|
@@ -1193,8 +1268,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1193
1268
|
const params = [];
|
|
1194
1269
|
for (const locale of options.locales ?? config.locales) {
|
|
1195
1270
|
const localeIdx = all.get(locale);
|
|
1271
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1196
1272
|
for (const doc of enIdx.bySlug.values()) {
|
|
1197
|
-
|
|
1273
|
+
let slug;
|
|
1274
|
+
if (locale === config.defaultLocale) {
|
|
1275
|
+
slug = doc.slug;
|
|
1276
|
+
} else {
|
|
1277
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1278
|
+
}
|
|
1198
1279
|
params.push({ locale, slug });
|
|
1199
1280
|
}
|
|
1200
1281
|
}
|
|
@@ -2013,7 +2094,7 @@ function validateTypeRedirects(project) {
|
|
|
2013
2094
|
}
|
|
2014
2095
|
return issues;
|
|
2015
2096
|
}
|
|
2016
|
-
function
|
|
2097
|
+
function getAtPath(obj, fieldPath) {
|
|
2017
2098
|
let current = obj;
|
|
2018
2099
|
for (const segment of fieldPath) {
|
|
2019
2100
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -2058,7 +2139,7 @@ function validateRelations(project) {
|
|
|
2058
2139
|
});
|
|
2059
2140
|
continue;
|
|
2060
2141
|
}
|
|
2061
|
-
const value =
|
|
2142
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
2062
2143
|
if (value === void 0 || value === null || value === "") continue;
|
|
2063
2144
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
2064
2145
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2425,6 +2506,59 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2425
2506
|
return config.locales.filter((l) => l !== config.defaultLocale);
|
|
2426
2507
|
}
|
|
2427
2508
|
|
|
2509
|
+
// src/storage/batch-jobs.ts
|
|
2510
|
+
function insertBatchJob(db, input) {
|
|
2511
|
+
const info = db.prepare(
|
|
2512
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2513
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2514
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2515
|
+
return Number(info.lastInsertRowid);
|
|
2516
|
+
}
|
|
2517
|
+
function insertBatchItems(db, jobId, items) {
|
|
2518
|
+
const stmt = db.prepare(
|
|
2519
|
+
`INSERT INTO translation_batch_items (
|
|
2520
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2521
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2522
|
+
);
|
|
2523
|
+
for (const item of items) {
|
|
2524
|
+
stmt.run(
|
|
2525
|
+
jobId,
|
|
2526
|
+
item.requestIndex,
|
|
2527
|
+
item.contentType,
|
|
2528
|
+
item.enSlug,
|
|
2529
|
+
item.locale,
|
|
2530
|
+
item.enHash,
|
|
2531
|
+
item.snapshotId
|
|
2532
|
+
);
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
function listPendingBatchJobs(db) {
|
|
2536
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2537
|
+
}
|
|
2538
|
+
function listBatchItems(db, jobId) {
|
|
2539
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2540
|
+
}
|
|
2541
|
+
function listPendingBatchItems(db) {
|
|
2542
|
+
return db.prepare(
|
|
2543
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2544
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2545
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2546
|
+
ORDER BY i.job_id, i.request_index`
|
|
2547
|
+
).all();
|
|
2548
|
+
}
|
|
2549
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2550
|
+
const info = db.prepare(
|
|
2551
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2552
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2553
|
+
).run(state, completedAt, jobId);
|
|
2554
|
+
return info.changes > 0;
|
|
2555
|
+
}
|
|
2556
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2557
|
+
db.prepare(
|
|
2558
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2559
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2428
2562
|
// src/history/record-snapshot.ts
|
|
2429
2563
|
function recordEnSnapshot(config, input, db) {
|
|
2430
2564
|
const ownDb = db ?? openStore(config, "readwrite");
|
|
@@ -2439,8 +2573,102 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2439
2573
|
if (!db) ownDb.close();
|
|
2440
2574
|
return id;
|
|
2441
2575
|
}
|
|
2576
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2577
|
+
function statusFromError(error) {
|
|
2578
|
+
if (error instanceof ApiError && typeof error.status === "number") {
|
|
2579
|
+
return error.status;
|
|
2580
|
+
}
|
|
2581
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2582
|
+
const status = error.status;
|
|
2583
|
+
if (typeof status === "number") return status;
|
|
2584
|
+
}
|
|
2585
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2586
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2587
|
+
if (match) return Number(match[1]);
|
|
2588
|
+
return void 0;
|
|
2589
|
+
}
|
|
2590
|
+
function isNetworkError(error) {
|
|
2591
|
+
const err = error;
|
|
2592
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2593
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2594
|
+
return true;
|
|
2595
|
+
}
|
|
2596
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2597
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2598
|
+
}
|
|
2599
|
+
function isTransientError(error) {
|
|
2600
|
+
const status = statusFromError(error);
|
|
2601
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2602
|
+
return isNetworkError(error);
|
|
2603
|
+
}
|
|
2604
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2605
|
+
async function withRetry(fn, options = {}) {
|
|
2606
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2607
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2608
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2609
|
+
const random = options.random ?? Math.random;
|
|
2610
|
+
let lastError;
|
|
2611
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2612
|
+
try {
|
|
2613
|
+
return await fn();
|
|
2614
|
+
} catch (error) {
|
|
2615
|
+
lastError = error;
|
|
2616
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2617
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2618
|
+
const delay = backoff + random() * backoff;
|
|
2619
|
+
await sleep2(delay);
|
|
2620
|
+
}
|
|
2621
|
+
}
|
|
2622
|
+
throw lastError;
|
|
2623
|
+
}
|
|
2442
2624
|
|
|
2443
|
-
// src/translate/gemini-
|
|
2625
|
+
// src/translate/gemini-batch.ts
|
|
2626
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2627
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2628
|
+
"SUCCEEDED",
|
|
2629
|
+
"FAILED",
|
|
2630
|
+
"CANCELLED",
|
|
2631
|
+
"EXPIRED",
|
|
2632
|
+
"PARTIALLY_SUCCEEDED"
|
|
2633
|
+
]);
|
|
2634
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2635
|
+
function normalizeBatchState(state) {
|
|
2636
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2637
|
+
}
|
|
2638
|
+
function isTerminalBatchState(state) {
|
|
2639
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2640
|
+
}
|
|
2641
|
+
function isSuccessfulBatchState(state) {
|
|
2642
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2643
|
+
}
|
|
2644
|
+
function makeClient(apiKey) {
|
|
2645
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2646
|
+
if (!key) {
|
|
2647
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2648
|
+
}
|
|
2649
|
+
return new GoogleGenAI({ apiKey: key });
|
|
2650
|
+
}
|
|
2651
|
+
function textFromBatchResponse(response) {
|
|
2652
|
+
if (typeof response.text === "string") return response.text;
|
|
2653
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2654
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2655
|
+
}
|
|
2656
|
+
async function createGeminiBatchJob(input) {
|
|
2657
|
+
const ai = makeClient(input.apiKey);
|
|
2658
|
+
const job = await withRetry(
|
|
2659
|
+
() => ai.batches.create({
|
|
2660
|
+
model: input.model,
|
|
2661
|
+
src: input.requests,
|
|
2662
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2663
|
+
})
|
|
2664
|
+
);
|
|
2665
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2666
|
+
return job;
|
|
2667
|
+
}
|
|
2668
|
+
async function getGeminiBatchJob(input) {
|
|
2669
|
+
const ai = makeClient(input.apiKey);
|
|
2670
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2671
|
+
}
|
|
2444
2672
|
var GEMINI_MODEL_IDS = {
|
|
2445
2673
|
"gemini-2.5-pro": "gemini-2.5-pro",
|
|
2446
2674
|
"gemini-3.1-pro": "gemini-3.1-pro-preview"
|
|
@@ -2460,6 +2688,12 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2460
2688
|
if (alias) return alias[0];
|
|
2461
2689
|
return name;
|
|
2462
2690
|
}
|
|
2691
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2692
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2693
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: ThinkingLevel.LOW };
|
|
2694
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2695
|
+
return void 0;
|
|
2696
|
+
}
|
|
2463
2697
|
|
|
2464
2698
|
// src/translate/gemini-client.ts
|
|
2465
2699
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
@@ -2473,11 +2707,36 @@ function extractJson(text) {
|
|
|
2473
2707
|
return trimmed;
|
|
2474
2708
|
}
|
|
2475
2709
|
function parseGeminiResponse(text) {
|
|
2710
|
+
let parsed;
|
|
2476
2711
|
try {
|
|
2477
|
-
|
|
2712
|
+
parsed = JSON.parse(text.trim());
|
|
2478
2713
|
} catch {
|
|
2479
|
-
|
|
2714
|
+
parsed = JSON.parse(extractJson(text));
|
|
2715
|
+
}
|
|
2716
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2717
|
+
parsed.frontmatter = {};
|
|
2480
2718
|
}
|
|
2719
|
+
return parsed;
|
|
2720
|
+
}
|
|
2721
|
+
function buildGeminiRequestConfig(input) {
|
|
2722
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2723
|
+
return {
|
|
2724
|
+
responseMimeType: "application/json",
|
|
2725
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2726
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2727
|
+
};
|
|
2728
|
+
}
|
|
2729
|
+
function usageFromResponse(response) {
|
|
2730
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2731
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2732
|
+
return {
|
|
2733
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2734
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2735
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2736
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2737
|
+
thoughtsTokens,
|
|
2738
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2739
|
+
};
|
|
2481
2740
|
}
|
|
2482
2741
|
async function translatePageWithGemini(input) {
|
|
2483
2742
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2489,26 +2748,22 @@ async function translatePageWithGemini(input) {
|
|
|
2489
2748
|
);
|
|
2490
2749
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2491
2750
|
const ai = new GoogleGenAI({ apiKey });
|
|
2492
|
-
const response = await
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2751
|
+
const response = await withRetry(
|
|
2752
|
+
() => ai.models.generateContent({
|
|
2753
|
+
model: apiModel,
|
|
2754
|
+
contents: input.prompt,
|
|
2755
|
+
config: buildGeminiRequestConfig({
|
|
2756
|
+
apiModelId: apiModel,
|
|
2757
|
+
responseSchema: input.responseSchema
|
|
2758
|
+
})
|
|
2759
|
+
})
|
|
2760
|
+
);
|
|
2500
2761
|
const raw = response.text ?? "";
|
|
2501
2762
|
const parsed = parseGeminiResponse(raw);
|
|
2502
2763
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2503
2764
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2504
2765
|
}
|
|
2505
|
-
|
|
2506
|
-
const usage = {
|
|
2507
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2508
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2509
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2510
|
-
};
|
|
2511
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2766
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2512
2767
|
}
|
|
2513
2768
|
|
|
2514
2769
|
// src/translate/gemini-pricing.ts
|
|
@@ -2528,12 +2783,14 @@ function resolveModelPricing(model) {
|
|
|
2528
2783
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2529
2784
|
return match?.[1];
|
|
2530
2785
|
}
|
|
2531
|
-
|
|
2786
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2787
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2532
2788
|
const pricing = resolveModelPricing(model);
|
|
2533
2789
|
if (!pricing) return void 0;
|
|
2534
2790
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2535
2791
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2536
|
-
|
|
2792
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2793
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2537
2794
|
}
|
|
2538
2795
|
|
|
2539
2796
|
// src/translate/prompts/translation-prompt.ts
|
|
@@ -2549,6 +2806,7 @@ var LOCALE_NAMES = {
|
|
|
2549
2806
|
"zh-CN": "Simplified Chinese",
|
|
2550
2807
|
"zh-TW": "Traditional Chinese",
|
|
2551
2808
|
pt: "Portuguese",
|
|
2809
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2552
2810
|
ja: "Japanese",
|
|
2553
2811
|
ru: "Russian",
|
|
2554
2812
|
it: "Italian",
|
|
@@ -2556,29 +2814,29 @@ var LOCALE_NAMES = {
|
|
|
2556
2814
|
};
|
|
2557
2815
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2558
2816
|
return [
|
|
2559
|
-
`Localize the content
|
|
2817
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2560
2818
|
"Do not translate word-for-word.",
|
|
2561
2819
|
"Preserve the source tone and brand voice.",
|
|
2562
2820
|
"Write as if a native speaker authored it for the target market."
|
|
2563
2821
|
].join(" ");
|
|
2564
2822
|
}
|
|
2823
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2824
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2825
|
+
if (!hasFrontmatter) {
|
|
2826
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2827
|
+
}
|
|
2828
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2829
|
+
}
|
|
2565
2830
|
function buildPageTranslationPrompt(input) {
|
|
2566
2831
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2567
|
-
const
|
|
2568
|
-
const
|
|
2569
|
-
|
|
2832
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2833
|
+
const prefix = [
|
|
2834
|
+
TASK_FRAMING,
|
|
2570
2835
|
"",
|
|
2571
2836
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2572
2837
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2573
2838
|
"## Rules",
|
|
2574
2839
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2575
|
-
...input.slugStrategy === "localized" ? [
|
|
2576
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2577
|
-
] : [],
|
|
2578
|
-
"",
|
|
2579
|
-
"## Output format",
|
|
2580
|
-
"Return ONLY valid JSON with keys:",
|
|
2581
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2582
2840
|
"",
|
|
2583
2841
|
"## EN translatable frontmatter (JSON)",
|
|
2584
2842
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2586,7 +2844,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2586
2844
|
"## EN body (MDX)",
|
|
2587
2845
|
input.enBody
|
|
2588
2846
|
];
|
|
2589
|
-
|
|
2847
|
+
const suffix = [
|
|
2848
|
+
"",
|
|
2849
|
+
"## Target language",
|
|
2850
|
+
localizationPrompt,
|
|
2851
|
+
...input.slugStrategy === "localized" ? [
|
|
2852
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2853
|
+
] : [],
|
|
2854
|
+
"",
|
|
2855
|
+
"## Output format",
|
|
2856
|
+
"Return ONLY valid JSON with keys:",
|
|
2857
|
+
buildOutputFormatLine(
|
|
2858
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2859
|
+
input.slugStrategy
|
|
2860
|
+
)
|
|
2861
|
+
];
|
|
2862
|
+
return [...prefix, ...suffix].join("\n");
|
|
2590
2863
|
}
|
|
2591
2864
|
function getObjectShape(schema) {
|
|
2592
2865
|
const base = peelOptionalWrappers(schema);
|
|
@@ -2634,9 +2907,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2634
2907
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2635
2908
|
try {
|
|
2636
2909
|
const translatable = buildTranslatableSubschema(schema);
|
|
2637
|
-
if (!translatable) return null;
|
|
2638
2910
|
const responseShape = {
|
|
2639
|
-
frontmatter: translatable,
|
|
2911
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2640
2912
|
body: z.string()
|
|
2641
2913
|
};
|
|
2642
2914
|
if (slugStrategy === "localized") {
|
|
@@ -2715,7 +2987,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2715
2987
|
return { ok: true, frontmatter };
|
|
2716
2988
|
}
|
|
2717
2989
|
|
|
2718
|
-
// src/translate/
|
|
2990
|
+
// src/translate/translate-core.ts
|
|
2991
|
+
function translationItemKey(item) {
|
|
2992
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
2993
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
2994
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
2995
|
+
}
|
|
2719
2996
|
function summarizeResults(results, durationMs) {
|
|
2720
2997
|
return results.reduce(
|
|
2721
2998
|
(totals, result) => {
|
|
@@ -2751,17 +3028,6 @@ function formatTranslateError(error) {
|
|
|
2751
3028
|
}
|
|
2752
3029
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2753
3030
|
}
|
|
2754
|
-
async function runPool(items, concurrency, worker) {
|
|
2755
|
-
if (items.length === 0) return;
|
|
2756
|
-
let nextIndex = 0;
|
|
2757
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2758
|
-
while (nextIndex < items.length) {
|
|
2759
|
-
const index = nextIndex++;
|
|
2760
|
-
await worker(items[index], index);
|
|
2761
|
-
}
|
|
2762
|
-
});
|
|
2763
|
-
await Promise.all(runners);
|
|
2764
|
-
}
|
|
2765
3031
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2766
3032
|
const fm = enDoc.frontmatter;
|
|
2767
3033
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2770,73 +3036,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2770
3036
|
}
|
|
2771
3037
|
return enSlug;
|
|
2772
3038
|
}
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
const base = {
|
|
3039
|
+
function baseForItem(item) {
|
|
3040
|
+
return {
|
|
2776
3041
|
contentType: item.contentType,
|
|
2777
3042
|
enSlug: item.enSlug,
|
|
2778
3043
|
locale: item.locale
|
|
2779
3044
|
};
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
3045
|
+
}
|
|
3046
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
3047
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
3048
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
3049
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
3050
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
3051
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
3052
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
3053
|
+
const db = openStore(config, "readonly");
|
|
3054
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
3055
|
+
db.close();
|
|
3056
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
3057
|
+
return {
|
|
3058
|
+
status: "done",
|
|
3059
|
+
result: {
|
|
3060
|
+
...baseForItem(item),
|
|
2793
3061
|
skipped: true,
|
|
2794
3062
|
reason: "fresh",
|
|
2795
3063
|
durationMs: Date.now() - startedAt
|
|
2796
|
-
}
|
|
2797
|
-
}
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
3064
|
+
}
|
|
3065
|
+
};
|
|
3066
|
+
}
|
|
3067
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
3068
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
3069
|
+
const prompt = buildPageTranslationPrompt({
|
|
3070
|
+
resolved: resolvedTranslate,
|
|
3071
|
+
targetLocale: item.locale,
|
|
3072
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
3073
|
+
translatableFrontmatter: payload.frontmatter,
|
|
3074
|
+
enBody: payload.body,
|
|
3075
|
+
slugStrategy: type.slugStrategy
|
|
3076
|
+
});
|
|
3077
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
3078
|
+
return {
|
|
3079
|
+
status: "ready",
|
|
3080
|
+
prepared: {
|
|
3081
|
+
item,
|
|
3082
|
+
type,
|
|
3083
|
+
enDoc,
|
|
3084
|
+
payload,
|
|
3085
|
+
currentEnHash,
|
|
3086
|
+
existingSlug: existing?.slug,
|
|
2819
3087
|
model,
|
|
3088
|
+
prompt,
|
|
2820
3089
|
responseSchema: responseSchema ?? void 0
|
|
2821
|
-
}
|
|
2822
|
-
|
|
3090
|
+
}
|
|
3091
|
+
};
|
|
3092
|
+
}
|
|
3093
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
3094
|
+
const { item, type, enDoc, payload } = prepared;
|
|
3095
|
+
const base = baseForItem(item);
|
|
3096
|
+
try {
|
|
3097
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2823
3098
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2824
3099
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2825
3100
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2826
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
3101
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2827
3102
|
if (!validated.ok) {
|
|
2828
3103
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2829
3104
|
}
|
|
2830
3105
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2831
|
-
|
|
3106
|
+
output.parsed.body
|
|
2832
3107
|
);
|
|
2833
3108
|
const writeDb = openStore(config, "readwrite");
|
|
2834
|
-
const snapshotId = recordEnSnapshot(
|
|
3109
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2835
3110
|
config,
|
|
2836
3111
|
{
|
|
2837
3112
|
contentType: type.id,
|
|
2838
3113
|
enSlug: item.enSlug,
|
|
2839
|
-
enHash: currentEnHash,
|
|
3114
|
+
enHash: prepared.currentEnHash,
|
|
2840
3115
|
frontmatter: payload.frontmatter,
|
|
2841
3116
|
body: payload.body
|
|
2842
3117
|
},
|
|
@@ -2849,24 +3124,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2849
3124
|
slug,
|
|
2850
3125
|
frontmatter: validated.frontmatter,
|
|
2851
3126
|
body: translatedBody,
|
|
2852
|
-
enHash: currentEnHash,
|
|
3127
|
+
enHash: prepared.currentEnHash,
|
|
2853
3128
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2854
|
-
model:
|
|
3129
|
+
model: output.model,
|
|
2855
3130
|
snapshotId
|
|
2856
3131
|
});
|
|
2857
3132
|
writeDb.close();
|
|
2858
3133
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2859
|
-
normalizeGeminiDisplayName(
|
|
2860
|
-
|
|
2861
|
-
|
|
3134
|
+
normalizeGeminiDisplayName(output.model),
|
|
3135
|
+
output.usage.inputTokens,
|
|
3136
|
+
output.usage.outputTokens,
|
|
3137
|
+
options.costMode
|
|
2862
3138
|
);
|
|
2863
3139
|
return {
|
|
2864
3140
|
...base,
|
|
2865
3141
|
skipped: false,
|
|
2866
|
-
model:
|
|
2867
|
-
usage:
|
|
3142
|
+
model: output.model,
|
|
3143
|
+
usage: output.usage,
|
|
2868
3144
|
estimatedCostUsd,
|
|
2869
|
-
durationMs: Date.now() - startedAt,
|
|
3145
|
+
durationMs: Date.now() - options.startedAt,
|
|
2870
3146
|
slugAdjusted,
|
|
2871
3147
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2872
3148
|
};
|
|
@@ -2876,33 +3152,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2876
3152
|
skipped: false,
|
|
2877
3153
|
failed: true,
|
|
2878
3154
|
error: formatTranslateError(error),
|
|
3155
|
+
durationMs: Date.now() - options.startedAt
|
|
3156
|
+
};
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
function displayModelFor(prepared) {
|
|
3160
|
+
return normalizeGeminiDisplayName(
|
|
3161
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
3162
|
+
);
|
|
3163
|
+
}
|
|
3164
|
+
|
|
3165
|
+
// src/translate/batch-worklist.ts
|
|
3166
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
3167
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
3168
|
+
var INITIAL_POLL_MS = 5e3;
|
|
3169
|
+
var MAX_POLL_MS = 3e4;
|
|
3170
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
3171
|
+
function sleep(ms) {
|
|
3172
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3173
|
+
}
|
|
3174
|
+
function planBatchJobs(entries) {
|
|
3175
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
3176
|
+
for (const entry of entries) {
|
|
3177
|
+
const group = byModel.get(entry.apiModel);
|
|
3178
|
+
if (group) group.push(entry);
|
|
3179
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
3180
|
+
}
|
|
3181
|
+
const plans = [];
|
|
3182
|
+
for (const [apiModel, group] of byModel) {
|
|
3183
|
+
let current = [];
|
|
3184
|
+
let currentBytes = 0;
|
|
3185
|
+
for (const entry of group) {
|
|
3186
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
3187
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
3188
|
+
plans.push({ apiModel, entries: current });
|
|
3189
|
+
current = [];
|
|
3190
|
+
currentBytes = 0;
|
|
3191
|
+
}
|
|
3192
|
+
current.push(entry);
|
|
3193
|
+
currentBytes += size;
|
|
3194
|
+
}
|
|
3195
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
3196
|
+
}
|
|
3197
|
+
return plans;
|
|
3198
|
+
}
|
|
3199
|
+
function readPendingBatchWork(config) {
|
|
3200
|
+
const db = openStore(config, "readwrite");
|
|
3201
|
+
const jobs = listPendingBatchJobs(db);
|
|
3202
|
+
const pendingItems = listPendingBatchItems(db);
|
|
3203
|
+
db.close();
|
|
3204
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
3205
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
3206
|
+
}
|
|
3207
|
+
async function submitBatchJobPlan(config, input) {
|
|
3208
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
3209
|
+
const job = await createGeminiBatchJob({
|
|
3210
|
+
model: plan.apiModel,
|
|
3211
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
3212
|
+
contents: prepared.prompt,
|
|
3213
|
+
config: buildGeminiRequestConfig({
|
|
3214
|
+
apiModelId: plan.apiModel,
|
|
3215
|
+
responseSchema: prepared.responseSchema
|
|
3216
|
+
})
|
|
3217
|
+
})),
|
|
3218
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
3219
|
+
});
|
|
3220
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3221
|
+
const db = openStore(config, "readwrite");
|
|
3222
|
+
const jobId = insertBatchJob(db, {
|
|
3223
|
+
jobName: job.name,
|
|
3224
|
+
model: plan.apiModel,
|
|
3225
|
+
displayModel,
|
|
3226
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3227
|
+
createdAt
|
|
3228
|
+
});
|
|
3229
|
+
insertBatchItems(
|
|
3230
|
+
db,
|
|
3231
|
+
jobId,
|
|
3232
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
3233
|
+
requestIndex,
|
|
3234
|
+
contentType: prepared.item.contentType,
|
|
3235
|
+
enSlug: prepared.item.enSlug,
|
|
3236
|
+
locale: prepared.item.locale,
|
|
3237
|
+
enHash: prepared.currentEnHash,
|
|
3238
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
3239
|
+
// on the EN files still matching (or existing) on disk.
|
|
3240
|
+
snapshotId: recordEnSnapshot(
|
|
3241
|
+
config,
|
|
3242
|
+
{
|
|
3243
|
+
contentType: prepared.item.contentType,
|
|
3244
|
+
enSlug: prepared.item.enSlug,
|
|
3245
|
+
enHash: prepared.currentEnHash,
|
|
3246
|
+
frontmatter: prepared.payload.frontmatter,
|
|
3247
|
+
body: prepared.payload.body
|
|
3248
|
+
},
|
|
3249
|
+
db
|
|
3250
|
+
)
|
|
3251
|
+
}))
|
|
3252
|
+
);
|
|
3253
|
+
const row = {
|
|
3254
|
+
id: jobId,
|
|
3255
|
+
job_name: job.name,
|
|
3256
|
+
model: plan.apiModel,
|
|
3257
|
+
display_model: displayModel,
|
|
3258
|
+
created_at: createdAt,
|
|
3259
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3260
|
+
completed_at: null
|
|
3261
|
+
};
|
|
3262
|
+
db.close();
|
|
3263
|
+
input.onProgress?.({
|
|
3264
|
+
type: "batch-submitted",
|
|
3265
|
+
name: job.name,
|
|
3266
|
+
count: plan.entries.length,
|
|
3267
|
+
model: displayModel,
|
|
3268
|
+
jobIndex,
|
|
3269
|
+
jobCount,
|
|
3270
|
+
createdAt
|
|
3271
|
+
});
|
|
3272
|
+
return row;
|
|
3273
|
+
}
|
|
3274
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3275
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3276
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3277
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3278
|
+
if (!enDoc) {
|
|
3279
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3280
|
+
if (!snapshot) {
|
|
3281
|
+
throw new Error(
|
|
3282
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3283
|
+
);
|
|
3284
|
+
}
|
|
3285
|
+
enDoc = {
|
|
3286
|
+
slug: itemRow.en_slug,
|
|
3287
|
+
enSlug: itemRow.en_slug,
|
|
3288
|
+
locale: config.defaultLocale,
|
|
3289
|
+
noindex: false,
|
|
3290
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3291
|
+
content: snapshot.body
|
|
3292
|
+
};
|
|
3293
|
+
}
|
|
3294
|
+
const item = {
|
|
3295
|
+
contentType: itemRow.content_type,
|
|
3296
|
+
enSlug: itemRow.en_slug,
|
|
3297
|
+
locale: itemRow.locale,
|
|
3298
|
+
reason: "missing",
|
|
3299
|
+
currentEnHash: itemRow.en_hash
|
|
3300
|
+
};
|
|
3301
|
+
return {
|
|
3302
|
+
item,
|
|
3303
|
+
type,
|
|
3304
|
+
enDoc,
|
|
3305
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3306
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3307
|
+
currentEnHash: itemRow.en_hash,
|
|
3308
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3309
|
+
model: jobRow.display_model,
|
|
3310
|
+
prompt: "",
|
|
3311
|
+
responseSchema: void 0
|
|
3312
|
+
};
|
|
3313
|
+
}
|
|
3314
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3315
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3316
|
+
const db = openStore(config, "readwrite");
|
|
3317
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3318
|
+
db.close();
|
|
3319
|
+
return null;
|
|
3320
|
+
}
|
|
3321
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3322
|
+
const results = [];
|
|
3323
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3324
|
+
const result = {
|
|
3325
|
+
contentType: itemRow.content_type,
|
|
3326
|
+
enSlug: itemRow.en_slug,
|
|
3327
|
+
locale: itemRow.locale,
|
|
3328
|
+
skipped: false,
|
|
3329
|
+
failed: true,
|
|
3330
|
+
error: formatTranslateError(new Error(message)),
|
|
2879
3331
|
durationMs: Date.now() - startedAt
|
|
2880
3332
|
};
|
|
3333
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3334
|
+
results.push(result);
|
|
3335
|
+
onResult?.(result);
|
|
3336
|
+
};
|
|
3337
|
+
if (isSuccessfulBatchState(state)) {
|
|
3338
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3339
|
+
for (const itemRow of pendingItems) {
|
|
3340
|
+
const startedAt = Date.now();
|
|
3341
|
+
const inlined = responses[itemRow.request_index];
|
|
3342
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3343
|
+
failItem(
|
|
3344
|
+
itemRow,
|
|
3345
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3346
|
+
startedAt
|
|
3347
|
+
);
|
|
3348
|
+
continue;
|
|
3349
|
+
}
|
|
3350
|
+
let result;
|
|
3351
|
+
try {
|
|
3352
|
+
const response = inlined.response;
|
|
3353
|
+
const raw = textFromBatchResponse(response);
|
|
3354
|
+
const parsed = parseGeminiResponse(raw);
|
|
3355
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3356
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3357
|
+
}
|
|
3358
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3359
|
+
result = finalizeTranslation(
|
|
3360
|
+
config,
|
|
3361
|
+
prepared,
|
|
3362
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3363
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3364
|
+
);
|
|
3365
|
+
} catch (error) {
|
|
3366
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3367
|
+
continue;
|
|
3368
|
+
}
|
|
3369
|
+
updateBatchItemStatus(
|
|
3370
|
+
db,
|
|
3371
|
+
jobRow.id,
|
|
3372
|
+
itemRow.request_index,
|
|
3373
|
+
result.failed ? "failed" : "done",
|
|
3374
|
+
result.error
|
|
3375
|
+
);
|
|
3376
|
+
results.push(result);
|
|
3377
|
+
onResult?.(result);
|
|
3378
|
+
}
|
|
3379
|
+
} else {
|
|
3380
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3381
|
+
const startedAt = Date.now();
|
|
3382
|
+
for (const itemRow of pendingItems) {
|
|
3383
|
+
failItem(itemRow, message, startedAt);
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
db.close();
|
|
3387
|
+
return results;
|
|
3388
|
+
}
|
|
3389
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3390
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3391
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3392
|
+
const loopStartedAt = Date.now();
|
|
3393
|
+
let delay = INITIAL_POLL_MS;
|
|
3394
|
+
let firstRound = true;
|
|
3395
|
+
const elapsedFor = (row) => {
|
|
3396
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3397
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3398
|
+
};
|
|
3399
|
+
while (active.length > 0) {
|
|
3400
|
+
if (!firstRound) {
|
|
3401
|
+
await sleep(delay);
|
|
3402
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3403
|
+
}
|
|
3404
|
+
firstRound = false;
|
|
3405
|
+
for (const tracked of [...active]) {
|
|
3406
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3407
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3408
|
+
options.onProgress?.({
|
|
3409
|
+
type: "batch-polling",
|
|
3410
|
+
name: tracked.row.job_name,
|
|
3411
|
+
state,
|
|
3412
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3413
|
+
jobIndex: tracked.jobIndex,
|
|
3414
|
+
jobCount
|
|
3415
|
+
});
|
|
3416
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3417
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3418
|
+
const results = ingestBatchJob(
|
|
3419
|
+
config,
|
|
3420
|
+
tracked.row,
|
|
3421
|
+
batchJob,
|
|
3422
|
+
options.onResult
|
|
3423
|
+
);
|
|
3424
|
+
if (results === null) {
|
|
3425
|
+
options.onProgress?.({
|
|
3426
|
+
type: "batch-done",
|
|
3427
|
+
name: tracked.row.job_name,
|
|
3428
|
+
state,
|
|
3429
|
+
model: tracked.row.display_model,
|
|
3430
|
+
count: 0,
|
|
3431
|
+
jobIndex: tracked.jobIndex,
|
|
3432
|
+
jobCount,
|
|
3433
|
+
translated: 0,
|
|
3434
|
+
failed: 0,
|
|
3435
|
+
inputTokens: 0,
|
|
3436
|
+
outputTokens: 0,
|
|
3437
|
+
estimatedCostUsd: 0,
|
|
3438
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3439
|
+
alreadyIngested: true
|
|
3440
|
+
});
|
|
3441
|
+
continue;
|
|
3442
|
+
}
|
|
3443
|
+
options.onProgress?.({
|
|
3444
|
+
type: "batch-done",
|
|
3445
|
+
name: tracked.row.job_name,
|
|
3446
|
+
state,
|
|
3447
|
+
model: tracked.row.display_model,
|
|
3448
|
+
count: results.length,
|
|
3449
|
+
jobIndex: tracked.jobIndex,
|
|
3450
|
+
jobCount,
|
|
3451
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3452
|
+
failed: results.filter((r) => r.failed).length,
|
|
3453
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3454
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3455
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3456
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3457
|
+
});
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
2881
3460
|
}
|
|
2882
3461
|
}
|
|
3462
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3463
|
+
return {
|
|
3464
|
+
...baseForItem(item),
|
|
3465
|
+
skipped: false,
|
|
3466
|
+
failed: true,
|
|
3467
|
+
error: formatTranslateError(error),
|
|
3468
|
+
durationMs: Date.now() - startedAt
|
|
3469
|
+
};
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
// src/translate/page-translator.ts
|
|
3473
|
+
async function runPool(items, concurrency, worker) {
|
|
3474
|
+
if (items.length === 0) return;
|
|
3475
|
+
let nextIndex = 0;
|
|
3476
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3477
|
+
while (nextIndex < items.length) {
|
|
3478
|
+
const index = nextIndex++;
|
|
3479
|
+
await worker(items[index], index);
|
|
3480
|
+
}
|
|
3481
|
+
});
|
|
3482
|
+
await Promise.all(runners);
|
|
3483
|
+
}
|
|
2883
3484
|
function labelForItem(item) {
|
|
2884
3485
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2885
3486
|
}
|
|
3487
|
+
async function translatePage(config, item, options = {}) {
|
|
3488
|
+
const startedAt = Date.now();
|
|
3489
|
+
let outcome;
|
|
3490
|
+
try {
|
|
3491
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3492
|
+
} catch (error) {
|
|
3493
|
+
return failedResultForItem(item, error, startedAt);
|
|
3494
|
+
}
|
|
3495
|
+
if (outcome.status === "done") return outcome.result;
|
|
3496
|
+
const prepared = outcome.prepared;
|
|
3497
|
+
if (options.dryRun) {
|
|
3498
|
+
return {
|
|
3499
|
+
...baseForItem(item),
|
|
3500
|
+
skipped: false,
|
|
3501
|
+
model: prepared.model,
|
|
3502
|
+
durationMs: Date.now() - startedAt
|
|
3503
|
+
};
|
|
3504
|
+
}
|
|
3505
|
+
try {
|
|
3506
|
+
const result = await translatePageWithGemini({
|
|
3507
|
+
prompt: prepared.prompt,
|
|
3508
|
+
model: prepared.model,
|
|
3509
|
+
responseSchema: prepared.responseSchema
|
|
3510
|
+
});
|
|
3511
|
+
return finalizeTranslation(
|
|
3512
|
+
config,
|
|
3513
|
+
prepared,
|
|
3514
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3515
|
+
{ costMode: "interactive", startedAt }
|
|
3516
|
+
);
|
|
3517
|
+
} catch (error) {
|
|
3518
|
+
return failedResultForItem(item, error, startedAt);
|
|
3519
|
+
}
|
|
3520
|
+
}
|
|
3521
|
+
function createResultCollector(items, onProgress) {
|
|
3522
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3523
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3524
|
+
const slots = new Array(items.length);
|
|
3525
|
+
const extras = [];
|
|
3526
|
+
return {
|
|
3527
|
+
emit(result) {
|
|
3528
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3529
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3530
|
+
else extras.push(result);
|
|
3531
|
+
onProgress?.({ type: "item-done", result });
|
|
3532
|
+
},
|
|
3533
|
+
assemble() {
|
|
3534
|
+
return [
|
|
3535
|
+
...slots.filter((result) => result !== void 0),
|
|
3536
|
+
...extras
|
|
3537
|
+
];
|
|
3538
|
+
}
|
|
3539
|
+
};
|
|
3540
|
+
}
|
|
2886
3541
|
async function translateWorklist(config, items, options = {}) {
|
|
3542
|
+
const mode = options.mode ?? "batch";
|
|
2887
3543
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2888
3544
|
const startedAt = Date.now();
|
|
2889
|
-
|
|
2890
|
-
|
|
3545
|
+
if (options.dryRun) {
|
|
3546
|
+
options.onProgress?.({
|
|
3547
|
+
type: "start",
|
|
3548
|
+
total: items.length,
|
|
3549
|
+
concurrency,
|
|
3550
|
+
dryRun: true,
|
|
3551
|
+
model: options.model,
|
|
3552
|
+
mode
|
|
3553
|
+
});
|
|
3554
|
+
const results2 = items.map((item) => {
|
|
3555
|
+
const itemStartedAt = Date.now();
|
|
3556
|
+
try {
|
|
3557
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3558
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3559
|
+
...baseForItem(item),
|
|
3560
|
+
skipped: false,
|
|
3561
|
+
model: outcome.prepared.model,
|
|
3562
|
+
durationMs: Date.now() - itemStartedAt
|
|
3563
|
+
};
|
|
3564
|
+
} catch (error) {
|
|
3565
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3566
|
+
}
|
|
3567
|
+
});
|
|
3568
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3569
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3570
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3571
|
+
return results2;
|
|
3572
|
+
}
|
|
3573
|
+
const pending = readPendingBatchWork(config);
|
|
3574
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3575
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3576
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3577
|
+
).length;
|
|
2891
3578
|
options.onProgress?.({
|
|
2892
3579
|
type: "start",
|
|
2893
|
-
total: items.length,
|
|
3580
|
+
total: items.length + resumedExtraCount,
|
|
2894
3581
|
concurrency,
|
|
2895
|
-
dryRun:
|
|
2896
|
-
model: options.model
|
|
3582
|
+
dryRun: false,
|
|
3583
|
+
model: options.model,
|
|
3584
|
+
mode
|
|
3585
|
+
});
|
|
3586
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3587
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3588
|
+
if (mode === "direct") {
|
|
3589
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3590
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3591
|
+
const active = /* @__PURE__ */ new Set();
|
|
3592
|
+
await Promise.all([
|
|
3593
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3594
|
+
const label = labelForItem(item);
|
|
3595
|
+
active.add(label);
|
|
3596
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3597
|
+
const result = await translatePage(config, item, options);
|
|
3598
|
+
active.delete(label);
|
|
3599
|
+
collector.emit(result);
|
|
3600
|
+
}),
|
|
3601
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3602
|
+
onProgress: options.onProgress,
|
|
3603
|
+
onResult: collector.emit
|
|
3604
|
+
}) : Promise.resolve()
|
|
3605
|
+
]);
|
|
3606
|
+
} else {
|
|
3607
|
+
const entries = [];
|
|
3608
|
+
for (const item of workItems) {
|
|
3609
|
+
const itemStartedAt = Date.now();
|
|
3610
|
+
try {
|
|
3611
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3612
|
+
if (outcome.status === "done") {
|
|
3613
|
+
collector.emit(outcome.result);
|
|
3614
|
+
} else {
|
|
3615
|
+
entries.push({
|
|
3616
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3617
|
+
prompt: outcome.prepared.prompt,
|
|
3618
|
+
prepared: outcome.prepared
|
|
3619
|
+
});
|
|
3620
|
+
}
|
|
3621
|
+
} catch (error) {
|
|
3622
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
const plans = planBatchJobs(entries);
|
|
3626
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3627
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3628
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3629
|
+
const submitted = await Promise.all(
|
|
3630
|
+
plans.map(async (plan, planIndex) => {
|
|
3631
|
+
try {
|
|
3632
|
+
return await submitBatchJobPlan(config, {
|
|
3633
|
+
plan,
|
|
3634
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3635
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3636
|
+
jobCount,
|
|
3637
|
+
onProgress: options.onProgress
|
|
3638
|
+
});
|
|
3639
|
+
} catch (error) {
|
|
3640
|
+
for (const entry of plan.entries) {
|
|
3641
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3642
|
+
}
|
|
3643
|
+
return void 0;
|
|
3644
|
+
}
|
|
3645
|
+
})
|
|
3646
|
+
);
|
|
3647
|
+
const jobsToPoll = [
|
|
3648
|
+
...pending.jobs,
|
|
3649
|
+
...submitted.filter((row) => row !== void 0)
|
|
3650
|
+
];
|
|
3651
|
+
if (jobsToPoll.length > 0) {
|
|
3652
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3653
|
+
jobCount,
|
|
3654
|
+
onProgress: options.onProgress,
|
|
3655
|
+
onResult: collector.emit
|
|
3656
|
+
});
|
|
3657
|
+
}
|
|
3658
|
+
}
|
|
3659
|
+
const results = collector.assemble();
|
|
3660
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3661
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3662
|
+
return results;
|
|
3663
|
+
}
|
|
3664
|
+
function countPendingItems(config, jobs) {
|
|
3665
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3666
|
+
const db = openStore(config, "readonly");
|
|
3667
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3668
|
+
for (const job of jobs) {
|
|
3669
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3670
|
+
}
|
|
3671
|
+
db.close();
|
|
3672
|
+
return counts;
|
|
3673
|
+
}
|
|
3674
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3675
|
+
jobs.forEach((job, jobIndex) => {
|
|
3676
|
+
onProgress?.({
|
|
3677
|
+
type: "batch-submitted",
|
|
3678
|
+
name: job.job_name,
|
|
3679
|
+
count: counts.get(job.id) ?? 0,
|
|
3680
|
+
model: job.display_model,
|
|
3681
|
+
jobIndex,
|
|
3682
|
+
jobCount,
|
|
3683
|
+
resumed: true,
|
|
3684
|
+
createdAt: job.created_at
|
|
3685
|
+
});
|
|
2897
3686
|
});
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
3687
|
+
}
|
|
3688
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3689
|
+
const startedAt = Date.now();
|
|
3690
|
+
const pending = readPendingBatchWork(config);
|
|
3691
|
+
if (pending.jobs.length === 0) return null;
|
|
3692
|
+
options.onProgress?.({
|
|
3693
|
+
type: "start",
|
|
3694
|
+
total: pending.pendingItems.length,
|
|
3695
|
+
concurrency: 1,
|
|
3696
|
+
dryRun: false,
|
|
3697
|
+
mode: "batch"
|
|
3698
|
+
});
|
|
3699
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3700
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3701
|
+
const results = [];
|
|
3702
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3703
|
+
onProgress: options.onProgress,
|
|
3704
|
+
onResult: (result) => {
|
|
3705
|
+
results.push(result);
|
|
3706
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3707
|
+
}
|
|
2906
3708
|
});
|
|
2907
3709
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2908
3710
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -3004,6 +3806,6 @@ function writeStaticRawExports(project, options = {}) {
|
|
|
3004
3806
|
return { exports, written: exports.length };
|
|
3005
3807
|
}
|
|
3006
3808
|
|
|
3007
|
-
export { buildAllContentRedirects, buildStaticRawExports, buildWorklist, createProject, createScribe, createUrlBuilder, defineConfig, defineContentType, exportDirSegment, field, findConfigPath, generateSitemap, getFieldKind, getRedirectSourceSlugs, getRelationTarget, getStaticExportRoots, isResolvedConfig, isRoutableType, loadConfigSync, resolveConfig, resolveLocalesFromPreset, serializeMdx, translatePage, translateWorklist, unwrapSchema, validateProject, writeStaticRawExports };
|
|
3809
|
+
export { buildAllContentRedirects, buildStaticRawExports, buildWorklist, createProject, createScribe, createUrlBuilder, defineConfig, defineContentType, exportDirSegment, field, findConfigPath, generateSitemap, getFieldKind, getRedirectSourceSlugs, getRelationTarget, getStaticExportRoots, isResolvedConfig, isRoutableType, loadConfigSync, resolveConfig, resolveLocalesFromPreset, resumeTranslationJobs, serializeMdx, translatePage, translateWorklist, unwrapSchema, validateProject, writeStaticRawExports };
|
|
3008
3810
|
//# sourceMappingURL=index.js.map
|
|
3009
3811
|
//# sourceMappingURL=index.js.map
|