scribe-cms 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/cli/index.cjs +1258 -219
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +1259 -220
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/prompt-translate.d.ts +3 -0
- package/dist/cli/prompt-translate.d.ts.map +1 -1
- package/dist/cli/translate-progress.d.ts.map +1 -1
- package/dist/index.cjs +1020 -181
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1021 -183
- package/dist/index.js.map +1 -1
- package/dist/runtime.cjs +149 -62
- package/dist/runtime.cjs.map +1 -1
- package/dist/runtime.js +149 -62
- package/dist/runtime.js.map +1 -1
- package/dist/src/config/resolve-config.d.ts.map +1 -1
- package/dist/src/core/introspect-schema.d.ts +7 -1
- package/dist/src/core/introspect-schema.d.ts.map +1 -1
- package/dist/src/core/types.d.ts +6 -0
- package/dist/src/core/types.d.ts.map +1 -1
- package/dist/src/create-project.d.ts.map +1 -1
- package/dist/src/i18n/resolve-document.d.ts +1 -1
- package/dist/src/i18n/resolve-document.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/storage/batch-jobs.d.ts +53 -0
- package/dist/src/storage/batch-jobs.d.ts.map +1 -0
- package/dist/src/storage/sqlite.d.ts.map +1 -1
- package/dist/src/translate/batch-worklist.d.ts +87 -0
- package/dist/src/translate/batch-worklist.d.ts.map +1 -0
- package/dist/src/translate/gemini-batch.d.ts +26 -0
- package/dist/src/translate/gemini-batch.d.ts.map +1 -0
- package/dist/src/translate/gemini-client.d.ts +23 -0
- package/dist/src/translate/gemini-client.d.ts.map +1 -1
- package/dist/src/translate/gemini-models.d.ts +9 -0
- package/dist/src/translate/gemini-models.d.ts.map +1 -1
- package/dist/src/translate/gemini-pricing.d.ts +2 -1
- package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
- package/dist/src/translate/page-translator.d.ts +25 -52
- package/dist/src/translate/page-translator.d.ts.map +1 -1
- package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
- package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
- package/dist/src/translate/response-schema.d.ts +8 -1
- package/dist/src/translate/response-schema.d.ts.map +1 -1
- package/dist/src/translate/retry.d.ts +16 -0
- package/dist/src/translate/retry.d.ts.map +1 -0
- package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
- package/dist/src/translate/translate-core.d.ts +155 -0
- package/dist/src/translate/translate-core.d.ts.map +1 -0
- package/dist/src/validate/validate-project.d.ts.map +1 -1
- package/dist/studio/server.cjs +84 -47
- package/dist/studio/server.cjs.map +1 -1
- package/dist/studio/server.js +84 -47
- package/dist/studio/server.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import remarkParse from 'remark-parse';
|
|
|
8
8
|
import { unified } from 'unified';
|
|
9
9
|
import { createJiti } from 'jiti';
|
|
10
10
|
import { createHash } from 'crypto';
|
|
11
|
-
import { GoogleGenAI } from '@google/genai';
|
|
11
|
+
import { GoogleGenAI, ThinkingLevel, ApiError } from '@google/genai';
|
|
12
12
|
|
|
13
13
|
// src/core/field.ts
|
|
14
14
|
var FIELD_KIND = /* @__PURE__ */ Symbol.for("@genlook/scribe/fieldKind");
|
|
@@ -244,6 +244,39 @@ function resolveConfig(input, baseDir) {
|
|
|
244
244
|
if (localeRouting.strategy === "search-param" && !localeRouting.param) {
|
|
245
245
|
throw new Error('scribe config: localeRouting search-param requires a "param" name');
|
|
246
246
|
}
|
|
247
|
+
const localeFallbacks = {};
|
|
248
|
+
for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
|
|
249
|
+
if (!raw.locales.includes(locale)) {
|
|
250
|
+
throw new Error(
|
|
251
|
+
`scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
const seen = /* @__PURE__ */ new Set();
|
|
255
|
+
for (const fallback of chain) {
|
|
256
|
+
if (!raw.locales.includes(fallback)) {
|
|
257
|
+
throw new Error(
|
|
258
|
+
`scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
if (fallback === locale) {
|
|
262
|
+
throw new Error(
|
|
263
|
+
`scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
if (fallback === defaultLocale) {
|
|
267
|
+
throw new Error(
|
|
268
|
+
`scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
if (seen.has(fallback)) {
|
|
272
|
+
throw new Error(
|
|
273
|
+
`scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
seen.add(fallback);
|
|
277
|
+
}
|
|
278
|
+
localeFallbacks[locale] = [...chain];
|
|
279
|
+
}
|
|
247
280
|
const projectRoot = path3.resolve(baseDir ?? process.cwd(), raw.rootDir);
|
|
248
281
|
const contentRoot = path3.resolve(projectRoot, raw.contentDir ?? "content");
|
|
249
282
|
const storePath = path3.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
|
|
@@ -273,6 +306,7 @@ function resolveConfig(input, baseDir) {
|
|
|
273
306
|
defaultLocale,
|
|
274
307
|
localeRouting,
|
|
275
308
|
localePresets: raw.localePresets,
|
|
309
|
+
localeFallbacks,
|
|
276
310
|
translate: raw.translate,
|
|
277
311
|
types
|
|
278
312
|
};
|
|
@@ -284,6 +318,12 @@ function resolveConfig(input, baseDir) {
|
|
|
284
318
|
}
|
|
285
319
|
|
|
286
320
|
// src/core/introspect-schema.ts
|
|
321
|
+
function getArrayElement(schema) {
|
|
322
|
+
if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
|
|
323
|
+
return schema.element;
|
|
324
|
+
}
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
287
327
|
function introspectSchema(schema, prefix = []) {
|
|
288
328
|
const relation = getRelationTarget(schema);
|
|
289
329
|
if (relation && prefix.length > 0) {
|
|
@@ -297,7 +337,10 @@ function introspectSchema(schema, prefix = []) {
|
|
|
297
337
|
}
|
|
298
338
|
];
|
|
299
339
|
}
|
|
300
|
-
|
|
340
|
+
if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
|
|
341
|
+
return [{ path: prefix, kind: "translatable" }];
|
|
342
|
+
}
|
|
343
|
+
const base = peelOptionalWrappers(schema);
|
|
301
344
|
if (base instanceof Object && "shape" in base) {
|
|
302
345
|
const shape = base.shape;
|
|
303
346
|
const fields = [];
|
|
@@ -306,9 +349,9 @@ function introspectSchema(schema, prefix = []) {
|
|
|
306
349
|
}
|
|
307
350
|
return fields;
|
|
308
351
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
const elementBase =
|
|
352
|
+
const element = getArrayElement(base);
|
|
353
|
+
if (element) {
|
|
354
|
+
const elementBase = peelOptionalWrappers(element);
|
|
312
355
|
if (elementBase instanceof Object && "shape" in elementBase) {
|
|
313
356
|
const shape = elementBase.shape;
|
|
314
357
|
const fields = [];
|
|
@@ -320,15 +363,48 @@ function introspectSchema(schema, prefix = []) {
|
|
|
320
363
|
}
|
|
321
364
|
return [{ path: prefix, kind: getFieldKind(schema) }];
|
|
322
365
|
}
|
|
323
|
-
function
|
|
324
|
-
const
|
|
366
|
+
function buildPathTrie(paths) {
|
|
367
|
+
const root = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
325
368
|
for (const path13 of paths) {
|
|
326
|
-
|
|
369
|
+
let node = root;
|
|
370
|
+
for (const segment of path13) {
|
|
371
|
+
let child = node.children.get(segment);
|
|
372
|
+
if (!child) {
|
|
373
|
+
child = { leaf: false, children: /* @__PURE__ */ new Map() };
|
|
374
|
+
node.children.set(segment, child);
|
|
375
|
+
}
|
|
376
|
+
node = child;
|
|
377
|
+
}
|
|
378
|
+
node.leaf = true;
|
|
379
|
+
}
|
|
380
|
+
return root;
|
|
381
|
+
}
|
|
382
|
+
function extractNode(data, trie) {
|
|
383
|
+
if (trie.leaf) return data;
|
|
384
|
+
const star = trie.children.get("*");
|
|
385
|
+
if (star) {
|
|
386
|
+
if (!Array.isArray(data)) return void 0;
|
|
387
|
+
return data.map(
|
|
388
|
+
(item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
|
|
392
|
+
const record = data;
|
|
393
|
+
const out = {};
|
|
394
|
+
let any = false;
|
|
395
|
+
for (const [key, child] of trie.children) {
|
|
396
|
+
const value = extractNode(record[key], child);
|
|
327
397
|
if (value !== void 0) {
|
|
328
|
-
|
|
398
|
+
out[key] = value;
|
|
399
|
+
any = true;
|
|
329
400
|
}
|
|
330
401
|
}
|
|
331
|
-
return out;
|
|
402
|
+
return any ? out : void 0;
|
|
403
|
+
}
|
|
404
|
+
function extractByPaths(data, paths) {
|
|
405
|
+
if (paths.length === 0) return {};
|
|
406
|
+
const extracted = extractNode(data, buildPathTrie(paths));
|
|
407
|
+
return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
|
|
332
408
|
}
|
|
333
409
|
function pickTranslatable(data, schema) {
|
|
334
410
|
const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
|
|
@@ -343,32 +419,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
|
|
|
343
419
|
const translatable = pickTranslatable(localeData, schema);
|
|
344
420
|
return deepMerge(structural, translatable);
|
|
345
421
|
}
|
|
346
|
-
function getAtPath(obj, path13) {
|
|
347
|
-
let current = obj;
|
|
348
|
-
for (const segment of path13) {
|
|
349
|
-
if (segment === "*") {
|
|
350
|
-
if (!Array.isArray(current)) return void 0;
|
|
351
|
-
return current.map(
|
|
352
|
-
(item) => typeof item === "object" && item !== null ? getAtPath(item, path13.slice(path13.indexOf("*") + 1)) : void 0
|
|
353
|
-
);
|
|
354
|
-
}
|
|
355
|
-
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
356
|
-
current = current[segment];
|
|
357
|
-
}
|
|
358
|
-
return current;
|
|
359
|
-
}
|
|
360
|
-
function setAtPath(obj, path13, value) {
|
|
361
|
-
if (path13.length === 0) return;
|
|
362
|
-
if (path13.length === 1) {
|
|
363
|
-
obj[path13[0]] = value;
|
|
364
|
-
return;
|
|
365
|
-
}
|
|
366
|
-
const [head, ...rest] = path13;
|
|
367
|
-
if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
|
|
368
|
-
obj[head] = {};
|
|
369
|
-
}
|
|
370
|
-
setAtPath(obj[head], rest, value);
|
|
371
|
-
}
|
|
372
422
|
function deepMerge(base, overlay) {
|
|
373
423
|
const out = { ...base };
|
|
374
424
|
for (const [key, value] of Object.entries(overlay)) {
|
|
@@ -538,7 +588,7 @@ function seoFieldsFromEn(enDoc) {
|
|
|
538
588
|
canonicalPathOverride: enDoc.canonicalPathOverride
|
|
539
589
|
};
|
|
540
590
|
}
|
|
541
|
-
var SCHEMA_VERSION =
|
|
591
|
+
var SCHEMA_VERSION = 5;
|
|
542
592
|
var MIGRATIONS = [
|
|
543
593
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
544
594
|
key TEXT PRIMARY KEY,
|
|
@@ -586,7 +636,30 @@ var MIGRATIONS = [
|
|
|
586
636
|
UNIQUE (content_type, en_slug, en_hash)
|
|
587
637
|
)`,
|
|
588
638
|
`CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
|
|
589
|
-
ON en_snapshots(content_type, en_slug, created_at DESC)
|
|
639
|
+
ON en_snapshots(content_type, en_slug, created_at DESC)`,
|
|
640
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_jobs (
|
|
641
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
642
|
+
job_name TEXT NOT NULL UNIQUE,
|
|
643
|
+
model TEXT NOT NULL,
|
|
644
|
+
display_model TEXT NOT NULL,
|
|
645
|
+
created_at TEXT NOT NULL,
|
|
646
|
+
state TEXT NOT NULL,
|
|
647
|
+
completed_at TEXT
|
|
648
|
+
)`,
|
|
649
|
+
`CREATE TABLE IF NOT EXISTS translation_batch_items (
|
|
650
|
+
job_id INTEGER NOT NULL,
|
|
651
|
+
request_index INTEGER NOT NULL,
|
|
652
|
+
content_type TEXT NOT NULL,
|
|
653
|
+
en_slug TEXT NOT NULL,
|
|
654
|
+
locale TEXT NOT NULL,
|
|
655
|
+
en_hash TEXT NOT NULL,
|
|
656
|
+
snapshot_id INTEGER NOT NULL,
|
|
657
|
+
status TEXT NOT NULL,
|
|
658
|
+
error TEXT,
|
|
659
|
+
PRIMARY KEY (job_id, request_index)
|
|
660
|
+
)`,
|
|
661
|
+
`CREATE INDEX IF NOT EXISTS idx_batch_items_status
|
|
662
|
+
ON translation_batch_items(status)`
|
|
590
663
|
];
|
|
591
664
|
function resolveStorePath(config) {
|
|
592
665
|
return config.storePath;
|
|
@@ -609,6 +682,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
|
|
|
609
682
|
}
|
|
610
683
|
}
|
|
611
684
|
function readSchemaVersion(db) {
|
|
685
|
+
const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
|
|
686
|
+
if (!metaTable) return 0;
|
|
612
687
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
613
688
|
return row ? Number.parseInt(row.value, 10) || 0 : 0;
|
|
614
689
|
}
|
|
@@ -647,6 +722,9 @@ function getOrCreateEnSnapshot(db, input) {
|
|
|
647
722
|
).get(input.contentType, input.enSlug, input.enHash);
|
|
648
723
|
return row.id;
|
|
649
724
|
}
|
|
725
|
+
function getEnSnapshot(db, snapshotId) {
|
|
726
|
+
return db.prepare(`SELECT * FROM en_snapshots WHERE id = ?`).get(snapshotId);
|
|
727
|
+
}
|
|
650
728
|
function upsertTranslation(db, input) {
|
|
651
729
|
db.prepare(
|
|
652
730
|
`INSERT INTO translations (
|
|
@@ -727,7 +805,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
727
805
|
if (i >= body.length || body[i] === ">" || body[i] === "/") break;
|
|
728
806
|
const attrStart = i;
|
|
729
807
|
while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
|
|
730
|
-
body.slice(attrStart, i);
|
|
808
|
+
const attrName = body.slice(attrStart, i);
|
|
809
|
+
if (!attrName) break;
|
|
731
810
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
732
811
|
if (body[i] !== "=") {
|
|
733
812
|
tagOut += body.slice(attrStart, i);
|
|
@@ -736,10 +815,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
|
|
|
736
815
|
tagOut += body.slice(attrStart, i);
|
|
737
816
|
tagOut += "=";
|
|
738
817
|
i += 1;
|
|
818
|
+
const wsStart = i;
|
|
739
819
|
while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
|
|
820
|
+
tagOut += body.slice(wsStart, i);
|
|
740
821
|
const quote = body[i];
|
|
741
822
|
if (quote !== '"' && quote !== "'") {
|
|
742
|
-
|
|
823
|
+
break;
|
|
743
824
|
}
|
|
744
825
|
if (quote === "'") {
|
|
745
826
|
const valStart2 = i + 1;
|
|
@@ -1059,7 +1140,7 @@ function getTranslatablePayload(doc, type) {
|
|
|
1059
1140
|
}
|
|
1060
1141
|
|
|
1061
1142
|
// src/i18n/resolve-document.ts
|
|
1062
|
-
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
|
|
1143
|
+
function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
|
|
1063
1144
|
const urlBuilder = createUrlBuilder({
|
|
1064
1145
|
locales: [defaultLocale, locale],
|
|
1065
1146
|
defaultLocale,
|
|
@@ -1073,15 +1154,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1073
1154
|
for (const [docLocale, docIdx] of allDocs) {
|
|
1074
1155
|
const found = docIdx.bySlug.get(slug);
|
|
1075
1156
|
if (!found) continue;
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1157
|
+
const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
|
|
1158
|
+
let target;
|
|
1159
|
+
let targetLocale = locale;
|
|
1160
|
+
for (const candidateLocale of [locale, ...fallbackLocales]) {
|
|
1161
|
+
const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
|
|
1162
|
+
if (cand) {
|
|
1163
|
+
target = cand;
|
|
1164
|
+
targetLocale = candidateLocale;
|
|
1165
|
+
break;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
if (target) {
|
|
1169
|
+
if (target.slug === slug) {
|
|
1170
|
+
return { document: target, actualLocale: targetLocale };
|
|
1171
|
+
}
|
|
1078
1172
|
if (!type.path) {
|
|
1079
1173
|
return { document: null, actualLocale: locale };
|
|
1080
1174
|
}
|
|
1081
1175
|
return {
|
|
1082
1176
|
document: null,
|
|
1083
1177
|
actualLocale: locale,
|
|
1084
|
-
shouldRedirectTo: urlBuilder.resolvePath(type.path,
|
|
1178
|
+
shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
|
|
1085
1179
|
};
|
|
1086
1180
|
}
|
|
1087
1181
|
if (docLocale === defaultLocale) {
|
|
@@ -1112,13 +1206,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
|
|
|
1112
1206
|
}
|
|
1113
1207
|
return { document: null, actualLocale: locale };
|
|
1114
1208
|
}
|
|
1115
|
-
function getSlugForLocale(document, sourceLocale, targetLocale, allDocs, defaultLocale) {
|
|
1116
|
-
if (sourceLocale === targetLocale) return document.slug;
|
|
1117
|
-
const englishSlug = sourceLocale === defaultLocale ? document.slug : document.enSlug;
|
|
1118
|
-
if (!englishSlug) return null;
|
|
1119
|
-
if (targetLocale === defaultLocale) return englishSlug;
|
|
1120
|
-
return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
|
|
1121
|
-
}
|
|
1122
1209
|
|
|
1123
1210
|
// src/create-project.ts
|
|
1124
1211
|
function comparatorFor(orderBy) {
|
|
@@ -1171,7 +1258,8 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1171
1258
|
config.defaultLocale,
|
|
1172
1259
|
load(),
|
|
1173
1260
|
type,
|
|
1174
|
-
config.localeRouting
|
|
1261
|
+
config.localeRouting,
|
|
1262
|
+
config.localeFallbacks[locale] ?? []
|
|
1175
1263
|
);
|
|
1176
1264
|
if (result.document && type.path) {
|
|
1177
1265
|
return {
|
|
@@ -1193,8 +1281,14 @@ function buildRuntime(config, type, getRuntime) {
|
|
|
1193
1281
|
const params = [];
|
|
1194
1282
|
for (const locale of options.locales ?? config.locales) {
|
|
1195
1283
|
const localeIdx = all.get(locale);
|
|
1284
|
+
const fallbacks = config.localeFallbacks[locale] ?? [];
|
|
1196
1285
|
for (const doc of enIdx.bySlug.values()) {
|
|
1197
|
-
|
|
1286
|
+
let slug;
|
|
1287
|
+
if (locale === config.defaultLocale) {
|
|
1288
|
+
slug = doc.slug;
|
|
1289
|
+
} else {
|
|
1290
|
+
slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
|
|
1291
|
+
}
|
|
1198
1292
|
params.push({ locale, slug });
|
|
1199
1293
|
}
|
|
1200
1294
|
}
|
|
@@ -2013,7 +2107,7 @@ function validateTypeRedirects(project) {
|
|
|
2013
2107
|
}
|
|
2014
2108
|
return issues;
|
|
2015
2109
|
}
|
|
2016
|
-
function
|
|
2110
|
+
function getAtPath(obj, fieldPath) {
|
|
2017
2111
|
let current = obj;
|
|
2018
2112
|
for (const segment of fieldPath) {
|
|
2019
2113
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
|
|
@@ -2058,7 +2152,7 @@ function validateRelations(project) {
|
|
|
2058
2152
|
});
|
|
2059
2153
|
continue;
|
|
2060
2154
|
}
|
|
2061
|
-
const value =
|
|
2155
|
+
const value = getAtPath(doc.frontmatter, fieldMeta.path);
|
|
2062
2156
|
if (value === void 0 || value === null || value === "") continue;
|
|
2063
2157
|
const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
|
|
2064
2158
|
const fieldLabel = fieldMeta.path.join(".");
|
|
@@ -2207,6 +2301,20 @@ function listEnSlugs3(rootDir, contentDir) {
|
|
|
2207
2301
|
if (!fs2.existsSync(dir)) return [];
|
|
2208
2302
|
return fs2.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
|
|
2209
2303
|
}
|
|
2304
|
+
function validateDocumentMdxBody(issues, input) {
|
|
2305
|
+
const preparedBody = prepareTranslatedMdxBody(input.body).body;
|
|
2306
|
+
const mdxValidation = validateMdxBody(preparedBody);
|
|
2307
|
+
if (!mdxValidation.ok) {
|
|
2308
|
+
issues.push({
|
|
2309
|
+
level: "error",
|
|
2310
|
+
contentType: input.contentType,
|
|
2311
|
+
enSlug: input.enSlug,
|
|
2312
|
+
locale: input.locale,
|
|
2313
|
+
field: "body",
|
|
2314
|
+
message: `Invalid MDX: ${mdxValidation.error}`
|
|
2315
|
+
});
|
|
2316
|
+
}
|
|
2317
|
+
}
|
|
2210
2318
|
function validateProject(config) {
|
|
2211
2319
|
const issues = [];
|
|
2212
2320
|
const project = createProject(config);
|
|
@@ -2265,6 +2373,12 @@ function validateProject(config) {
|
|
|
2265
2373
|
})) {
|
|
2266
2374
|
issues.push(issue);
|
|
2267
2375
|
}
|
|
2376
|
+
validateDocumentMdxBody(issues, {
|
|
2377
|
+
contentType: type.id,
|
|
2378
|
+
enSlug,
|
|
2379
|
+
locale: config.defaultLocale,
|
|
2380
|
+
body: enDoc.content
|
|
2381
|
+
});
|
|
2268
2382
|
for (const locale of config.locales) {
|
|
2269
2383
|
if (locale === config.defaultLocale) continue;
|
|
2270
2384
|
const row = getTranslation(db, type.id, enSlug, locale);
|
|
@@ -2290,17 +2404,12 @@ function validateProject(config) {
|
|
|
2290
2404
|
})) {
|
|
2291
2405
|
issues.push(issue);
|
|
2292
2406
|
}
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
locale,
|
|
2300
|
-
field: "body",
|
|
2301
|
-
message: `Invalid MDX: ${mdxValidation.error}`
|
|
2302
|
-
});
|
|
2303
|
-
}
|
|
2407
|
+
validateDocumentMdxBody(issues, {
|
|
2408
|
+
contentType: type.id,
|
|
2409
|
+
enSlug,
|
|
2410
|
+
locale,
|
|
2411
|
+
body: row.body
|
|
2412
|
+
});
|
|
2304
2413
|
}
|
|
2305
2414
|
}
|
|
2306
2415
|
try {
|
|
@@ -2410,6 +2519,59 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
|
|
|
2410
2519
|
return config.locales.filter((l) => l !== config.defaultLocale);
|
|
2411
2520
|
}
|
|
2412
2521
|
|
|
2522
|
+
// src/storage/batch-jobs.ts
|
|
2523
|
+
function insertBatchJob(db, input) {
|
|
2524
|
+
const info = db.prepare(
|
|
2525
|
+
`INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
|
|
2526
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
2527
|
+
).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
|
|
2528
|
+
return Number(info.lastInsertRowid);
|
|
2529
|
+
}
|
|
2530
|
+
function insertBatchItems(db, jobId, items) {
|
|
2531
|
+
const stmt = db.prepare(
|
|
2532
|
+
`INSERT INTO translation_batch_items (
|
|
2533
|
+
job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
|
|
2534
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
|
|
2535
|
+
);
|
|
2536
|
+
for (const item of items) {
|
|
2537
|
+
stmt.run(
|
|
2538
|
+
jobId,
|
|
2539
|
+
item.requestIndex,
|
|
2540
|
+
item.contentType,
|
|
2541
|
+
item.enSlug,
|
|
2542
|
+
item.locale,
|
|
2543
|
+
item.enHash,
|
|
2544
|
+
item.snapshotId
|
|
2545
|
+
);
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
function listPendingBatchJobs(db) {
|
|
2549
|
+
return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
|
|
2550
|
+
}
|
|
2551
|
+
function listBatchItems(db, jobId) {
|
|
2552
|
+
return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
|
|
2553
|
+
}
|
|
2554
|
+
function listPendingBatchItems(db) {
|
|
2555
|
+
return db.prepare(
|
|
2556
|
+
`SELECT i.* FROM translation_batch_items i
|
|
2557
|
+
JOIN translation_batch_jobs j ON j.id = i.job_id
|
|
2558
|
+
WHERE j.completed_at IS NULL AND i.status = 'pending'
|
|
2559
|
+
ORDER BY i.job_id, i.request_index`
|
|
2560
|
+
).all();
|
|
2561
|
+
}
|
|
2562
|
+
function claimBatchJobCompletion(db, jobId, state, completedAt) {
|
|
2563
|
+
const info = db.prepare(
|
|
2564
|
+
`UPDATE translation_batch_jobs SET state = ?, completed_at = ?
|
|
2565
|
+
WHERE id = ? AND completed_at IS NULL`
|
|
2566
|
+
).run(state, completedAt, jobId);
|
|
2567
|
+
return info.changes > 0;
|
|
2568
|
+
}
|
|
2569
|
+
function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
|
|
2570
|
+
db.prepare(
|
|
2571
|
+
`UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
|
|
2572
|
+
).run(status, error ?? null, jobId, requestIndex);
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2413
2575
|
// src/history/record-snapshot.ts
|
|
2414
2576
|
function recordEnSnapshot(config, input, db) {
|
|
2415
2577
|
const ownDb = db ?? openStore(config, "readwrite");
|
|
@@ -2424,8 +2586,102 @@ function recordEnSnapshot(config, input, db) {
|
|
|
2424
2586
|
if (!db) ownDb.close();
|
|
2425
2587
|
return id;
|
|
2426
2588
|
}
|
|
2589
|
+
var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
2590
|
+
function statusFromError(error) {
|
|
2591
|
+
if (error instanceof ApiError && typeof error.status === "number") {
|
|
2592
|
+
return error.status;
|
|
2593
|
+
}
|
|
2594
|
+
if (error && typeof error === "object" && "status" in error) {
|
|
2595
|
+
const status = error.status;
|
|
2596
|
+
if (typeof status === "number") return status;
|
|
2597
|
+
}
|
|
2598
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2599
|
+
const match = message.match(/\b(429|500|502|503|504)\b/);
|
|
2600
|
+
if (match) return Number(match[1]);
|
|
2601
|
+
return void 0;
|
|
2602
|
+
}
|
|
2603
|
+
function isNetworkError(error) {
|
|
2604
|
+
const err = error;
|
|
2605
|
+
const code = typeof err?.code === "string" ? err.code : "";
|
|
2606
|
+
if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
|
|
2607
|
+
return true;
|
|
2608
|
+
}
|
|
2609
|
+
const message = error instanceof Error ? error.message.toLowerCase() : "";
|
|
2610
|
+
return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
|
|
2611
|
+
}
|
|
2612
|
+
function isTransientError(error) {
|
|
2613
|
+
const status = statusFromError(error);
|
|
2614
|
+
if (status !== void 0) return TRANSIENT_STATUS.has(status);
|
|
2615
|
+
return isNetworkError(error);
|
|
2616
|
+
}
|
|
2617
|
+
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2618
|
+
async function withRetry(fn, options = {}) {
|
|
2619
|
+
const attempts = Math.max(1, options.attempts ?? 3);
|
|
2620
|
+
const baseDelayMs = options.baseDelayMs ?? 1e3;
|
|
2621
|
+
const sleep2 = options.sleep ?? defaultSleep;
|
|
2622
|
+
const random = options.random ?? Math.random;
|
|
2623
|
+
let lastError;
|
|
2624
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
2625
|
+
try {
|
|
2626
|
+
return await fn();
|
|
2627
|
+
} catch (error) {
|
|
2628
|
+
lastError = error;
|
|
2629
|
+
if (attempt >= attempts || !isTransientError(error)) throw error;
|
|
2630
|
+
const backoff = baseDelayMs * 2 ** (attempt - 1);
|
|
2631
|
+
const delay = backoff + random() * backoff;
|
|
2632
|
+
await sleep2(delay);
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
throw lastError;
|
|
2636
|
+
}
|
|
2427
2637
|
|
|
2428
|
-
// src/translate/gemini-
|
|
2638
|
+
// src/translate/gemini-batch.ts
|
|
2639
|
+
var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
|
|
2640
|
+
var TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
2641
|
+
"SUCCEEDED",
|
|
2642
|
+
"FAILED",
|
|
2643
|
+
"CANCELLED",
|
|
2644
|
+
"EXPIRED",
|
|
2645
|
+
"PARTIALLY_SUCCEEDED"
|
|
2646
|
+
]);
|
|
2647
|
+
var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
|
|
2648
|
+
function normalizeBatchState(state) {
|
|
2649
|
+
return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
|
|
2650
|
+
}
|
|
2651
|
+
function isTerminalBatchState(state) {
|
|
2652
|
+
return TERMINAL_STATES.has(normalizeBatchState(state));
|
|
2653
|
+
}
|
|
2654
|
+
function isSuccessfulBatchState(state) {
|
|
2655
|
+
return SUCCESSFUL_STATES.has(normalizeBatchState(state));
|
|
2656
|
+
}
|
|
2657
|
+
function makeClient(apiKey) {
|
|
2658
|
+
const key = apiKey ?? process.env.GEMINI_API_KEY;
|
|
2659
|
+
if (!key) {
|
|
2660
|
+
throw new Error("GEMINI_API_KEY is required for scribe translate");
|
|
2661
|
+
}
|
|
2662
|
+
return new GoogleGenAI({ apiKey: key });
|
|
2663
|
+
}
|
|
2664
|
+
function textFromBatchResponse(response) {
|
|
2665
|
+
if (typeof response.text === "string") return response.text;
|
|
2666
|
+
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2667
|
+
return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
|
|
2668
|
+
}
|
|
2669
|
+
async function createGeminiBatchJob(input) {
|
|
2670
|
+
const ai = makeClient(input.apiKey);
|
|
2671
|
+
const job = await withRetry(
|
|
2672
|
+
() => ai.batches.create({
|
|
2673
|
+
model: input.model,
|
|
2674
|
+
src: input.requests,
|
|
2675
|
+
config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
|
|
2676
|
+
})
|
|
2677
|
+
);
|
|
2678
|
+
if (!job.name) throw new Error("Gemini batch job was created without a name");
|
|
2679
|
+
return job;
|
|
2680
|
+
}
|
|
2681
|
+
async function getGeminiBatchJob(input) {
|
|
2682
|
+
const ai = makeClient(input.apiKey);
|
|
2683
|
+
return withRetry(() => ai.batches.get({ name: input.name }));
|
|
2684
|
+
}
|
|
2429
2685
|
var GEMINI_MODEL_IDS = {
|
|
2430
2686
|
"gemini-2.5-pro": "gemini-2.5-pro",
|
|
2431
2687
|
"gemini-3.1-pro": "gemini-3.1-pro-preview"
|
|
@@ -2445,16 +2701,55 @@ function normalizeGeminiDisplayName(model) {
|
|
|
2445
2701
|
if (alias) return alias[0];
|
|
2446
2702
|
return name;
|
|
2447
2703
|
}
|
|
2704
|
+
function resolveThinkingConfig(apiModelId) {
|
|
2705
|
+
const id = stripModelsPrefix(apiModelId).toLowerCase();
|
|
2706
|
+
if (id.startsWith("gemini-3")) return { thinkingLevel: ThinkingLevel.LOW };
|
|
2707
|
+
if (id.includes("2.5")) return { thinkingBudget: 128 };
|
|
2708
|
+
return void 0;
|
|
2709
|
+
}
|
|
2448
2710
|
|
|
2449
2711
|
// src/translate/gemini-client.ts
|
|
2450
2712
|
var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
|
|
2451
2713
|
function extractJson(text) {
|
|
2452
|
-
const
|
|
2714
|
+
const trimmed = text.trim();
|
|
2715
|
+
const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
|
2453
2716
|
if (fenced?.[1]) return fenced[1].trim();
|
|
2454
|
-
const start =
|
|
2455
|
-
const end =
|
|
2456
|
-
if (start >= 0 && end > start) return
|
|
2457
|
-
return
|
|
2717
|
+
const start = trimmed.indexOf("{");
|
|
2718
|
+
const end = trimmed.lastIndexOf("}");
|
|
2719
|
+
if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
|
|
2720
|
+
return trimmed;
|
|
2721
|
+
}
|
|
2722
|
+
function parseGeminiResponse(text) {
|
|
2723
|
+
let parsed;
|
|
2724
|
+
try {
|
|
2725
|
+
parsed = JSON.parse(text.trim());
|
|
2726
|
+
} catch {
|
|
2727
|
+
parsed = JSON.parse(extractJson(text));
|
|
2728
|
+
}
|
|
2729
|
+
if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
|
|
2730
|
+
parsed.frontmatter = {};
|
|
2731
|
+
}
|
|
2732
|
+
return parsed;
|
|
2733
|
+
}
|
|
2734
|
+
function buildGeminiRequestConfig(input) {
|
|
2735
|
+
const thinkingConfig = resolveThinkingConfig(input.apiModelId);
|
|
2736
|
+
return {
|
|
2737
|
+
responseMimeType: "application/json",
|
|
2738
|
+
...input.responseSchema ? { responseSchema: input.responseSchema } : {},
|
|
2739
|
+
...thinkingConfig ? { thinkingConfig } : {}
|
|
2740
|
+
};
|
|
2741
|
+
}
|
|
2742
|
+
function usageFromResponse(response) {
|
|
2743
|
+
const meta = response.usageMetadata ?? response.usage_metadata ?? {};
|
|
2744
|
+
const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
|
|
2745
|
+
return {
|
|
2746
|
+
inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
|
|
2747
|
+
// candidatesTokenCount does not include thoughts, but thoughts are billed as
|
|
2748
|
+
// output tokens, so fold them in here for accurate cost accounting.
|
|
2749
|
+
outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
|
|
2750
|
+
thoughtsTokens,
|
|
2751
|
+
totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
|
|
2752
|
+
};
|
|
2458
2753
|
}
|
|
2459
2754
|
async function translatePageWithGemini(input) {
|
|
2460
2755
|
const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
|
|
@@ -2466,26 +2761,22 @@ async function translatePageWithGemini(input) {
|
|
|
2466
2761
|
);
|
|
2467
2762
|
const apiModel = resolveGeminiModelId(displayModel);
|
|
2468
2763
|
const ai = new GoogleGenAI({ apiKey });
|
|
2469
|
-
const response = await
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2764
|
+
const response = await withRetry(
|
|
2765
|
+
() => ai.models.generateContent({
|
|
2766
|
+
model: apiModel,
|
|
2767
|
+
contents: input.prompt,
|
|
2768
|
+
config: buildGeminiRequestConfig({
|
|
2769
|
+
apiModelId: apiModel,
|
|
2770
|
+
responseSchema: input.responseSchema
|
|
2771
|
+
})
|
|
2772
|
+
})
|
|
2773
|
+
);
|
|
2477
2774
|
const raw = response.text ?? "";
|
|
2478
|
-
const parsed =
|
|
2775
|
+
const parsed = parseGeminiResponse(raw);
|
|
2479
2776
|
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
2480
2777
|
throw new Error("Gemini response missing frontmatter/body");
|
|
2481
2778
|
}
|
|
2482
|
-
|
|
2483
|
-
const usage = {
|
|
2484
|
-
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
2485
|
-
outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
|
|
2486
|
-
totalTokens: usageMetadata?.totalTokenCount ?? 0
|
|
2487
|
-
};
|
|
2488
|
-
return { model: displayModel, raw, parsed, usage };
|
|
2779
|
+
return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
|
|
2489
2780
|
}
|
|
2490
2781
|
|
|
2491
2782
|
// src/translate/gemini-pricing.ts
|
|
@@ -2505,12 +2796,14 @@ function resolveModelPricing(model) {
|
|
|
2505
2796
|
const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
|
|
2506
2797
|
return match?.[1];
|
|
2507
2798
|
}
|
|
2508
|
-
|
|
2799
|
+
var BATCH_DISCOUNT = 0.5;
|
|
2800
|
+
function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
|
|
2509
2801
|
const pricing = resolveModelPricing(model);
|
|
2510
2802
|
if (!pricing) return void 0;
|
|
2511
2803
|
const inputRate = tierRate(inputTokens, pricing.input);
|
|
2512
2804
|
const outputRate = tierRate(outputTokens, pricing.output);
|
|
2513
|
-
|
|
2805
|
+
const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
|
|
2806
|
+
return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
|
|
2514
2807
|
}
|
|
2515
2808
|
|
|
2516
2809
|
// src/translate/prompts/translation-prompt.ts
|
|
@@ -2526,6 +2819,7 @@ var LOCALE_NAMES = {
|
|
|
2526
2819
|
"zh-CN": "Simplified Chinese",
|
|
2527
2820
|
"zh-TW": "Traditional Chinese",
|
|
2528
2821
|
pt: "Portuguese",
|
|
2822
|
+
"pt-BR": "Brazilian Portuguese",
|
|
2529
2823
|
ja: "Japanese",
|
|
2530
2824
|
ru: "Russian",
|
|
2531
2825
|
it: "Italian",
|
|
@@ -2533,29 +2827,29 @@ var LOCALE_NAMES = {
|
|
|
2533
2827
|
};
|
|
2534
2828
|
function defaultLocalizationPrompt(localeName, locale) {
|
|
2535
2829
|
return [
|
|
2536
|
-
`Localize the content
|
|
2830
|
+
`Localize the content above into natural ${localeName} (${locale}).`,
|
|
2537
2831
|
"Do not translate word-for-word.",
|
|
2538
2832
|
"Preserve the source tone and brand voice.",
|
|
2539
2833
|
"Write as if a native speaker authored it for the target market."
|
|
2540
2834
|
].join(" ");
|
|
2541
2835
|
}
|
|
2836
|
+
var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
|
|
2837
|
+
function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
|
|
2838
|
+
if (!hasFrontmatter) {
|
|
2839
|
+
return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
|
|
2840
|
+
}
|
|
2841
|
+
return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
|
|
2842
|
+
}
|
|
2542
2843
|
function buildPageTranslationPrompt(input) {
|
|
2543
2844
|
const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
|
|
2544
|
-
const
|
|
2545
|
-
const
|
|
2546
|
-
|
|
2845
|
+
const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
|
|
2846
|
+
const prefix = [
|
|
2847
|
+
TASK_FRAMING,
|
|
2547
2848
|
"",
|
|
2548
2849
|
...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
|
|
2549
2850
|
...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
|
|
2550
2851
|
"## Rules",
|
|
2551
2852
|
...input.resolved.rules.map((rule) => `- ${rule}`),
|
|
2552
|
-
...input.slugStrategy === "localized" ? [
|
|
2553
|
-
`- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
|
|
2554
|
-
] : [],
|
|
2555
|
-
"",
|
|
2556
|
-
"## Output format",
|
|
2557
|
-
"Return ONLY valid JSON with keys:",
|
|
2558
|
-
input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
|
|
2559
2853
|
"",
|
|
2560
2854
|
"## EN translatable frontmatter (JSON)",
|
|
2561
2855
|
JSON.stringify(input.translatableFrontmatter, null, 2),
|
|
@@ -2563,7 +2857,22 @@ function buildPageTranslationPrompt(input) {
|
|
|
2563
2857
|
"## EN body (MDX)",
|
|
2564
2858
|
input.enBody
|
|
2565
2859
|
];
|
|
2566
|
-
|
|
2860
|
+
const suffix = [
|
|
2861
|
+
"",
|
|
2862
|
+
"## Target language",
|
|
2863
|
+
localizationPrompt,
|
|
2864
|
+
...input.slugStrategy === "localized" ? [
|
|
2865
|
+
`The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
|
|
2866
|
+
] : [],
|
|
2867
|
+
"",
|
|
2868
|
+
"## Output format",
|
|
2869
|
+
"Return ONLY valid JSON with keys:",
|
|
2870
|
+
buildOutputFormatLine(
|
|
2871
|
+
Object.keys(input.translatableFrontmatter).length > 0,
|
|
2872
|
+
input.slugStrategy
|
|
2873
|
+
)
|
|
2874
|
+
];
|
|
2875
|
+
return [...prefix, ...suffix].join("\n");
|
|
2567
2876
|
}
|
|
2568
2877
|
function getObjectShape(schema) {
|
|
2569
2878
|
const base = peelOptionalWrappers(schema);
|
|
@@ -2611,9 +2920,8 @@ function buildTranslatableSubschema(schema) {
|
|
|
2611
2920
|
function buildGeminiResponseSchema(schema, slugStrategy) {
|
|
2612
2921
|
try {
|
|
2613
2922
|
const translatable = buildTranslatableSubschema(schema);
|
|
2614
|
-
if (!translatable) return null;
|
|
2615
2923
|
const responseShape = {
|
|
2616
|
-
frontmatter: translatable,
|
|
2924
|
+
...translatable ? { frontmatter: translatable } : {},
|
|
2617
2925
|
body: z.string()
|
|
2618
2926
|
};
|
|
2619
2927
|
if (slugStrategy === "localized") {
|
|
@@ -2692,7 +3000,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
|
|
|
2692
3000
|
return { ok: true, frontmatter };
|
|
2693
3001
|
}
|
|
2694
3002
|
|
|
2695
|
-
// src/translate/
|
|
3003
|
+
// src/translate/translate-core.ts
|
|
3004
|
+
function translationItemKey(item) {
|
|
3005
|
+
const contentType = item.contentType ?? item.content_type ?? "";
|
|
3006
|
+
const enSlug = item.enSlug ?? item.en_slug ?? "";
|
|
3007
|
+
return `${contentType}\0${enSlug}\0${item.locale}`;
|
|
3008
|
+
}
|
|
2696
3009
|
function summarizeResults(results, durationMs) {
|
|
2697
3010
|
return results.reduce(
|
|
2698
3011
|
(totals, result) => {
|
|
@@ -2728,17 +3041,6 @@ function formatTranslateError(error) {
|
|
|
2728
3041
|
}
|
|
2729
3042
|
return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
|
|
2730
3043
|
}
|
|
2731
|
-
async function runPool(items, concurrency, worker) {
|
|
2732
|
-
if (items.length === 0) return;
|
|
2733
|
-
let nextIndex = 0;
|
|
2734
|
-
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
2735
|
-
while (nextIndex < items.length) {
|
|
2736
|
-
const index = nextIndex++;
|
|
2737
|
-
await worker(items[index], index);
|
|
2738
|
-
}
|
|
2739
|
-
});
|
|
2740
|
-
await Promise.all(runners);
|
|
2741
|
-
}
|
|
2742
3044
|
function resolveContextLabel(enDoc, enSlug) {
|
|
2743
3045
|
const fm = enDoc.frontmatter;
|
|
2744
3046
|
for (const key of ["title", "name", "h1"]) {
|
|
@@ -2747,73 +3049,82 @@ function resolveContextLabel(enDoc, enSlug) {
|
|
|
2747
3049
|
}
|
|
2748
3050
|
return enSlug;
|
|
2749
3051
|
}
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
const base = {
|
|
3052
|
+
function baseForItem(item) {
|
|
3053
|
+
return {
|
|
2753
3054
|
contentType: item.contentType,
|
|
2754
3055
|
enSlug: item.enSlug,
|
|
2755
3056
|
locale: item.locale
|
|
2756
3057
|
};
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
3058
|
+
}
|
|
3059
|
+
function prepareTranslation(config, item, options, startedAt) {
|
|
3060
|
+
const type = config.types.find((t) => t.id === item.contentType);
|
|
3061
|
+
if (!type) throw new Error(`Unknown content type ${item.contentType}`);
|
|
3062
|
+
const enDoc = readEnDocument(config, type, item.enSlug);
|
|
3063
|
+
if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
|
|
3064
|
+
const payload = getTranslatablePayload(enDoc, type);
|
|
3065
|
+
const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
|
|
3066
|
+
const db = openStore(config, "readonly");
|
|
3067
|
+
const existing = getTranslation(db, type.id, item.enSlug, item.locale);
|
|
3068
|
+
db.close();
|
|
3069
|
+
if (!options.force && existing && existing.en_hash === currentEnHash) {
|
|
3070
|
+
return {
|
|
3071
|
+
status: "done",
|
|
3072
|
+
result: {
|
|
3073
|
+
...baseForItem(item),
|
|
2770
3074
|
skipped: true,
|
|
2771
3075
|
reason: "fresh",
|
|
2772
3076
|
durationMs: Date.now() - startedAt
|
|
2773
|
-
}
|
|
2774
|
-
}
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
3077
|
+
}
|
|
3078
|
+
};
|
|
3079
|
+
}
|
|
3080
|
+
const resolvedTranslate = resolveTranslateConfig(config, type);
|
|
3081
|
+
const model = options.model ?? resolvedTranslate.model;
|
|
3082
|
+
const prompt = buildPageTranslationPrompt({
|
|
3083
|
+
resolved: resolvedTranslate,
|
|
3084
|
+
targetLocale: item.locale,
|
|
3085
|
+
contextLabel: resolveContextLabel(enDoc, item.enSlug),
|
|
3086
|
+
translatableFrontmatter: payload.frontmatter,
|
|
3087
|
+
enBody: payload.body,
|
|
3088
|
+
slugStrategy: type.slugStrategy
|
|
3089
|
+
});
|
|
3090
|
+
const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
|
|
3091
|
+
return {
|
|
3092
|
+
status: "ready",
|
|
3093
|
+
prepared: {
|
|
3094
|
+
item,
|
|
3095
|
+
type,
|
|
3096
|
+
enDoc,
|
|
3097
|
+
payload,
|
|
3098
|
+
currentEnHash,
|
|
3099
|
+
existingSlug: existing?.slug,
|
|
2796
3100
|
model,
|
|
3101
|
+
prompt,
|
|
2797
3102
|
responseSchema: responseSchema ?? void 0
|
|
2798
|
-
}
|
|
2799
|
-
|
|
3103
|
+
}
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
function finalizeTranslation(config, prepared, output, options) {
|
|
3107
|
+
const { item, type, enDoc, payload } = prepared;
|
|
3108
|
+
const base = baseForItem(item);
|
|
3109
|
+
try {
|
|
3110
|
+
const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
|
|
2800
3111
|
const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
|
|
2801
3112
|
const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
|
|
2802
3113
|
const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
|
|
2803
|
-
const validated = validateTranslatedFrontmatter(enDoc,
|
|
3114
|
+
const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
|
|
2804
3115
|
if (!validated.ok) {
|
|
2805
3116
|
throw new Error(`Translation validation failed: ${validated.error}`);
|
|
2806
3117
|
}
|
|
2807
3118
|
const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
|
|
2808
|
-
|
|
3119
|
+
output.parsed.body
|
|
2809
3120
|
);
|
|
2810
3121
|
const writeDb = openStore(config, "readwrite");
|
|
2811
|
-
const snapshotId = recordEnSnapshot(
|
|
3122
|
+
const snapshotId = options.snapshotId ?? recordEnSnapshot(
|
|
2812
3123
|
config,
|
|
2813
3124
|
{
|
|
2814
3125
|
contentType: type.id,
|
|
2815
3126
|
enSlug: item.enSlug,
|
|
2816
|
-
enHash: currentEnHash,
|
|
3127
|
+
enHash: prepared.currentEnHash,
|
|
2817
3128
|
frontmatter: payload.frontmatter,
|
|
2818
3129
|
body: payload.body
|
|
2819
3130
|
},
|
|
@@ -2826,24 +3137,25 @@ async function translatePage(config, item, options = {}) {
|
|
|
2826
3137
|
slug,
|
|
2827
3138
|
frontmatter: validated.frontmatter,
|
|
2828
3139
|
body: translatedBody,
|
|
2829
|
-
enHash: currentEnHash,
|
|
3140
|
+
enHash: prepared.currentEnHash,
|
|
2830
3141
|
translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2831
|
-
model:
|
|
3142
|
+
model: output.model,
|
|
2832
3143
|
snapshotId
|
|
2833
3144
|
});
|
|
2834
3145
|
writeDb.close();
|
|
2835
3146
|
const estimatedCostUsd = estimateTranslationCostUsd(
|
|
2836
|
-
normalizeGeminiDisplayName(
|
|
2837
|
-
|
|
2838
|
-
|
|
3147
|
+
normalizeGeminiDisplayName(output.model),
|
|
3148
|
+
output.usage.inputTokens,
|
|
3149
|
+
output.usage.outputTokens,
|
|
3150
|
+
options.costMode
|
|
2839
3151
|
);
|
|
2840
3152
|
return {
|
|
2841
3153
|
...base,
|
|
2842
3154
|
skipped: false,
|
|
2843
|
-
model:
|
|
2844
|
-
usage:
|
|
3155
|
+
model: output.model,
|
|
3156
|
+
usage: output.usage,
|
|
2845
3157
|
estimatedCostUsd,
|
|
2846
|
-
durationMs: Date.now() - startedAt,
|
|
3158
|
+
durationMs: Date.now() - options.startedAt,
|
|
2847
3159
|
slugAdjusted,
|
|
2848
3160
|
mdxAdjusted: mdxAdjusted || void 0
|
|
2849
3161
|
};
|
|
@@ -2853,33 +3165,559 @@ async function translatePage(config, item, options = {}) {
|
|
|
2853
3165
|
skipped: false,
|
|
2854
3166
|
failed: true,
|
|
2855
3167
|
error: formatTranslateError(error),
|
|
3168
|
+
durationMs: Date.now() - options.startedAt
|
|
3169
|
+
};
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
function displayModelFor(prepared) {
|
|
3173
|
+
return normalizeGeminiDisplayName(
|
|
3174
|
+
prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
|
|
3175
|
+
);
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
// src/translate/batch-worklist.ts
|
|
3179
|
+
var MAX_REQUESTS_PER_JOB = 100;
|
|
3180
|
+
var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
|
|
3181
|
+
var INITIAL_POLL_MS = 5e3;
|
|
3182
|
+
var MAX_POLL_MS = 3e4;
|
|
3183
|
+
var POLL_BACKOFF_FACTOR = 1.5;
|
|
3184
|
+
function sleep(ms) {
|
|
3185
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3186
|
+
}
|
|
3187
|
+
function planBatchJobs(entries) {
|
|
3188
|
+
const byModel = /* @__PURE__ */ new Map();
|
|
3189
|
+
for (const entry of entries) {
|
|
3190
|
+
const group = byModel.get(entry.apiModel);
|
|
3191
|
+
if (group) group.push(entry);
|
|
3192
|
+
else byModel.set(entry.apiModel, [entry]);
|
|
3193
|
+
}
|
|
3194
|
+
const plans = [];
|
|
3195
|
+
for (const [apiModel, group] of byModel) {
|
|
3196
|
+
let current = [];
|
|
3197
|
+
let currentBytes = 0;
|
|
3198
|
+
for (const entry of group) {
|
|
3199
|
+
const size = Buffer.byteLength(entry.prompt, "utf8");
|
|
3200
|
+
if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
|
|
3201
|
+
plans.push({ apiModel, entries: current });
|
|
3202
|
+
current = [];
|
|
3203
|
+
currentBytes = 0;
|
|
3204
|
+
}
|
|
3205
|
+
current.push(entry);
|
|
3206
|
+
currentBytes += size;
|
|
3207
|
+
}
|
|
3208
|
+
if (current.length > 0) plans.push({ apiModel, entries: current });
|
|
3209
|
+
}
|
|
3210
|
+
return plans;
|
|
3211
|
+
}
|
|
3212
|
+
function readPendingBatchWork(config) {
|
|
3213
|
+
const db = openStore(config, "readwrite");
|
|
3214
|
+
const jobs = listPendingBatchJobs(db);
|
|
3215
|
+
const pendingItems = listPendingBatchItems(db);
|
|
3216
|
+
db.close();
|
|
3217
|
+
const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
|
|
3218
|
+
return { jobs, inFlightKeys, pendingItems };
|
|
3219
|
+
}
|
|
3220
|
+
async function submitBatchJobPlan(config, input) {
|
|
3221
|
+
const { plan, displayModel, jobIndex, jobCount } = input;
|
|
3222
|
+
const job = await createGeminiBatchJob({
|
|
3223
|
+
model: plan.apiModel,
|
|
3224
|
+
requests: plan.entries.map(({ prepared }) => ({
|
|
3225
|
+
contents: prepared.prompt,
|
|
3226
|
+
config: buildGeminiRequestConfig({
|
|
3227
|
+
apiModelId: plan.apiModel,
|
|
3228
|
+
responseSchema: prepared.responseSchema
|
|
3229
|
+
})
|
|
3230
|
+
})),
|
|
3231
|
+
displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
|
|
3232
|
+
});
|
|
3233
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3234
|
+
const db = openStore(config, "readwrite");
|
|
3235
|
+
const jobId = insertBatchJob(db, {
|
|
3236
|
+
jobName: job.name,
|
|
3237
|
+
model: plan.apiModel,
|
|
3238
|
+
displayModel,
|
|
3239
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3240
|
+
createdAt
|
|
3241
|
+
});
|
|
3242
|
+
insertBatchItems(
|
|
3243
|
+
db,
|
|
3244
|
+
jobId,
|
|
3245
|
+
plan.entries.map(({ prepared }, requestIndex) => ({
|
|
3246
|
+
requestIndex,
|
|
3247
|
+
contentType: prepared.item.contentType,
|
|
3248
|
+
enSlug: prepared.item.enSlug,
|
|
3249
|
+
locale: prepared.item.locale,
|
|
3250
|
+
enHash: prepared.currentEnHash,
|
|
3251
|
+
// Snapshot the EN source now so ingestion after a resume does not depend
|
|
3252
|
+
// on the EN files still matching (or existing) on disk.
|
|
3253
|
+
snapshotId: recordEnSnapshot(
|
|
3254
|
+
config,
|
|
3255
|
+
{
|
|
3256
|
+
contentType: prepared.item.contentType,
|
|
3257
|
+
enSlug: prepared.item.enSlug,
|
|
3258
|
+
enHash: prepared.currentEnHash,
|
|
3259
|
+
frontmatter: prepared.payload.frontmatter,
|
|
3260
|
+
body: prepared.payload.body
|
|
3261
|
+
},
|
|
3262
|
+
db
|
|
3263
|
+
)
|
|
3264
|
+
}))
|
|
3265
|
+
);
|
|
3266
|
+
const row = {
|
|
3267
|
+
id: jobId,
|
|
3268
|
+
job_name: job.name,
|
|
3269
|
+
model: plan.apiModel,
|
|
3270
|
+
display_model: displayModel,
|
|
3271
|
+
created_at: createdAt,
|
|
3272
|
+
state: String(job.state ?? "JOB_STATE_PENDING"),
|
|
3273
|
+
completed_at: null
|
|
3274
|
+
};
|
|
3275
|
+
db.close();
|
|
3276
|
+
input.onProgress?.({
|
|
3277
|
+
type: "batch-submitted",
|
|
3278
|
+
name: job.name,
|
|
3279
|
+
count: plan.entries.length,
|
|
3280
|
+
model: displayModel,
|
|
3281
|
+
jobIndex,
|
|
3282
|
+
jobCount,
|
|
3283
|
+
createdAt
|
|
3284
|
+
});
|
|
3285
|
+
return row;
|
|
3286
|
+
}
|
|
3287
|
+
function buildIngestContext(config, db, jobRow, itemRow) {
|
|
3288
|
+
const type = config.types.find((t) => t.id === itemRow.content_type);
|
|
3289
|
+
if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
|
|
3290
|
+
let enDoc = readEnDocument(config, type, itemRow.en_slug);
|
|
3291
|
+
if (!enDoc) {
|
|
3292
|
+
const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
|
|
3293
|
+
if (!snapshot) {
|
|
3294
|
+
throw new Error(
|
|
3295
|
+
`EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
|
|
3296
|
+
);
|
|
3297
|
+
}
|
|
3298
|
+
enDoc = {
|
|
3299
|
+
slug: itemRow.en_slug,
|
|
3300
|
+
enSlug: itemRow.en_slug,
|
|
3301
|
+
locale: config.defaultLocale,
|
|
3302
|
+
noindex: false,
|
|
3303
|
+
frontmatter: JSON.parse(snapshot.frontmatter_json),
|
|
3304
|
+
content: snapshot.body
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
const item = {
|
|
3308
|
+
contentType: itemRow.content_type,
|
|
3309
|
+
enSlug: itemRow.en_slug,
|
|
3310
|
+
locale: itemRow.locale,
|
|
3311
|
+
reason: "missing",
|
|
3312
|
+
currentEnHash: itemRow.en_hash
|
|
3313
|
+
};
|
|
3314
|
+
return {
|
|
3315
|
+
item,
|
|
3316
|
+
type,
|
|
3317
|
+
enDoc,
|
|
3318
|
+
// Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
|
|
3319
|
+
payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
|
|
3320
|
+
currentEnHash: itemRow.en_hash,
|
|
3321
|
+
existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
|
|
3322
|
+
model: jobRow.display_model,
|
|
3323
|
+
prompt: "",
|
|
3324
|
+
responseSchema: void 0
|
|
3325
|
+
};
|
|
3326
|
+
}
|
|
3327
|
+
function ingestBatchJob(config, jobRow, batchJob, onResult) {
|
|
3328
|
+
const state = String(batchJob.state ?? "UNKNOWN");
|
|
3329
|
+
const db = openStore(config, "readwrite");
|
|
3330
|
+
if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
|
|
3331
|
+
db.close();
|
|
3332
|
+
return null;
|
|
3333
|
+
}
|
|
3334
|
+
const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
|
|
3335
|
+
const results = [];
|
|
3336
|
+
const failItem = (itemRow, message, startedAt) => {
|
|
3337
|
+
const result = {
|
|
3338
|
+
contentType: itemRow.content_type,
|
|
3339
|
+
enSlug: itemRow.en_slug,
|
|
3340
|
+
locale: itemRow.locale,
|
|
3341
|
+
skipped: false,
|
|
3342
|
+
failed: true,
|
|
3343
|
+
error: formatTranslateError(new Error(message)),
|
|
2856
3344
|
durationMs: Date.now() - startedAt
|
|
2857
3345
|
};
|
|
3346
|
+
updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
|
|
3347
|
+
results.push(result);
|
|
3348
|
+
onResult?.(result);
|
|
3349
|
+
};
|
|
3350
|
+
if (isSuccessfulBatchState(state)) {
|
|
3351
|
+
const responses = batchJob.dest?.inlinedResponses ?? [];
|
|
3352
|
+
for (const itemRow of pendingItems) {
|
|
3353
|
+
const startedAt = Date.now();
|
|
3354
|
+
const inlined = responses[itemRow.request_index];
|
|
3355
|
+
if (!inlined || inlined.error || !inlined.response) {
|
|
3356
|
+
failItem(
|
|
3357
|
+
itemRow,
|
|
3358
|
+
inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
|
|
3359
|
+
startedAt
|
|
3360
|
+
);
|
|
3361
|
+
continue;
|
|
3362
|
+
}
|
|
3363
|
+
let result;
|
|
3364
|
+
try {
|
|
3365
|
+
const response = inlined.response;
|
|
3366
|
+
const raw = textFromBatchResponse(response);
|
|
3367
|
+
const parsed = parseGeminiResponse(raw);
|
|
3368
|
+
if (!parsed.frontmatter || typeof parsed.body !== "string") {
|
|
3369
|
+
throw new Error("Gemini response missing frontmatter/body");
|
|
3370
|
+
}
|
|
3371
|
+
const prepared = buildIngestContext(config, db, jobRow, itemRow);
|
|
3372
|
+
result = finalizeTranslation(
|
|
3373
|
+
config,
|
|
3374
|
+
prepared,
|
|
3375
|
+
{ model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
|
|
3376
|
+
{ costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
|
|
3377
|
+
);
|
|
3378
|
+
} catch (error) {
|
|
3379
|
+
failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
|
|
3380
|
+
continue;
|
|
3381
|
+
}
|
|
3382
|
+
updateBatchItemStatus(
|
|
3383
|
+
db,
|
|
3384
|
+
jobRow.id,
|
|
3385
|
+
itemRow.request_index,
|
|
3386
|
+
result.failed ? "failed" : "done",
|
|
3387
|
+
result.error
|
|
3388
|
+
);
|
|
3389
|
+
results.push(result);
|
|
3390
|
+
onResult?.(result);
|
|
3391
|
+
}
|
|
3392
|
+
} else {
|
|
3393
|
+
const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
|
|
3394
|
+
const startedAt = Date.now();
|
|
3395
|
+
for (const itemRow of pendingItems) {
|
|
3396
|
+
failItem(itemRow, message, startedAt);
|
|
3397
|
+
}
|
|
2858
3398
|
}
|
|
3399
|
+
db.close();
|
|
3400
|
+
return results;
|
|
3401
|
+
}
|
|
3402
|
+
async function pollAndIngestBatchJobs(config, jobs, options = {}) {
|
|
3403
|
+
const jobCount = options.jobCount ?? jobs.length;
|
|
3404
|
+
const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
|
|
3405
|
+
const loopStartedAt = Date.now();
|
|
3406
|
+
let delay = INITIAL_POLL_MS;
|
|
3407
|
+
let firstRound = true;
|
|
3408
|
+
const elapsedFor = (row) => {
|
|
3409
|
+
const createdAtMs = Date.parse(row.created_at);
|
|
3410
|
+
return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
|
|
3411
|
+
};
|
|
3412
|
+
while (active.length > 0) {
|
|
3413
|
+
if (!firstRound) {
|
|
3414
|
+
await sleep(delay);
|
|
3415
|
+
delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
|
|
3416
|
+
}
|
|
3417
|
+
firstRound = false;
|
|
3418
|
+
for (const tracked of [...active]) {
|
|
3419
|
+
const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
|
|
3420
|
+
const state = normalizeBatchState(batchJob.state);
|
|
3421
|
+
options.onProgress?.({
|
|
3422
|
+
type: "batch-polling",
|
|
3423
|
+
name: tracked.row.job_name,
|
|
3424
|
+
state,
|
|
3425
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3426
|
+
jobIndex: tracked.jobIndex,
|
|
3427
|
+
jobCount
|
|
3428
|
+
});
|
|
3429
|
+
if (isTerminalBatchState(batchJob.state)) {
|
|
3430
|
+
active.splice(active.indexOf(tracked), 1);
|
|
3431
|
+
const results = ingestBatchJob(
|
|
3432
|
+
config,
|
|
3433
|
+
tracked.row,
|
|
3434
|
+
batchJob,
|
|
3435
|
+
options.onResult
|
|
3436
|
+
);
|
|
3437
|
+
if (results === null) {
|
|
3438
|
+
options.onProgress?.({
|
|
3439
|
+
type: "batch-done",
|
|
3440
|
+
name: tracked.row.job_name,
|
|
3441
|
+
state,
|
|
3442
|
+
model: tracked.row.display_model,
|
|
3443
|
+
count: 0,
|
|
3444
|
+
jobIndex: tracked.jobIndex,
|
|
3445
|
+
jobCount,
|
|
3446
|
+
translated: 0,
|
|
3447
|
+
failed: 0,
|
|
3448
|
+
inputTokens: 0,
|
|
3449
|
+
outputTokens: 0,
|
|
3450
|
+
estimatedCostUsd: 0,
|
|
3451
|
+
elapsedMs: elapsedFor(tracked.row),
|
|
3452
|
+
alreadyIngested: true
|
|
3453
|
+
});
|
|
3454
|
+
continue;
|
|
3455
|
+
}
|
|
3456
|
+
options.onProgress?.({
|
|
3457
|
+
type: "batch-done",
|
|
3458
|
+
name: tracked.row.job_name,
|
|
3459
|
+
state,
|
|
3460
|
+
model: tracked.row.display_model,
|
|
3461
|
+
count: results.length,
|
|
3462
|
+
jobIndex: tracked.jobIndex,
|
|
3463
|
+
jobCount,
|
|
3464
|
+
translated: results.filter((r) => !r.failed && !r.skipped).length,
|
|
3465
|
+
failed: results.filter((r) => r.failed).length,
|
|
3466
|
+
inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
|
|
3467
|
+
outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
|
|
3468
|
+
estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
|
|
3469
|
+
elapsedMs: elapsedFor(tracked.row)
|
|
3470
|
+
});
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
3473
|
+
}
|
|
3474
|
+
}
|
|
3475
|
+
function failedResultForItem(item, error, startedAt) {
|
|
3476
|
+
return {
|
|
3477
|
+
...baseForItem(item),
|
|
3478
|
+
skipped: false,
|
|
3479
|
+
failed: true,
|
|
3480
|
+
error: formatTranslateError(error),
|
|
3481
|
+
durationMs: Date.now() - startedAt
|
|
3482
|
+
};
|
|
3483
|
+
}
|
|
3484
|
+
|
|
3485
|
+
// src/translate/page-translator.ts
|
|
3486
|
+
async function runPool(items, concurrency, worker) {
|
|
3487
|
+
if (items.length === 0) return;
|
|
3488
|
+
let nextIndex = 0;
|
|
3489
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3490
|
+
while (nextIndex < items.length) {
|
|
3491
|
+
const index = nextIndex++;
|
|
3492
|
+
await worker(items[index], index);
|
|
3493
|
+
}
|
|
3494
|
+
});
|
|
3495
|
+
await Promise.all(runners);
|
|
2859
3496
|
}
|
|
2860
3497
|
function labelForItem(item) {
|
|
2861
3498
|
return `${item.contentType}/${item.enSlug}@${item.locale}`;
|
|
2862
3499
|
}
|
|
3500
|
+
async function translatePage(config, item, options = {}) {
|
|
3501
|
+
const startedAt = Date.now();
|
|
3502
|
+
let outcome;
|
|
3503
|
+
try {
|
|
3504
|
+
outcome = prepareTranslation(config, item, options, startedAt);
|
|
3505
|
+
} catch (error) {
|
|
3506
|
+
return failedResultForItem(item, error, startedAt);
|
|
3507
|
+
}
|
|
3508
|
+
if (outcome.status === "done") return outcome.result;
|
|
3509
|
+
const prepared = outcome.prepared;
|
|
3510
|
+
if (options.dryRun) {
|
|
3511
|
+
return {
|
|
3512
|
+
...baseForItem(item),
|
|
3513
|
+
skipped: false,
|
|
3514
|
+
model: prepared.model,
|
|
3515
|
+
durationMs: Date.now() - startedAt
|
|
3516
|
+
};
|
|
3517
|
+
}
|
|
3518
|
+
try {
|
|
3519
|
+
const result = await translatePageWithGemini({
|
|
3520
|
+
prompt: prepared.prompt,
|
|
3521
|
+
model: prepared.model,
|
|
3522
|
+
responseSchema: prepared.responseSchema
|
|
3523
|
+
});
|
|
3524
|
+
return finalizeTranslation(
|
|
3525
|
+
config,
|
|
3526
|
+
prepared,
|
|
3527
|
+
{ model: result.model, parsed: result.parsed, usage: result.usage },
|
|
3528
|
+
{ costMode: "interactive", startedAt }
|
|
3529
|
+
);
|
|
3530
|
+
} catch (error) {
|
|
3531
|
+
return failedResultForItem(item, error, startedAt);
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
function createResultCollector(items, onProgress) {
|
|
3535
|
+
const slotByKey = /* @__PURE__ */ new Map();
|
|
3536
|
+
items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
|
|
3537
|
+
const slots = new Array(items.length);
|
|
3538
|
+
const extras = [];
|
|
3539
|
+
return {
|
|
3540
|
+
emit(result) {
|
|
3541
|
+
const slot = slotByKey.get(translationItemKey(result));
|
|
3542
|
+
if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
|
|
3543
|
+
else extras.push(result);
|
|
3544
|
+
onProgress?.({ type: "item-done", result });
|
|
3545
|
+
},
|
|
3546
|
+
assemble() {
|
|
3547
|
+
return [
|
|
3548
|
+
...slots.filter((result) => result !== void 0),
|
|
3549
|
+
...extras
|
|
3550
|
+
];
|
|
3551
|
+
}
|
|
3552
|
+
};
|
|
3553
|
+
}
|
|
2863
3554
|
async function translateWorklist(config, items, options = {}) {
|
|
3555
|
+
const mode = options.mode ?? "batch";
|
|
2864
3556
|
const concurrency = Math.max(1, options.concurrency ?? 3);
|
|
2865
3557
|
const startedAt = Date.now();
|
|
2866
|
-
|
|
2867
|
-
|
|
3558
|
+
if (options.dryRun) {
|
|
3559
|
+
options.onProgress?.({
|
|
3560
|
+
type: "start",
|
|
3561
|
+
total: items.length,
|
|
3562
|
+
concurrency,
|
|
3563
|
+
dryRun: true,
|
|
3564
|
+
model: options.model,
|
|
3565
|
+
mode
|
|
3566
|
+
});
|
|
3567
|
+
const results2 = items.map((item) => {
|
|
3568
|
+
const itemStartedAt = Date.now();
|
|
3569
|
+
try {
|
|
3570
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3571
|
+
return outcome.status === "done" ? outcome.result : {
|
|
3572
|
+
...baseForItem(item),
|
|
3573
|
+
skipped: false,
|
|
3574
|
+
model: outcome.prepared.model,
|
|
3575
|
+
durationMs: Date.now() - itemStartedAt
|
|
3576
|
+
};
|
|
3577
|
+
} catch (error) {
|
|
3578
|
+
return failedResultForItem(item, error, itemStartedAt);
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
3581
|
+
for (const result of results2) options.onProgress?.({ type: "item-done", result });
|
|
3582
|
+
const totals2 = summarizeResults(results2, Date.now() - startedAt);
|
|
3583
|
+
options.onProgress?.({ type: "done", results: results2, totals: totals2 });
|
|
3584
|
+
return results2;
|
|
3585
|
+
}
|
|
3586
|
+
const pending = readPendingBatchWork(config);
|
|
3587
|
+
const inputKeys = new Set(items.map((item) => translationItemKey(item)));
|
|
3588
|
+
const resumedExtraCount = pending.pendingItems.filter(
|
|
3589
|
+
(item) => !inputKeys.has(translationItemKey(item))
|
|
3590
|
+
).length;
|
|
2868
3591
|
options.onProgress?.({
|
|
2869
3592
|
type: "start",
|
|
2870
|
-
total: items.length,
|
|
3593
|
+
total: items.length + resumedExtraCount,
|
|
2871
3594
|
concurrency,
|
|
2872
|
-
dryRun:
|
|
2873
|
-
model: options.model
|
|
3595
|
+
dryRun: false,
|
|
3596
|
+
model: options.model,
|
|
3597
|
+
mode
|
|
3598
|
+
});
|
|
3599
|
+
const collector = createResultCollector(items, options.onProgress);
|
|
3600
|
+
const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
|
|
3601
|
+
if (mode === "direct") {
|
|
3602
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3603
|
+
emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
|
|
3604
|
+
const active = /* @__PURE__ */ new Set();
|
|
3605
|
+
await Promise.all([
|
|
3606
|
+
runPool(workItems, concurrency, async (item) => {
|
|
3607
|
+
const label = labelForItem(item);
|
|
3608
|
+
active.add(label);
|
|
3609
|
+
options.onProgress?.({ type: "item-start", item, active: [...active] });
|
|
3610
|
+
const result = await translatePage(config, item, options);
|
|
3611
|
+
active.delete(label);
|
|
3612
|
+
collector.emit(result);
|
|
3613
|
+
}),
|
|
3614
|
+
pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3615
|
+
onProgress: options.onProgress,
|
|
3616
|
+
onResult: collector.emit
|
|
3617
|
+
}) : Promise.resolve()
|
|
3618
|
+
]);
|
|
3619
|
+
} else {
|
|
3620
|
+
const entries = [];
|
|
3621
|
+
for (const item of workItems) {
|
|
3622
|
+
const itemStartedAt = Date.now();
|
|
3623
|
+
try {
|
|
3624
|
+
const outcome = prepareTranslation(config, item, options, itemStartedAt);
|
|
3625
|
+
if (outcome.status === "done") {
|
|
3626
|
+
collector.emit(outcome.result);
|
|
3627
|
+
} else {
|
|
3628
|
+
entries.push({
|
|
3629
|
+
apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
|
|
3630
|
+
prompt: outcome.prepared.prompt,
|
|
3631
|
+
prepared: outcome.prepared
|
|
3632
|
+
});
|
|
3633
|
+
}
|
|
3634
|
+
} catch (error) {
|
|
3635
|
+
collector.emit(failedResultForItem(item, error, itemStartedAt));
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
const plans = planBatchJobs(entries);
|
|
3639
|
+
const jobCount = pending.jobs.length + plans.length;
|
|
3640
|
+
const pendingCounts = countPendingItems(config, pending.jobs);
|
|
3641
|
+
emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
|
|
3642
|
+
const submitted = await Promise.all(
|
|
3643
|
+
plans.map(async (plan, planIndex) => {
|
|
3644
|
+
try {
|
|
3645
|
+
return await submitBatchJobPlan(config, {
|
|
3646
|
+
plan,
|
|
3647
|
+
displayModel: normalizeGeminiDisplayName(plan.apiModel),
|
|
3648
|
+
jobIndex: pending.jobs.length + planIndex,
|
|
3649
|
+
jobCount,
|
|
3650
|
+
onProgress: options.onProgress
|
|
3651
|
+
});
|
|
3652
|
+
} catch (error) {
|
|
3653
|
+
for (const entry of plan.entries) {
|
|
3654
|
+
collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
|
|
3655
|
+
}
|
|
3656
|
+
return void 0;
|
|
3657
|
+
}
|
|
3658
|
+
})
|
|
3659
|
+
);
|
|
3660
|
+
const jobsToPoll = [
|
|
3661
|
+
...pending.jobs,
|
|
3662
|
+
...submitted.filter((row) => row !== void 0)
|
|
3663
|
+
];
|
|
3664
|
+
if (jobsToPoll.length > 0) {
|
|
3665
|
+
await pollAndIngestBatchJobs(config, jobsToPoll, {
|
|
3666
|
+
jobCount,
|
|
3667
|
+
onProgress: options.onProgress,
|
|
3668
|
+
onResult: collector.emit
|
|
3669
|
+
});
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
const results = collector.assemble();
|
|
3673
|
+
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
3674
|
+
options.onProgress?.({ type: "done", results, totals });
|
|
3675
|
+
return results;
|
|
3676
|
+
}
|
|
3677
|
+
function countPendingItems(config, jobs) {
|
|
3678
|
+
if (jobs.length === 0) return /* @__PURE__ */ new Map();
|
|
3679
|
+
const db = openStore(config, "readonly");
|
|
3680
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3681
|
+
for (const job of jobs) {
|
|
3682
|
+
counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
|
|
3683
|
+
}
|
|
3684
|
+
db.close();
|
|
3685
|
+
return counts;
|
|
3686
|
+
}
|
|
3687
|
+
function emitResumedJobs(jobs, counts, jobCount, onProgress) {
|
|
3688
|
+
jobs.forEach((job, jobIndex) => {
|
|
3689
|
+
onProgress?.({
|
|
3690
|
+
type: "batch-submitted",
|
|
3691
|
+
name: job.job_name,
|
|
3692
|
+
count: counts.get(job.id) ?? 0,
|
|
3693
|
+
model: job.display_model,
|
|
3694
|
+
jobIndex,
|
|
3695
|
+
jobCount,
|
|
3696
|
+
resumed: true,
|
|
3697
|
+
createdAt: job.created_at
|
|
3698
|
+
});
|
|
3699
|
+
});
|
|
3700
|
+
}
|
|
3701
|
+
async function resumeTranslationJobs(config, options = {}) {
|
|
3702
|
+
const startedAt = Date.now();
|
|
3703
|
+
const pending = readPendingBatchWork(config);
|
|
3704
|
+
if (pending.jobs.length === 0) return null;
|
|
3705
|
+
options.onProgress?.({
|
|
3706
|
+
type: "start",
|
|
3707
|
+
total: pending.pendingItems.length,
|
|
3708
|
+
concurrency: 1,
|
|
3709
|
+
dryRun: false,
|
|
3710
|
+
mode: "batch"
|
|
2874
3711
|
});
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
3712
|
+
const counts = countPendingItems(config, pending.jobs);
|
|
3713
|
+
emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
|
|
3714
|
+
const results = [];
|
|
3715
|
+
await pollAndIngestBatchJobs(config, pending.jobs, {
|
|
3716
|
+
onProgress: options.onProgress,
|
|
3717
|
+
onResult: (result) => {
|
|
3718
|
+
results.push(result);
|
|
3719
|
+
options.onProgress?.({ type: "item-done", result });
|
|
3720
|
+
}
|
|
2883
3721
|
});
|
|
2884
3722
|
const totals = summarizeResults(results, Date.now() - startedAt);
|
|
2885
3723
|
options.onProgress?.({ type: "done", results, totals });
|
|
@@ -2981,6 +3819,6 @@ function writeStaticRawExports(project, options = {}) {
|
|
|
2981
3819
|
return { exports, written: exports.length };
|
|
2982
3820
|
}
|
|
2983
3821
|
|
|
2984
|
-
export { buildAllContentRedirects, buildStaticRawExports, buildWorklist, createProject, createScribe, createUrlBuilder, defineConfig, defineContentType, exportDirSegment, field, findConfigPath, generateSitemap, getFieldKind, getRedirectSourceSlugs, getRelationTarget, getStaticExportRoots, isResolvedConfig, isRoutableType, loadConfigSync, resolveConfig, resolveLocalesFromPreset, serializeMdx, translatePage, translateWorklist, unwrapSchema, validateProject, writeStaticRawExports };
|
|
3822
|
+
export { buildAllContentRedirects, buildStaticRawExports, buildWorklist, createProject, createScribe, createUrlBuilder, defineConfig, defineContentType, exportDirSegment, field, findConfigPath, generateSitemap, getFieldKind, getRedirectSourceSlugs, getRelationTarget, getStaticExportRoots, isResolvedConfig, isRoutableType, loadConfigSync, resolveConfig, resolveLocalesFromPreset, resumeTranslationJobs, serializeMdx, translatePage, translateWorklist, unwrapSchema, validateProject, writeStaticRawExports };
|
|
2985
3823
|
//# sourceMappingURL=index.js.map
|
|
2986
3824
|
//# sourceMappingURL=index.js.map
|