scribe-cms 0.0.12 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +21 -6
  2. package/dist/cli/index.cjs +1220 -204
  3. package/dist/cli/index.cjs.map +1 -1
  4. package/dist/cli/index.js +1221 -205
  5. package/dist/cli/index.js.map +1 -1
  6. package/dist/cli/prompt-translate.d.ts +3 -0
  7. package/dist/cli/prompt-translate.d.ts.map +1 -1
  8. package/dist/cli/translate-progress.d.ts.map +1 -1
  9. package/dist/index.cjs +982 -166
  10. package/dist/index.cjs.map +1 -1
  11. package/dist/index.js +983 -168
  12. package/dist/index.js.map +1 -1
  13. package/dist/runtime.cjs +149 -62
  14. package/dist/runtime.cjs.map +1 -1
  15. package/dist/runtime.js +149 -62
  16. package/dist/runtime.js.map +1 -1
  17. package/dist/src/config/resolve-config.d.ts.map +1 -1
  18. package/dist/src/core/introspect-schema.d.ts +7 -1
  19. package/dist/src/core/introspect-schema.d.ts.map +1 -1
  20. package/dist/src/core/types.d.ts +6 -0
  21. package/dist/src/core/types.d.ts.map +1 -1
  22. package/dist/src/create-project.d.ts.map +1 -1
  23. package/dist/src/i18n/resolve-document.d.ts +1 -1
  24. package/dist/src/i18n/resolve-document.d.ts.map +1 -1
  25. package/dist/src/index.d.ts +3 -3
  26. package/dist/src/index.d.ts.map +1 -1
  27. package/dist/src/storage/batch-jobs.d.ts +53 -0
  28. package/dist/src/storage/batch-jobs.d.ts.map +1 -0
  29. package/dist/src/storage/sqlite.d.ts.map +1 -1
  30. package/dist/src/translate/batch-worklist.d.ts +87 -0
  31. package/dist/src/translate/batch-worklist.d.ts.map +1 -0
  32. package/dist/src/translate/gemini-batch.d.ts +26 -0
  33. package/dist/src/translate/gemini-batch.d.ts.map +1 -0
  34. package/dist/src/translate/gemini-client.d.ts +18 -0
  35. package/dist/src/translate/gemini-client.d.ts.map +1 -1
  36. package/dist/src/translate/gemini-models.d.ts +9 -0
  37. package/dist/src/translate/gemini-models.d.ts.map +1 -1
  38. package/dist/src/translate/gemini-pricing.d.ts +2 -1
  39. package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
  40. package/dist/src/translate/page-translator.d.ts +25 -52
  41. package/dist/src/translate/page-translator.d.ts.map +1 -1
  42. package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
  43. package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
  44. package/dist/src/translate/response-schema.d.ts +8 -1
  45. package/dist/src/translate/response-schema.d.ts.map +1 -1
  46. package/dist/src/translate/retry.d.ts +16 -0
  47. package/dist/src/translate/retry.d.ts.map +1 -0
  48. package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
  49. package/dist/src/translate/translate-core.d.ts +155 -0
  50. package/dist/src/translate/translate-core.d.ts.map +1 -0
  51. package/dist/studio/server.cjs +84 -47
  52. package/dist/studio/server.cjs.map +1 -1
  53. package/dist/studio/server.js +84 -47
  54. package/dist/studio/server.js.map +1 -1
  55. package/package.json +1 -1
package/dist/index.cjs CHANGED
@@ -257,6 +257,39 @@ function resolveConfig(input, baseDir) {
257
257
  if (localeRouting.strategy === "search-param" && !localeRouting.param) {
258
258
  throw new Error('scribe config: localeRouting search-param requires a "param" name');
259
259
  }
260
+ const localeFallbacks = {};
261
+ for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
262
+ if (!raw.locales.includes(locale)) {
263
+ throw new Error(
264
+ `scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
265
+ );
266
+ }
267
+ const seen = /* @__PURE__ */ new Set();
268
+ for (const fallback of chain) {
269
+ if (!raw.locales.includes(fallback)) {
270
+ throw new Error(
271
+ `scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
272
+ );
273
+ }
274
+ if (fallback === locale) {
275
+ throw new Error(
276
+ `scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
277
+ );
278
+ }
279
+ if (fallback === defaultLocale) {
280
+ throw new Error(
281
+ `scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
282
+ );
283
+ }
284
+ if (seen.has(fallback)) {
285
+ throw new Error(
286
+ `scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
287
+ );
288
+ }
289
+ seen.add(fallback);
290
+ }
291
+ localeFallbacks[locale] = [...chain];
292
+ }
260
293
  const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
261
294
  const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
262
295
  const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
@@ -286,6 +319,7 @@ function resolveConfig(input, baseDir) {
286
319
  defaultLocale,
287
320
  localeRouting,
288
321
  localePresets: raw.localePresets,
322
+ localeFallbacks,
289
323
  translate: raw.translate,
290
324
  types
291
325
  };
@@ -297,6 +331,12 @@ function resolveConfig(input, baseDir) {
297
331
  }
298
332
 
299
333
  // src/core/introspect-schema.ts
334
+ function getArrayElement(schema) {
335
+ if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
336
+ return schema.element;
337
+ }
338
+ return null;
339
+ }
300
340
  function introspectSchema(schema, prefix = []) {
301
341
  const relation = getRelationTarget(schema);
302
342
  if (relation && prefix.length > 0) {
@@ -310,7 +350,10 @@ function introspectSchema(schema, prefix = []) {
310
350
  }
311
351
  ];
312
352
  }
313
- const base = unwrapSchema(schema);
353
+ if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
354
+ return [{ path: prefix, kind: "translatable" }];
355
+ }
356
+ const base = peelOptionalWrappers(schema);
314
357
  if (base instanceof Object && "shape" in base) {
315
358
  const shape = base.shape;
316
359
  const fields = [];
@@ -319,9 +362,9 @@ function introspectSchema(schema, prefix = []) {
319
362
  }
320
363
  return fields;
321
364
  }
322
- if (base instanceof Object && "element" in base) {
323
- const element = base.element;
324
- const elementBase = unwrapSchema(element);
365
+ const element = getArrayElement(base);
366
+ if (element) {
367
+ const elementBase = peelOptionalWrappers(element);
325
368
  if (elementBase instanceof Object && "shape" in elementBase) {
326
369
  const shape = elementBase.shape;
327
370
  const fields = [];
@@ -333,15 +376,48 @@ function introspectSchema(schema, prefix = []) {
333
376
  }
334
377
  return [{ path: prefix, kind: getFieldKind(schema) }];
335
378
  }
336
- function extractByPaths(data, paths) {
337
- const out = {};
379
+ function buildPathTrie(paths) {
380
+ const root = { leaf: false, children: /* @__PURE__ */ new Map() };
338
381
  for (const path13 of paths) {
339
- const value = getAtPath(data, path13);
382
+ let node = root;
383
+ for (const segment of path13) {
384
+ let child = node.children.get(segment);
385
+ if (!child) {
386
+ child = { leaf: false, children: /* @__PURE__ */ new Map() };
387
+ node.children.set(segment, child);
388
+ }
389
+ node = child;
390
+ }
391
+ node.leaf = true;
392
+ }
393
+ return root;
394
+ }
395
+ function extractNode(data, trie) {
396
+ if (trie.leaf) return data;
397
+ const star = trie.children.get("*");
398
+ if (star) {
399
+ if (!Array.isArray(data)) return void 0;
400
+ return data.map(
401
+ (item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
402
+ );
403
+ }
404
+ if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
405
+ const record = data;
406
+ const out = {};
407
+ let any = false;
408
+ for (const [key, child] of trie.children) {
409
+ const value = extractNode(record[key], child);
340
410
  if (value !== void 0) {
341
- setAtPath(out, path13.filter((p) => p !== "*"), value);
411
+ out[key] = value;
412
+ any = true;
342
413
  }
343
414
  }
344
- return out;
415
+ return any ? out : void 0;
416
+ }
417
+ function extractByPaths(data, paths) {
418
+ if (paths.length === 0) return {};
419
+ const extracted = extractNode(data, buildPathTrie(paths));
420
+ return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
345
421
  }
346
422
  function pickTranslatable(data, schema) {
347
423
  const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
@@ -356,32 +432,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
356
432
  const translatable = pickTranslatable(localeData, schema);
357
433
  return deepMerge(structural, translatable);
358
434
  }
359
- function getAtPath(obj, path13) {
360
- let current = obj;
361
- for (const segment of path13) {
362
- if (segment === "*") {
363
- if (!Array.isArray(current)) return void 0;
364
- return current.map(
365
- (item) => typeof item === "object" && item !== null ? getAtPath(item, path13.slice(path13.indexOf("*") + 1)) : void 0
366
- );
367
- }
368
- if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
369
- current = current[segment];
370
- }
371
- return current;
372
- }
373
- function setAtPath(obj, path13, value) {
374
- if (path13.length === 0) return;
375
- if (path13.length === 1) {
376
- obj[path13[0]] = value;
377
- return;
378
- }
379
- const [head, ...rest] = path13;
380
- if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
381
- obj[head] = {};
382
- }
383
- setAtPath(obj[head], rest, value);
384
- }
385
435
  function deepMerge(base, overlay) {
386
436
  const out = { ...base };
387
437
  for (const [key, value] of Object.entries(overlay)) {
@@ -551,7 +601,7 @@ function seoFieldsFromEn(enDoc) {
551
601
  canonicalPathOverride: enDoc.canonicalPathOverride
552
602
  };
553
603
  }
554
- var SCHEMA_VERSION = 4;
604
+ var SCHEMA_VERSION = 5;
555
605
  var MIGRATIONS = [
556
606
  `CREATE TABLE IF NOT EXISTS meta (
557
607
  key TEXT PRIMARY KEY,
@@ -599,7 +649,30 @@ var MIGRATIONS = [
599
649
  UNIQUE (content_type, en_slug, en_hash)
600
650
  )`,
601
651
  `CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
602
- ON en_snapshots(content_type, en_slug, created_at DESC)`
652
+ ON en_snapshots(content_type, en_slug, created_at DESC)`,
653
+ `CREATE TABLE IF NOT EXISTS translation_batch_jobs (
654
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
655
+ job_name TEXT NOT NULL UNIQUE,
656
+ model TEXT NOT NULL,
657
+ display_model TEXT NOT NULL,
658
+ created_at TEXT NOT NULL,
659
+ state TEXT NOT NULL,
660
+ completed_at TEXT
661
+ )`,
662
+ `CREATE TABLE IF NOT EXISTS translation_batch_items (
663
+ job_id INTEGER NOT NULL,
664
+ request_index INTEGER NOT NULL,
665
+ content_type TEXT NOT NULL,
666
+ en_slug TEXT NOT NULL,
667
+ locale TEXT NOT NULL,
668
+ en_hash TEXT NOT NULL,
669
+ snapshot_id INTEGER NOT NULL,
670
+ status TEXT NOT NULL,
671
+ error TEXT,
672
+ PRIMARY KEY (job_id, request_index)
673
+ )`,
674
+ `CREATE INDEX IF NOT EXISTS idx_batch_items_status
675
+ ON translation_batch_items(status)`
603
676
  ];
604
677
  function resolveStorePath(config) {
605
678
  return config.storePath;
@@ -622,6 +695,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
622
695
  }
623
696
  }
624
697
  function readSchemaVersion(db) {
698
+ const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
699
+ if (!metaTable) return 0;
625
700
  const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
626
701
  return row ? Number.parseInt(row.value, 10) || 0 : 0;
627
702
  }
@@ -660,6 +735,9 @@ function getOrCreateEnSnapshot(db, input) {
660
735
  ).get(input.contentType, input.enSlug, input.enHash);
661
736
  return row.id;
662
737
  }
738
+ function getEnSnapshot(db, snapshotId) {
739
+ return db.prepare(`SELECT * FROM en_snapshots WHERE id = ?`).get(snapshotId);
740
+ }
663
741
  function upsertTranslation(db, input) {
664
742
  db.prepare(
665
743
  `INSERT INTO translations (
@@ -740,7 +818,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
740
818
  if (i >= body.length || body[i] === ">" || body[i] === "/") break;
741
819
  const attrStart = i;
742
820
  while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
743
- body.slice(attrStart, i);
821
+ const attrName = body.slice(attrStart, i);
822
+ if (!attrName) break;
744
823
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
745
824
  if (body[i] !== "=") {
746
825
  tagOut += body.slice(attrStart, i);
@@ -749,10 +828,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
749
828
  tagOut += body.slice(attrStart, i);
750
829
  tagOut += "=";
751
830
  i += 1;
831
+ const wsStart = i;
752
832
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
833
+ tagOut += body.slice(wsStart, i);
753
834
  const quote = body[i];
754
835
  if (quote !== '"' && quote !== "'") {
755
- continue;
836
+ break;
756
837
  }
757
838
  if (quote === "'") {
758
839
  const valStart2 = i + 1;
@@ -1072,7 +1153,7 @@ function getTranslatablePayload(doc, type) {
1072
1153
  }
1073
1154
 
1074
1155
  // src/i18n/resolve-document.ts
1075
- function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
1156
+ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
1076
1157
  const urlBuilder = createUrlBuilder({
1077
1158
  locales: [defaultLocale, locale],
1078
1159
  defaultLocale,
@@ -1086,15 +1167,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1086
1167
  for (const [docLocale, docIdx] of allDocs) {
1087
1168
  const found = docIdx.bySlug.get(slug);
1088
1169
  if (!found) continue;
1089
- const correctSlug = getSlugForLocale(found, docLocale, locale, allDocs, defaultLocale);
1090
- if (correctSlug && correctSlug !== slug) {
1170
+ const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
1171
+ let target;
1172
+ let targetLocale = locale;
1173
+ for (const candidateLocale of [locale, ...fallbackLocales]) {
1174
+ const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
1175
+ if (cand) {
1176
+ target = cand;
1177
+ targetLocale = candidateLocale;
1178
+ break;
1179
+ }
1180
+ }
1181
+ if (target) {
1182
+ if (target.slug === slug) {
1183
+ return { document: target, actualLocale: targetLocale };
1184
+ }
1091
1185
  if (!type.path) {
1092
1186
  return { document: null, actualLocale: locale };
1093
1187
  }
1094
1188
  return {
1095
1189
  document: null,
1096
1190
  actualLocale: locale,
1097
- shouldRedirectTo: urlBuilder.resolvePath(type.path, correctSlug, locale)
1191
+ shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
1098
1192
  };
1099
1193
  }
1100
1194
  if (docLocale === defaultLocale) {
@@ -1125,13 +1219,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1125
1219
  }
1126
1220
  return { document: null, actualLocale: locale };
1127
1221
  }
1128
- function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
1129
- if (sourceLocale === targetLocale) return document2.slug;
1130
- const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
1131
- if (!englishSlug) return null;
1132
- if (targetLocale === defaultLocale) return englishSlug;
1133
- return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
1134
- }
1135
1222
 
1136
1223
  // src/create-project.ts
1137
1224
  function comparatorFor(orderBy) {
@@ -1184,7 +1271,8 @@ function buildRuntime(config, type, getRuntime) {
1184
1271
  config.defaultLocale,
1185
1272
  load(),
1186
1273
  type,
1187
- config.localeRouting
1274
+ config.localeRouting,
1275
+ config.localeFallbacks[locale] ?? []
1188
1276
  );
1189
1277
  if (result.document && type.path) {
1190
1278
  return {
@@ -1206,8 +1294,14 @@ function buildRuntime(config, type, getRuntime) {
1206
1294
  const params = [];
1207
1295
  for (const locale of options.locales ?? config.locales) {
1208
1296
  const localeIdx = all.get(locale);
1297
+ const fallbacks = config.localeFallbacks[locale] ?? [];
1209
1298
  for (const doc of enIdx.bySlug.values()) {
1210
- const slug = locale === config.defaultLocale ? doc.slug : localeIdx?.byEnSlug.get(doc.slug)?.slug ?? doc.slug;
1299
+ let slug;
1300
+ if (locale === config.defaultLocale) {
1301
+ slug = doc.slug;
1302
+ } else {
1303
+ slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
1304
+ }
1211
1305
  params.push({ locale, slug });
1212
1306
  }
1213
1307
  }
@@ -2026,7 +2120,7 @@ function validateTypeRedirects(project) {
2026
2120
  }
2027
2121
  return issues;
2028
2122
  }
2029
- function getAtPath2(obj, fieldPath) {
2123
+ function getAtPath(obj, fieldPath) {
2030
2124
  let current = obj;
2031
2125
  for (const segment of fieldPath) {
2032
2126
  if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
@@ -2071,7 +2165,7 @@ function validateRelations(project) {
2071
2165
  });
2072
2166
  continue;
2073
2167
  }
2074
- const value = getAtPath2(doc.frontmatter, fieldMeta.path);
2168
+ const value = getAtPath(doc.frontmatter, fieldMeta.path);
2075
2169
  if (value === void 0 || value === null || value === "") continue;
2076
2170
  const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
2077
2171
  const fieldLabel = fieldMeta.path.join(".");
@@ -2438,6 +2532,59 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
2438
2532
  return config.locales.filter((l) => l !== config.defaultLocale);
2439
2533
  }
2440
2534
 
2535
+ // src/storage/batch-jobs.ts
2536
+ function insertBatchJob(db, input) {
2537
+ const info = db.prepare(
2538
+ `INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
2539
+ VALUES (?, ?, ?, ?, ?)`
2540
+ ).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
2541
+ return Number(info.lastInsertRowid);
2542
+ }
2543
+ function insertBatchItems(db, jobId, items) {
2544
+ const stmt = db.prepare(
2545
+ `INSERT INTO translation_batch_items (
2546
+ job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
2547
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
2548
+ );
2549
+ for (const item of items) {
2550
+ stmt.run(
2551
+ jobId,
2552
+ item.requestIndex,
2553
+ item.contentType,
2554
+ item.enSlug,
2555
+ item.locale,
2556
+ item.enHash,
2557
+ item.snapshotId
2558
+ );
2559
+ }
2560
+ }
2561
+ function listPendingBatchJobs(db) {
2562
+ return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
2563
+ }
2564
+ function listBatchItems(db, jobId) {
2565
+ return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
2566
+ }
2567
+ function listPendingBatchItems(db) {
2568
+ return db.prepare(
2569
+ `SELECT i.* FROM translation_batch_items i
2570
+ JOIN translation_batch_jobs j ON j.id = i.job_id
2571
+ WHERE j.completed_at IS NULL AND i.status = 'pending'
2572
+ ORDER BY i.job_id, i.request_index`
2573
+ ).all();
2574
+ }
2575
+ function claimBatchJobCompletion(db, jobId, state, completedAt) {
2576
+ const info = db.prepare(
2577
+ `UPDATE translation_batch_jobs SET state = ?, completed_at = ?
2578
+ WHERE id = ? AND completed_at IS NULL`
2579
+ ).run(state, completedAt, jobId);
2580
+ return info.changes > 0;
2581
+ }
2582
+ function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
2583
+ db.prepare(
2584
+ `UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
2585
+ ).run(status, error ?? null, jobId, requestIndex);
2586
+ }
2587
+
2441
2588
  // src/history/record-snapshot.ts
2442
2589
  function recordEnSnapshot(config, input, db) {
2443
2590
  const ownDb = db ?? openStore(config, "readwrite");
@@ -2452,8 +2599,102 @@ function recordEnSnapshot(config, input, db) {
2452
2599
  if (!db) ownDb.close();
2453
2600
  return id;
2454
2601
  }
2602
+ var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
2603
+ function statusFromError(error) {
2604
+ if (error instanceof genai.ApiError && typeof error.status === "number") {
2605
+ return error.status;
2606
+ }
2607
+ if (error && typeof error === "object" && "status" in error) {
2608
+ const status = error.status;
2609
+ if (typeof status === "number") return status;
2610
+ }
2611
+ const message = error instanceof Error ? error.message : String(error);
2612
+ const match = message.match(/\b(429|500|502|503|504)\b/);
2613
+ if (match) return Number(match[1]);
2614
+ return void 0;
2615
+ }
2616
+ function isNetworkError(error) {
2617
+ const err = error;
2618
+ const code = typeof err?.code === "string" ? err.code : "";
2619
+ if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
2620
+ return true;
2621
+ }
2622
+ const message = error instanceof Error ? error.message.toLowerCase() : "";
2623
+ return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
2624
+ }
2625
+ function isTransientError(error) {
2626
+ const status = statusFromError(error);
2627
+ if (status !== void 0) return TRANSIENT_STATUS.has(status);
2628
+ return isNetworkError(error);
2629
+ }
2630
+ var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
2631
+ async function withRetry(fn, options = {}) {
2632
+ const attempts = Math.max(1, options.attempts ?? 3);
2633
+ const baseDelayMs = options.baseDelayMs ?? 1e3;
2634
+ const sleep2 = options.sleep ?? defaultSleep;
2635
+ const random = options.random ?? Math.random;
2636
+ let lastError;
2637
+ for (let attempt = 1; attempt <= attempts; attempt++) {
2638
+ try {
2639
+ return await fn();
2640
+ } catch (error) {
2641
+ lastError = error;
2642
+ if (attempt >= attempts || !isTransientError(error)) throw error;
2643
+ const backoff = baseDelayMs * 2 ** (attempt - 1);
2644
+ const delay = backoff + random() * backoff;
2645
+ await sleep2(delay);
2646
+ }
2647
+ }
2648
+ throw lastError;
2649
+ }
2455
2650
 
2456
- // src/translate/gemini-models.ts
2651
+ // src/translate/gemini-batch.ts
2652
+ var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
2653
+ var TERMINAL_STATES = /* @__PURE__ */ new Set([
2654
+ "SUCCEEDED",
2655
+ "FAILED",
2656
+ "CANCELLED",
2657
+ "EXPIRED",
2658
+ "PARTIALLY_SUCCEEDED"
2659
+ ]);
2660
+ var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
2661
+ function normalizeBatchState(state) {
2662
+ return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
2663
+ }
2664
+ function isTerminalBatchState(state) {
2665
+ return TERMINAL_STATES.has(normalizeBatchState(state));
2666
+ }
2667
+ function isSuccessfulBatchState(state) {
2668
+ return SUCCESSFUL_STATES.has(normalizeBatchState(state));
2669
+ }
2670
+ function makeClient(apiKey) {
2671
+ const key = apiKey ?? process.env.GEMINI_API_KEY;
2672
+ if (!key) {
2673
+ throw new Error("GEMINI_API_KEY is required for scribe translate");
2674
+ }
2675
+ return new genai.GoogleGenAI({ apiKey: key });
2676
+ }
2677
+ function textFromBatchResponse(response) {
2678
+ if (typeof response.text === "string") return response.text;
2679
+ const parts = response.candidates?.[0]?.content?.parts ?? [];
2680
+ return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
2681
+ }
2682
+ async function createGeminiBatchJob(input) {
2683
+ const ai = makeClient(input.apiKey);
2684
+ const job = await withRetry(
2685
+ () => ai.batches.create({
2686
+ model: input.model,
2687
+ src: input.requests,
2688
+ config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
2689
+ })
2690
+ );
2691
+ if (!job.name) throw new Error("Gemini batch job was created without a name");
2692
+ return job;
2693
+ }
2694
+ async function getGeminiBatchJob(input) {
2695
+ const ai = makeClient(input.apiKey);
2696
+ return withRetry(() => ai.batches.get({ name: input.name }));
2697
+ }
2457
2698
  var GEMINI_MODEL_IDS = {
2458
2699
  "gemini-2.5-pro": "gemini-2.5-pro",
2459
2700
  "gemini-3.1-pro": "gemini-3.1-pro-preview"
@@ -2473,6 +2714,12 @@ function normalizeGeminiDisplayName(model) {
2473
2714
  if (alias) return alias[0];
2474
2715
  return name;
2475
2716
  }
2717
+ function resolveThinkingConfig(apiModelId) {
2718
+ const id = stripModelsPrefix(apiModelId).toLowerCase();
2719
+ if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
2720
+ if (id.includes("2.5")) return { thinkingBudget: 128 };
2721
+ return void 0;
2722
+ }
2476
2723
 
2477
2724
  // src/translate/gemini-client.ts
2478
2725
  var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
@@ -2486,11 +2733,36 @@ function extractJson(text) {
2486
2733
  return trimmed;
2487
2734
  }
2488
2735
  function parseGeminiResponse(text) {
2736
+ let parsed;
2489
2737
  try {
2490
- return JSON.parse(text.trim());
2738
+ parsed = JSON.parse(text.trim());
2491
2739
  } catch {
2492
- return JSON.parse(extractJson(text));
2740
+ parsed = JSON.parse(extractJson(text));
2741
+ }
2742
+ if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
2743
+ parsed.frontmatter = {};
2493
2744
  }
2745
+ return parsed;
2746
+ }
2747
+ function buildGeminiRequestConfig(input) {
2748
+ const thinkingConfig = resolveThinkingConfig(input.apiModelId);
2749
+ return {
2750
+ responseMimeType: "application/json",
2751
+ ...input.responseSchema ? { responseSchema: input.responseSchema } : {},
2752
+ ...thinkingConfig ? { thinkingConfig } : {}
2753
+ };
2754
+ }
2755
+ function usageFromResponse(response) {
2756
+ const meta = response.usageMetadata ?? response.usage_metadata ?? {};
2757
+ const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
2758
+ return {
2759
+ inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
2760
+ // candidatesTokenCount does not include thoughts, but thoughts are billed as
2761
+ // output tokens, so fold them in here for accurate cost accounting.
2762
+ outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
2763
+ thoughtsTokens,
2764
+ totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
2765
+ };
2494
2766
  }
2495
2767
  async function translatePageWithGemini(input) {
2496
2768
  const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
@@ -2502,26 +2774,22 @@ async function translatePageWithGemini(input) {
2502
2774
  );
2503
2775
  const apiModel = resolveGeminiModelId(displayModel);
2504
2776
  const ai = new genai.GoogleGenAI({ apiKey });
2505
- const response = await ai.models.generateContent({
2506
- model: apiModel,
2507
- contents: input.prompt,
2508
- config: {
2509
- responseMimeType: "application/json",
2510
- ...input.responseSchema ? { responseSchema: input.responseSchema } : {}
2511
- }
2512
- });
2777
+ const response = await withRetry(
2778
+ () => ai.models.generateContent({
2779
+ model: apiModel,
2780
+ contents: input.prompt,
2781
+ config: buildGeminiRequestConfig({
2782
+ apiModelId: apiModel,
2783
+ responseSchema: input.responseSchema
2784
+ })
2785
+ })
2786
+ );
2513
2787
  const raw = response.text ?? "";
2514
2788
  const parsed = parseGeminiResponse(raw);
2515
2789
  if (!parsed.frontmatter || typeof parsed.body !== "string") {
2516
2790
  throw new Error("Gemini response missing frontmatter/body");
2517
2791
  }
2518
- const usageMetadata = response.usageMetadata;
2519
- const usage = {
2520
- inputTokens: usageMetadata?.promptTokenCount ?? 0,
2521
- outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
2522
- totalTokens: usageMetadata?.totalTokenCount ?? 0
2523
- };
2524
- return { model: displayModel, raw, parsed, usage };
2792
+ return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
2525
2793
  }
2526
2794
 
2527
2795
  // src/translate/gemini-pricing.ts
@@ -2541,12 +2809,14 @@ function resolveModelPricing(model) {
2541
2809
  const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
2542
2810
  return match?.[1];
2543
2811
  }
2544
- function estimateTranslationCostUsd(model, inputTokens, outputTokens) {
2812
+ var BATCH_DISCOUNT = 0.5;
2813
+ function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
2545
2814
  const pricing = resolveModelPricing(model);
2546
2815
  if (!pricing) return void 0;
2547
2816
  const inputRate = tierRate(inputTokens, pricing.input);
2548
2817
  const outputRate = tierRate(outputTokens, pricing.output);
2549
- return inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate;
2818
+ const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
2819
+ return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
2550
2820
  }
2551
2821
 
2552
2822
  // src/translate/prompts/translation-prompt.ts
@@ -2562,6 +2832,7 @@ var LOCALE_NAMES = {
2562
2832
  "zh-CN": "Simplified Chinese",
2563
2833
  "zh-TW": "Traditional Chinese",
2564
2834
  pt: "Portuguese",
2835
+ "pt-BR": "Brazilian Portuguese",
2565
2836
  ja: "Japanese",
2566
2837
  ru: "Russian",
2567
2838
  it: "Italian",
@@ -2569,29 +2840,29 @@ var LOCALE_NAMES = {
2569
2840
  };
2570
2841
  function defaultLocalizationPrompt(localeName, locale) {
2571
2842
  return [
2572
- `Localize the content below into natural ${localeName} (${locale}).`,
2843
+ `Localize the content above into natural ${localeName} (${locale}).`,
2573
2844
  "Do not translate word-for-word.",
2574
2845
  "Preserve the source tone and brand voice.",
2575
2846
  "Write as if a native speaker authored it for the target market."
2576
2847
  ].join(" ");
2577
2848
  }
2849
+ var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
2850
+ function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
2851
+ if (!hasFrontmatter) {
2852
+ return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
2853
+ }
2854
+ return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
2855
+ }
2578
2856
  function buildPageTranslationPrompt(input) {
2579
2857
  const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
2580
- const prompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2581
- const lines = [
2582
- prompt,
2858
+ const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2859
+ const prefix = [
2860
+ TASK_FRAMING,
2583
2861
  "",
2584
2862
  ...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
2585
2863
  ...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
2586
2864
  "## Rules",
2587
2865
  ...input.resolved.rules.map((rule) => `- ${rule}`),
2588
- ...input.slugStrategy === "localized" ? [
2589
- `- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
2590
- ] : [],
2591
- "",
2592
- "## Output format",
2593
- "Return ONLY valid JSON with keys:",
2594
- input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
2595
2866
  "",
2596
2867
  "## EN translatable frontmatter (JSON)",
2597
2868
  JSON.stringify(input.translatableFrontmatter, null, 2),
@@ -2599,7 +2870,22 @@ function buildPageTranslationPrompt(input) {
2599
2870
  "## EN body (MDX)",
2600
2871
  input.enBody
2601
2872
  ];
2602
- return lines.join("\n");
2873
+ const suffix = [
2874
+ "",
2875
+ "## Target language",
2876
+ localizationPrompt,
2877
+ ...input.slugStrategy === "localized" ? [
2878
+ `The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
2879
+ ] : [],
2880
+ "",
2881
+ "## Output format",
2882
+ "Return ONLY valid JSON with keys:",
2883
+ buildOutputFormatLine(
2884
+ Object.keys(input.translatableFrontmatter).length > 0,
2885
+ input.slugStrategy
2886
+ )
2887
+ ];
2888
+ return [...prefix, ...suffix].join("\n");
2603
2889
  }
2604
2890
  function getObjectShape(schema) {
2605
2891
  const base = peelOptionalWrappers(schema);
@@ -2647,9 +2933,8 @@ function buildTranslatableSubschema(schema) {
2647
2933
  function buildGeminiResponseSchema(schema, slugStrategy) {
2648
2934
  try {
2649
2935
  const translatable = buildTranslatableSubschema(schema);
2650
- if (!translatable) return null;
2651
2936
  const responseShape = {
2652
- frontmatter: translatable,
2937
+ ...translatable ? { frontmatter: translatable } : {},
2653
2938
  body: zod.z.string()
2654
2939
  };
2655
2940
  if (slugStrategy === "localized") {
@@ -2728,7 +3013,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
2728
3013
  return { ok: true, frontmatter };
2729
3014
  }
2730
3015
 
2731
- // src/translate/page-translator.ts
3016
+ // src/translate/translate-core.ts
3017
+ function translationItemKey(item) {
3018
+ const contentType = item.contentType ?? item.content_type ?? "";
3019
+ const enSlug = item.enSlug ?? item.en_slug ?? "";
3020
+ return `${contentType}\0${enSlug}\0${item.locale}`;
3021
+ }
2732
3022
  function summarizeResults(results, durationMs) {
2733
3023
  return results.reduce(
2734
3024
  (totals, result) => {
@@ -2764,17 +3054,6 @@ function formatTranslateError(error) {
2764
3054
  }
2765
3055
  return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
2766
3056
  }
2767
- async function runPool(items, concurrency, worker) {
2768
- if (items.length === 0) return;
2769
- let nextIndex = 0;
2770
- const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
2771
- while (nextIndex < items.length) {
2772
- const index = nextIndex++;
2773
- await worker(items[index], index);
2774
- }
2775
- });
2776
- await Promise.all(runners);
2777
- }
2778
3057
  function resolveContextLabel(enDoc, enSlug) {
2779
3058
  const fm = enDoc.frontmatter;
2780
3059
  for (const key of ["title", "name", "h1"]) {
@@ -2783,73 +3062,82 @@ function resolveContextLabel(enDoc, enSlug) {
2783
3062
  }
2784
3063
  return enSlug;
2785
3064
  }
2786
- async function translatePage(config, item, options = {}) {
2787
- const startedAt = Date.now();
2788
- const base = {
3065
+ function baseForItem(item) {
3066
+ return {
2789
3067
  contentType: item.contentType,
2790
3068
  enSlug: item.enSlug,
2791
3069
  locale: item.locale
2792
3070
  };
2793
- try {
2794
- const type = config.types.find((t) => t.id === item.contentType);
2795
- if (!type) throw new Error(`Unknown content type ${item.contentType}`);
2796
- const enDoc = readEnDocument(config, type, item.enSlug);
2797
- if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
2798
- const payload = getTranslatablePayload(enDoc, type);
2799
- const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
2800
- const db = openStore(config, "readonly");
2801
- const existing = getTranslation(db, type.id, item.enSlug, item.locale);
2802
- db.close();
2803
- if (!options.force && existing && existing.en_hash === currentEnHash) {
2804
- return {
2805
- ...base,
3071
+ }
3072
+ function prepareTranslation(config, item, options, startedAt) {
3073
+ const type = config.types.find((t) => t.id === item.contentType);
3074
+ if (!type) throw new Error(`Unknown content type ${item.contentType}`);
3075
+ const enDoc = readEnDocument(config, type, item.enSlug);
3076
+ if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
3077
+ const payload = getTranslatablePayload(enDoc, type);
3078
+ const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
3079
+ const db = openStore(config, "readonly");
3080
+ const existing = getTranslation(db, type.id, item.enSlug, item.locale);
3081
+ db.close();
3082
+ if (!options.force && existing && existing.en_hash === currentEnHash) {
3083
+ return {
3084
+ status: "done",
3085
+ result: {
3086
+ ...baseForItem(item),
2806
3087
  skipped: true,
2807
3088
  reason: "fresh",
2808
3089
  durationMs: Date.now() - startedAt
2809
- };
2810
- }
2811
- const resolvedTranslate = resolveTranslateConfig(config, type);
2812
- const model = options.model ?? resolvedTranslate.model;
2813
- const prompt = buildPageTranslationPrompt({
2814
- resolved: resolvedTranslate,
2815
- targetLocale: item.locale,
2816
- contextLabel: resolveContextLabel(enDoc, item.enSlug),
2817
- translatableFrontmatter: payload.frontmatter,
2818
- enBody: payload.body,
2819
- slugStrategy: type.slugStrategy
2820
- });
2821
- if (options.dryRun) {
2822
- return {
2823
- ...base,
2824
- skipped: false,
2825
- model,
2826
- durationMs: Date.now() - startedAt
2827
- };
2828
- }
2829
- const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
2830
- const result = await translatePageWithGemini({
2831
- prompt,
3090
+ }
3091
+ };
3092
+ }
3093
+ const resolvedTranslate = resolveTranslateConfig(config, type);
3094
+ const model = options.model ?? resolvedTranslate.model;
3095
+ const prompt = buildPageTranslationPrompt({
3096
+ resolved: resolvedTranslate,
3097
+ targetLocale: item.locale,
3098
+ contextLabel: resolveContextLabel(enDoc, item.enSlug),
3099
+ translatableFrontmatter: payload.frontmatter,
3100
+ enBody: payload.body,
3101
+ slugStrategy: type.slugStrategy
3102
+ });
3103
+ const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
3104
+ return {
3105
+ status: "ready",
3106
+ prepared: {
3107
+ item,
3108
+ type,
3109
+ enDoc,
3110
+ payload,
3111
+ currentEnHash,
3112
+ existingSlug: existing?.slug,
2832
3113
  model,
3114
+ prompt,
2833
3115
  responseSchema: responseSchema ?? void 0
2834
- });
2835
- const rawSlug = type.slugStrategy === "localized" ? result.parsed.slug ?? existing?.slug ?? item.enSlug : item.enSlug;
3116
+ }
3117
+ };
3118
+ }
3119
+ function finalizeTranslation(config, prepared, output, options) {
3120
+ const { item, type, enDoc, payload } = prepared;
3121
+ const base = baseForItem(item);
3122
+ try {
3123
+ const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
2836
3124
  const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
2837
3125
  const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
2838
3126
  const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
2839
- const validated = validateTranslatedFrontmatter(enDoc, result.parsed.frontmatter, type.schema);
3127
+ const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
2840
3128
  if (!validated.ok) {
2841
3129
  throw new Error(`Translation validation failed: ${validated.error}`);
2842
3130
  }
2843
3131
  const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
2844
- result.parsed.body
3132
+ output.parsed.body
2845
3133
  );
2846
3134
  const writeDb = openStore(config, "readwrite");
2847
- const snapshotId = recordEnSnapshot(
3135
+ const snapshotId = options.snapshotId ?? recordEnSnapshot(
2848
3136
  config,
2849
3137
  {
2850
3138
  contentType: type.id,
2851
3139
  enSlug: item.enSlug,
2852
- enHash: currentEnHash,
3140
+ enHash: prepared.currentEnHash,
2853
3141
  frontmatter: payload.frontmatter,
2854
3142
  body: payload.body
2855
3143
  },
@@ -2862,24 +3150,25 @@ async function translatePage(config, item, options = {}) {
2862
3150
  slug,
2863
3151
  frontmatter: validated.frontmatter,
2864
3152
  body: translatedBody,
2865
- enHash: currentEnHash,
3153
+ enHash: prepared.currentEnHash,
2866
3154
  translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
2867
- model: result.model,
3155
+ model: output.model,
2868
3156
  snapshotId
2869
3157
  });
2870
3158
  writeDb.close();
2871
3159
  const estimatedCostUsd = estimateTranslationCostUsd(
2872
- normalizeGeminiDisplayName(result.model),
2873
- result.usage.inputTokens,
2874
- result.usage.outputTokens
3160
+ normalizeGeminiDisplayName(output.model),
3161
+ output.usage.inputTokens,
3162
+ output.usage.outputTokens,
3163
+ options.costMode
2875
3164
  );
2876
3165
  return {
2877
3166
  ...base,
2878
3167
  skipped: false,
2879
- model: result.model,
2880
- usage: result.usage,
3168
+ model: output.model,
3169
+ usage: output.usage,
2881
3170
  estimatedCostUsd,
2882
- durationMs: Date.now() - startedAt,
3171
+ durationMs: Date.now() - options.startedAt,
2883
3172
  slugAdjusted,
2884
3173
  mdxAdjusted: mdxAdjusted || void 0
2885
3174
  };
@@ -2889,33 +3178,559 @@ async function translatePage(config, item, options = {}) {
2889
3178
  skipped: false,
2890
3179
  failed: true,
2891
3180
  error: formatTranslateError(error),
3181
+ durationMs: Date.now() - options.startedAt
3182
+ };
3183
+ }
3184
+ }
3185
+ function displayModelFor(prepared) {
3186
+ return normalizeGeminiDisplayName(
3187
+ prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
3188
+ );
3189
+ }
3190
+
3191
+ // src/translate/batch-worklist.ts
3192
+ var MAX_REQUESTS_PER_JOB = 100;
3193
+ var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
3194
+ var INITIAL_POLL_MS = 5e3;
3195
+ var MAX_POLL_MS = 3e4;
3196
+ var POLL_BACKOFF_FACTOR = 1.5;
3197
+ function sleep(ms) {
3198
+ return new Promise((resolve) => setTimeout(resolve, ms));
3199
+ }
3200
+ function planBatchJobs(entries) {
3201
+ const byModel = /* @__PURE__ */ new Map();
3202
+ for (const entry of entries) {
3203
+ const group = byModel.get(entry.apiModel);
3204
+ if (group) group.push(entry);
3205
+ else byModel.set(entry.apiModel, [entry]);
3206
+ }
3207
+ const plans = [];
3208
+ for (const [apiModel, group] of byModel) {
3209
+ let current = [];
3210
+ let currentBytes = 0;
3211
+ for (const entry of group) {
3212
+ const size = Buffer.byteLength(entry.prompt, "utf8");
3213
+ if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
3214
+ plans.push({ apiModel, entries: current });
3215
+ current = [];
3216
+ currentBytes = 0;
3217
+ }
3218
+ current.push(entry);
3219
+ currentBytes += size;
3220
+ }
3221
+ if (current.length > 0) plans.push({ apiModel, entries: current });
3222
+ }
3223
+ return plans;
3224
+ }
3225
+ function readPendingBatchWork(config) {
3226
+ const db = openStore(config, "readwrite");
3227
+ const jobs = listPendingBatchJobs(db);
3228
+ const pendingItems = listPendingBatchItems(db);
3229
+ db.close();
3230
+ const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
3231
+ return { jobs, inFlightKeys, pendingItems };
3232
+ }
3233
+ async function submitBatchJobPlan(config, input) {
3234
+ const { plan, displayModel, jobIndex, jobCount } = input;
3235
+ const job = await createGeminiBatchJob({
3236
+ model: plan.apiModel,
3237
+ requests: plan.entries.map(({ prepared }) => ({
3238
+ contents: prepared.prompt,
3239
+ config: buildGeminiRequestConfig({
3240
+ apiModelId: plan.apiModel,
3241
+ responseSchema: prepared.responseSchema
3242
+ })
3243
+ })),
3244
+ displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
3245
+ });
3246
+ const createdAt = (/* @__PURE__ */ new Date()).toISOString();
3247
+ const db = openStore(config, "readwrite");
3248
+ const jobId = insertBatchJob(db, {
3249
+ jobName: job.name,
3250
+ model: plan.apiModel,
3251
+ displayModel,
3252
+ state: String(job.state ?? "JOB_STATE_PENDING"),
3253
+ createdAt
3254
+ });
3255
+ insertBatchItems(
3256
+ db,
3257
+ jobId,
3258
+ plan.entries.map(({ prepared }, requestIndex) => ({
3259
+ requestIndex,
3260
+ contentType: prepared.item.contentType,
3261
+ enSlug: prepared.item.enSlug,
3262
+ locale: prepared.item.locale,
3263
+ enHash: prepared.currentEnHash,
3264
+ // Snapshot the EN source now so ingestion after a resume does not depend
3265
+ // on the EN files still matching (or existing) on disk.
3266
+ snapshotId: recordEnSnapshot(
3267
+ config,
3268
+ {
3269
+ contentType: prepared.item.contentType,
3270
+ enSlug: prepared.item.enSlug,
3271
+ enHash: prepared.currentEnHash,
3272
+ frontmatter: prepared.payload.frontmatter,
3273
+ body: prepared.payload.body
3274
+ },
3275
+ db
3276
+ )
3277
+ }))
3278
+ );
3279
+ const row = {
3280
+ id: jobId,
3281
+ job_name: job.name,
3282
+ model: plan.apiModel,
3283
+ display_model: displayModel,
3284
+ created_at: createdAt,
3285
+ state: String(job.state ?? "JOB_STATE_PENDING"),
3286
+ completed_at: null
3287
+ };
3288
+ db.close();
3289
+ input.onProgress?.({
3290
+ type: "batch-submitted",
3291
+ name: job.name,
3292
+ count: plan.entries.length,
3293
+ model: displayModel,
3294
+ jobIndex,
3295
+ jobCount,
3296
+ createdAt
3297
+ });
3298
+ return row;
3299
+ }
3300
+ function buildIngestContext(config, db, jobRow, itemRow) {
3301
+ const type = config.types.find((t) => t.id === itemRow.content_type);
3302
+ if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
3303
+ let enDoc = readEnDocument(config, type, itemRow.en_slug);
3304
+ if (!enDoc) {
3305
+ const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
3306
+ if (!snapshot) {
3307
+ throw new Error(
3308
+ `EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
3309
+ );
3310
+ }
3311
+ enDoc = {
3312
+ slug: itemRow.en_slug,
3313
+ enSlug: itemRow.en_slug,
3314
+ locale: config.defaultLocale,
3315
+ noindex: false,
3316
+ frontmatter: JSON.parse(snapshot.frontmatter_json),
3317
+ content: snapshot.body
3318
+ };
3319
+ }
3320
+ const item = {
3321
+ contentType: itemRow.content_type,
3322
+ enSlug: itemRow.en_slug,
3323
+ locale: itemRow.locale,
3324
+ reason: "missing",
3325
+ currentEnHash: itemRow.en_hash
3326
+ };
3327
+ return {
3328
+ item,
3329
+ type,
3330
+ enDoc,
3331
+ // Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
3332
+ payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
3333
+ currentEnHash: itemRow.en_hash,
3334
+ existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
3335
+ model: jobRow.display_model,
3336
+ prompt: "",
3337
+ responseSchema: void 0
3338
+ };
3339
+ }
3340
+ function ingestBatchJob(config, jobRow, batchJob, onResult) {
3341
+ const state = String(batchJob.state ?? "UNKNOWN");
3342
+ const db = openStore(config, "readwrite");
3343
+ if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
3344
+ db.close();
3345
+ return null;
3346
+ }
3347
+ const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
3348
+ const results = [];
3349
+ const failItem = (itemRow, message, startedAt) => {
3350
+ const result = {
3351
+ contentType: itemRow.content_type,
3352
+ enSlug: itemRow.en_slug,
3353
+ locale: itemRow.locale,
3354
+ skipped: false,
3355
+ failed: true,
3356
+ error: formatTranslateError(new Error(message)),
2892
3357
  durationMs: Date.now() - startedAt
2893
3358
  };
3359
+ updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
3360
+ results.push(result);
3361
+ onResult?.(result);
3362
+ };
3363
+ if (isSuccessfulBatchState(state)) {
3364
+ const responses = batchJob.dest?.inlinedResponses ?? [];
3365
+ for (const itemRow of pendingItems) {
3366
+ const startedAt = Date.now();
3367
+ const inlined = responses[itemRow.request_index];
3368
+ if (!inlined || inlined.error || !inlined.response) {
3369
+ failItem(
3370
+ itemRow,
3371
+ inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
3372
+ startedAt
3373
+ );
3374
+ continue;
3375
+ }
3376
+ let result;
3377
+ try {
3378
+ const response = inlined.response;
3379
+ const raw = textFromBatchResponse(response);
3380
+ const parsed = parseGeminiResponse(raw);
3381
+ if (!parsed.frontmatter || typeof parsed.body !== "string") {
3382
+ throw new Error("Gemini response missing frontmatter/body");
3383
+ }
3384
+ const prepared = buildIngestContext(config, db, jobRow, itemRow);
3385
+ result = finalizeTranslation(
3386
+ config,
3387
+ prepared,
3388
+ { model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
3389
+ { costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
3390
+ );
3391
+ } catch (error) {
3392
+ failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
3393
+ continue;
3394
+ }
3395
+ updateBatchItemStatus(
3396
+ db,
3397
+ jobRow.id,
3398
+ itemRow.request_index,
3399
+ result.failed ? "failed" : "done",
3400
+ result.error
3401
+ );
3402
+ results.push(result);
3403
+ onResult?.(result);
3404
+ }
3405
+ } else {
3406
+ const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
3407
+ const startedAt = Date.now();
3408
+ for (const itemRow of pendingItems) {
3409
+ failItem(itemRow, message, startedAt);
3410
+ }
3411
+ }
3412
+ db.close();
3413
+ return results;
3414
+ }
3415
+ async function pollAndIngestBatchJobs(config, jobs, options = {}) {
3416
+ const jobCount = options.jobCount ?? jobs.length;
3417
+ const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
3418
+ const loopStartedAt = Date.now();
3419
+ let delay = INITIAL_POLL_MS;
3420
+ let firstRound = true;
3421
+ const elapsedFor = (row) => {
3422
+ const createdAtMs = Date.parse(row.created_at);
3423
+ return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
3424
+ };
3425
+ while (active.length > 0) {
3426
+ if (!firstRound) {
3427
+ await sleep(delay);
3428
+ delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
3429
+ }
3430
+ firstRound = false;
3431
+ for (const tracked of [...active]) {
3432
+ const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
3433
+ const state = normalizeBatchState(batchJob.state);
3434
+ options.onProgress?.({
3435
+ type: "batch-polling",
3436
+ name: tracked.row.job_name,
3437
+ state,
3438
+ elapsedMs: elapsedFor(tracked.row),
3439
+ jobIndex: tracked.jobIndex,
3440
+ jobCount
3441
+ });
3442
+ if (isTerminalBatchState(batchJob.state)) {
3443
+ active.splice(active.indexOf(tracked), 1);
3444
+ const results = ingestBatchJob(
3445
+ config,
3446
+ tracked.row,
3447
+ batchJob,
3448
+ options.onResult
3449
+ );
3450
+ if (results === null) {
3451
+ options.onProgress?.({
3452
+ type: "batch-done",
3453
+ name: tracked.row.job_name,
3454
+ state,
3455
+ model: tracked.row.display_model,
3456
+ count: 0,
3457
+ jobIndex: tracked.jobIndex,
3458
+ jobCount,
3459
+ translated: 0,
3460
+ failed: 0,
3461
+ inputTokens: 0,
3462
+ outputTokens: 0,
3463
+ estimatedCostUsd: 0,
3464
+ elapsedMs: elapsedFor(tracked.row),
3465
+ alreadyIngested: true
3466
+ });
3467
+ continue;
3468
+ }
3469
+ options.onProgress?.({
3470
+ type: "batch-done",
3471
+ name: tracked.row.job_name,
3472
+ state,
3473
+ model: tracked.row.display_model,
3474
+ count: results.length,
3475
+ jobIndex: tracked.jobIndex,
3476
+ jobCount,
3477
+ translated: results.filter((r) => !r.failed && !r.skipped).length,
3478
+ failed: results.filter((r) => r.failed).length,
3479
+ inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
3480
+ outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
3481
+ estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
3482
+ elapsedMs: elapsedFor(tracked.row)
3483
+ });
3484
+ }
3485
+ }
2894
3486
  }
2895
3487
  }
3488
+ function failedResultForItem(item, error, startedAt) {
3489
+ return {
3490
+ ...baseForItem(item),
3491
+ skipped: false,
3492
+ failed: true,
3493
+ error: formatTranslateError(error),
3494
+ durationMs: Date.now() - startedAt
3495
+ };
3496
+ }
3497
+
3498
+ // src/translate/page-translator.ts
3499
+ async function runPool(items, concurrency, worker) {
3500
+ if (items.length === 0) return;
3501
+ let nextIndex = 0;
3502
+ const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
3503
+ while (nextIndex < items.length) {
3504
+ const index = nextIndex++;
3505
+ await worker(items[index], index);
3506
+ }
3507
+ });
3508
+ await Promise.all(runners);
3509
+ }
2896
3510
  function labelForItem(item) {
2897
3511
  return `${item.contentType}/${item.enSlug}@${item.locale}`;
2898
3512
  }
3513
+ async function translatePage(config, item, options = {}) {
3514
+ const startedAt = Date.now();
3515
+ let outcome;
3516
+ try {
3517
+ outcome = prepareTranslation(config, item, options, startedAt);
3518
+ } catch (error) {
3519
+ return failedResultForItem(item, error, startedAt);
3520
+ }
3521
+ if (outcome.status === "done") return outcome.result;
3522
+ const prepared = outcome.prepared;
3523
+ if (options.dryRun) {
3524
+ return {
3525
+ ...baseForItem(item),
3526
+ skipped: false,
3527
+ model: prepared.model,
3528
+ durationMs: Date.now() - startedAt
3529
+ };
3530
+ }
3531
+ try {
3532
+ const result = await translatePageWithGemini({
3533
+ prompt: prepared.prompt,
3534
+ model: prepared.model,
3535
+ responseSchema: prepared.responseSchema
3536
+ });
3537
+ return finalizeTranslation(
3538
+ config,
3539
+ prepared,
3540
+ { model: result.model, parsed: result.parsed, usage: result.usage },
3541
+ { costMode: "interactive", startedAt }
3542
+ );
3543
+ } catch (error) {
3544
+ return failedResultForItem(item, error, startedAt);
3545
+ }
3546
+ }
3547
+ function createResultCollector(items, onProgress) {
3548
+ const slotByKey = /* @__PURE__ */ new Map();
3549
+ items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
3550
+ const slots = new Array(items.length);
3551
+ const extras = [];
3552
+ return {
3553
+ emit(result) {
3554
+ const slot = slotByKey.get(translationItemKey(result));
3555
+ if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
3556
+ else extras.push(result);
3557
+ onProgress?.({ type: "item-done", result });
3558
+ },
3559
+ assemble() {
3560
+ return [
3561
+ ...slots.filter((result) => result !== void 0),
3562
+ ...extras
3563
+ ];
3564
+ }
3565
+ };
3566
+ }
2899
3567
  async function translateWorklist(config, items, options = {}) {
3568
+ const mode = options.mode ?? "batch";
2900
3569
  const concurrency = Math.max(1, options.concurrency ?? 3);
2901
3570
  const startedAt = Date.now();
2902
- const results = new Array(items.length);
2903
- const active = /* @__PURE__ */ new Set();
3571
+ if (options.dryRun) {
3572
+ options.onProgress?.({
3573
+ type: "start",
3574
+ total: items.length,
3575
+ concurrency,
3576
+ dryRun: true,
3577
+ model: options.model,
3578
+ mode
3579
+ });
3580
+ const results2 = items.map((item) => {
3581
+ const itemStartedAt = Date.now();
3582
+ try {
3583
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3584
+ return outcome.status === "done" ? outcome.result : {
3585
+ ...baseForItem(item),
3586
+ skipped: false,
3587
+ model: outcome.prepared.model,
3588
+ durationMs: Date.now() - itemStartedAt
3589
+ };
3590
+ } catch (error) {
3591
+ return failedResultForItem(item, error, itemStartedAt);
3592
+ }
3593
+ });
3594
+ for (const result of results2) options.onProgress?.({ type: "item-done", result });
3595
+ const totals2 = summarizeResults(results2, Date.now() - startedAt);
3596
+ options.onProgress?.({ type: "done", results: results2, totals: totals2 });
3597
+ return results2;
3598
+ }
3599
+ const pending = readPendingBatchWork(config);
3600
+ const inputKeys = new Set(items.map((item) => translationItemKey(item)));
3601
+ const resumedExtraCount = pending.pendingItems.filter(
3602
+ (item) => !inputKeys.has(translationItemKey(item))
3603
+ ).length;
2904
3604
  options.onProgress?.({
2905
3605
  type: "start",
2906
- total: items.length,
3606
+ total: items.length + resumedExtraCount,
2907
3607
  concurrency,
2908
- dryRun: Boolean(options.dryRun),
2909
- model: options.model
3608
+ dryRun: false,
3609
+ model: options.model,
3610
+ mode
2910
3611
  });
2911
- await runPool(items, concurrency, async (item, index) => {
2912
- const label = labelForItem(item);
2913
- active.add(label);
2914
- options.onProgress?.({ type: "item-start", item, active: [...active] });
2915
- const result = await translatePage(config, item, options);
2916
- results[index] = result;
2917
- active.delete(label);
2918
- options.onProgress?.({ type: "item-done", result });
3612
+ const collector = createResultCollector(items, options.onProgress);
3613
+ const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
3614
+ if (mode === "direct") {
3615
+ const pendingCounts = countPendingItems(config, pending.jobs);
3616
+ emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
3617
+ const active = /* @__PURE__ */ new Set();
3618
+ await Promise.all([
3619
+ runPool(workItems, concurrency, async (item) => {
3620
+ const label = labelForItem(item);
3621
+ active.add(label);
3622
+ options.onProgress?.({ type: "item-start", item, active: [...active] });
3623
+ const result = await translatePage(config, item, options);
3624
+ active.delete(label);
3625
+ collector.emit(result);
3626
+ }),
3627
+ pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
3628
+ onProgress: options.onProgress,
3629
+ onResult: collector.emit
3630
+ }) : Promise.resolve()
3631
+ ]);
3632
+ } else {
3633
+ const entries = [];
3634
+ for (const item of workItems) {
3635
+ const itemStartedAt = Date.now();
3636
+ try {
3637
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3638
+ if (outcome.status === "done") {
3639
+ collector.emit(outcome.result);
3640
+ } else {
3641
+ entries.push({
3642
+ apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
3643
+ prompt: outcome.prepared.prompt,
3644
+ prepared: outcome.prepared
3645
+ });
3646
+ }
3647
+ } catch (error) {
3648
+ collector.emit(failedResultForItem(item, error, itemStartedAt));
3649
+ }
3650
+ }
3651
+ const plans = planBatchJobs(entries);
3652
+ const jobCount = pending.jobs.length + plans.length;
3653
+ const pendingCounts = countPendingItems(config, pending.jobs);
3654
+ emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
3655
+ const submitted = await Promise.all(
3656
+ plans.map(async (plan, planIndex) => {
3657
+ try {
3658
+ return await submitBatchJobPlan(config, {
3659
+ plan,
3660
+ displayModel: normalizeGeminiDisplayName(plan.apiModel),
3661
+ jobIndex: pending.jobs.length + planIndex,
3662
+ jobCount,
3663
+ onProgress: options.onProgress
3664
+ });
3665
+ } catch (error) {
3666
+ for (const entry of plan.entries) {
3667
+ collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
3668
+ }
3669
+ return void 0;
3670
+ }
3671
+ })
3672
+ );
3673
+ const jobsToPoll = [
3674
+ ...pending.jobs,
3675
+ ...submitted.filter((row) => row !== void 0)
3676
+ ];
3677
+ if (jobsToPoll.length > 0) {
3678
+ await pollAndIngestBatchJobs(config, jobsToPoll, {
3679
+ jobCount,
3680
+ onProgress: options.onProgress,
3681
+ onResult: collector.emit
3682
+ });
3683
+ }
3684
+ }
3685
+ const results = collector.assemble();
3686
+ const totals = summarizeResults(results, Date.now() - startedAt);
3687
+ options.onProgress?.({ type: "done", results, totals });
3688
+ return results;
3689
+ }
3690
+ function countPendingItems(config, jobs) {
3691
+ if (jobs.length === 0) return /* @__PURE__ */ new Map();
3692
+ const db = openStore(config, "readonly");
3693
+ const counts = /* @__PURE__ */ new Map();
3694
+ for (const job of jobs) {
3695
+ counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
3696
+ }
3697
+ db.close();
3698
+ return counts;
3699
+ }
3700
+ function emitResumedJobs(jobs, counts, jobCount, onProgress) {
3701
+ jobs.forEach((job, jobIndex) => {
3702
+ onProgress?.({
3703
+ type: "batch-submitted",
3704
+ name: job.job_name,
3705
+ count: counts.get(job.id) ?? 0,
3706
+ model: job.display_model,
3707
+ jobIndex,
3708
+ jobCount,
3709
+ resumed: true,
3710
+ createdAt: job.created_at
3711
+ });
3712
+ });
3713
+ }
3714
+ async function resumeTranslationJobs(config, options = {}) {
3715
+ const startedAt = Date.now();
3716
+ const pending = readPendingBatchWork(config);
3717
+ if (pending.jobs.length === 0) return null;
3718
+ options.onProgress?.({
3719
+ type: "start",
3720
+ total: pending.pendingItems.length,
3721
+ concurrency: 1,
3722
+ dryRun: false,
3723
+ mode: "batch"
3724
+ });
3725
+ const counts = countPendingItems(config, pending.jobs);
3726
+ emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
3727
+ const results = [];
3728
+ await pollAndIngestBatchJobs(config, pending.jobs, {
3729
+ onProgress: options.onProgress,
3730
+ onResult: (result) => {
3731
+ results.push(result);
3732
+ options.onProgress?.({ type: "item-done", result });
3733
+ }
2919
3734
  });
2920
3735
  const totals = summarizeResults(results, Date.now() - startedAt);
2921
3736
  options.onProgress?.({ type: "done", results, totals });
@@ -3038,6 +3853,7 @@ exports.isRoutableType = isRoutableType;
3038
3853
  exports.loadConfigSync = loadConfigSync;
3039
3854
  exports.resolveConfig = resolveConfig;
3040
3855
  exports.resolveLocalesFromPreset = resolveLocalesFromPreset;
3856
+ exports.resumeTranslationJobs = resumeTranslationJobs;
3041
3857
  exports.serializeMdx = serializeMdx;
3042
3858
  exports.translatePage = translatePage;
3043
3859
  exports.translateWorklist = translateWorklist;