scribe-cms 0.0.11 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +21 -6
  2. package/dist/cli/index.cjs +1258 -219
  3. package/dist/cli/index.cjs.map +1 -1
  4. package/dist/cli/index.js +1259 -220
  5. package/dist/cli/index.js.map +1 -1
  6. package/dist/cli/prompt-translate.d.ts +3 -0
  7. package/dist/cli/prompt-translate.d.ts.map +1 -1
  8. package/dist/cli/translate-progress.d.ts.map +1 -1
  9. package/dist/index.cjs +1020 -181
  10. package/dist/index.cjs.map +1 -1
  11. package/dist/index.js +1021 -183
  12. package/dist/index.js.map +1 -1
  13. package/dist/runtime.cjs +149 -62
  14. package/dist/runtime.cjs.map +1 -1
  15. package/dist/runtime.js +149 -62
  16. package/dist/runtime.js.map +1 -1
  17. package/dist/src/config/resolve-config.d.ts.map +1 -1
  18. package/dist/src/core/introspect-schema.d.ts +7 -1
  19. package/dist/src/core/introspect-schema.d.ts.map +1 -1
  20. package/dist/src/core/types.d.ts +6 -0
  21. package/dist/src/core/types.d.ts.map +1 -1
  22. package/dist/src/create-project.d.ts.map +1 -1
  23. package/dist/src/i18n/resolve-document.d.ts +1 -1
  24. package/dist/src/i18n/resolve-document.d.ts.map +1 -1
  25. package/dist/src/index.d.ts +3 -3
  26. package/dist/src/index.d.ts.map +1 -1
  27. package/dist/src/storage/batch-jobs.d.ts +53 -0
  28. package/dist/src/storage/batch-jobs.d.ts.map +1 -0
  29. package/dist/src/storage/sqlite.d.ts.map +1 -1
  30. package/dist/src/translate/batch-worklist.d.ts +87 -0
  31. package/dist/src/translate/batch-worklist.d.ts.map +1 -0
  32. package/dist/src/translate/gemini-batch.d.ts +26 -0
  33. package/dist/src/translate/gemini-batch.d.ts.map +1 -0
  34. package/dist/src/translate/gemini-client.d.ts +23 -0
  35. package/dist/src/translate/gemini-client.d.ts.map +1 -1
  36. package/dist/src/translate/gemini-models.d.ts +9 -0
  37. package/dist/src/translate/gemini-models.d.ts.map +1 -1
  38. package/dist/src/translate/gemini-pricing.d.ts +2 -1
  39. package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
  40. package/dist/src/translate/page-translator.d.ts +25 -52
  41. package/dist/src/translate/page-translator.d.ts.map +1 -1
  42. package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
  43. package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
  44. package/dist/src/translate/response-schema.d.ts +8 -1
  45. package/dist/src/translate/response-schema.d.ts.map +1 -1
  46. package/dist/src/translate/retry.d.ts +16 -0
  47. package/dist/src/translate/retry.d.ts.map +1 -0
  48. package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
  49. package/dist/src/translate/translate-core.d.ts +155 -0
  50. package/dist/src/translate/translate-core.d.ts.map +1 -0
  51. package/dist/src/validate/validate-project.d.ts.map +1 -1
  52. package/dist/studio/server.cjs +84 -47
  53. package/dist/studio/server.cjs.map +1 -1
  54. package/dist/studio/server.js +84 -47
  55. package/dist/studio/server.js.map +1 -1
  56. package/package.json +1 -1
@@ -76,6 +76,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
76
76
  }
77
77
  }
78
78
  function readSchemaVersion(db) {
79
+ const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
80
+ if (!metaTable) return 0;
79
81
  const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
80
82
  return row ? Number.parseInt(row.value, 10) || 0 : 0;
81
83
  }
@@ -100,7 +102,7 @@ var SCHEMA_VERSION, MIGRATIONS;
100
102
  var init_sqlite = __esm({
101
103
  "src/storage/sqlite.ts"() {
102
104
  init_cjs_shims();
103
- SCHEMA_VERSION = 4;
105
+ SCHEMA_VERSION = 5;
104
106
  MIGRATIONS = [
105
107
  `CREATE TABLE IF NOT EXISTS meta (
106
108
  key TEXT PRIMARY KEY,
@@ -148,7 +150,30 @@ var init_sqlite = __esm({
148
150
  UNIQUE (content_type, en_slug, en_hash)
149
151
  )`,
150
152
  `CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
151
- ON en_snapshots(content_type, en_slug, created_at DESC)`
153
+ ON en_snapshots(content_type, en_slug, created_at DESC)`,
154
+ `CREATE TABLE IF NOT EXISTS translation_batch_jobs (
155
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
156
+ job_name TEXT NOT NULL UNIQUE,
157
+ model TEXT NOT NULL,
158
+ display_model TEXT NOT NULL,
159
+ created_at TEXT NOT NULL,
160
+ state TEXT NOT NULL,
161
+ completed_at TEXT
162
+ )`,
163
+ `CREATE TABLE IF NOT EXISTS translation_batch_items (
164
+ job_id INTEGER NOT NULL,
165
+ request_index INTEGER NOT NULL,
166
+ content_type TEXT NOT NULL,
167
+ en_slug TEXT NOT NULL,
168
+ locale TEXT NOT NULL,
169
+ en_hash TEXT NOT NULL,
170
+ snapshot_id INTEGER NOT NULL,
171
+ status TEXT NOT NULL,
172
+ error TEXT,
173
+ PRIMARY KEY (job_id, request_index)
174
+ )`,
175
+ `CREATE INDEX IF NOT EXISTS idx_batch_items_status
176
+ ON translation_batch_items(status)`
152
177
  ];
153
178
  }
154
179
  });
@@ -190,26 +215,6 @@ function getRelationTarget(schema) {
190
215
  }
191
216
  return null;
192
217
  }
193
- function unwrapSchema(schema) {
194
- let current = schema;
195
- for (let i = 0; i < 8; i++) {
196
- const anySchema = current;
197
- if (typeof anySchema.unwrap === "function") {
198
- current = anySchema.unwrap();
199
- continue;
200
- }
201
- if (typeof anySchema.removeDefault === "function") {
202
- current = anySchema.removeDefault();
203
- continue;
204
- }
205
- if (anySchema._def?.innerType) {
206
- current = anySchema._def.innerType;
207
- continue;
208
- }
209
- break;
210
- }
211
- return current;
212
- }
213
218
  function peelOptionalWrappers(schema) {
214
219
  let current = schema;
215
220
  for (let i = 0; i < 8; i++) {
@@ -355,6 +360,39 @@ function resolveConfig(input, baseDir) {
355
360
  if (localeRouting.strategy === "search-param" && !localeRouting.param) {
356
361
  throw new Error('scribe config: localeRouting search-param requires a "param" name');
357
362
  }
363
+ const localeFallbacks = {};
364
+ for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
365
+ if (!raw.locales.includes(locale)) {
366
+ throw new Error(
367
+ `scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
368
+ );
369
+ }
370
+ const seen = /* @__PURE__ */ new Set();
371
+ for (const fallback of chain) {
372
+ if (!raw.locales.includes(fallback)) {
373
+ throw new Error(
374
+ `scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
375
+ );
376
+ }
377
+ if (fallback === locale) {
378
+ throw new Error(
379
+ `scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
380
+ );
381
+ }
382
+ if (fallback === defaultLocale) {
383
+ throw new Error(
384
+ `scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
385
+ );
386
+ }
387
+ if (seen.has(fallback)) {
388
+ throw new Error(
389
+ `scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
390
+ );
391
+ }
392
+ seen.add(fallback);
393
+ }
394
+ localeFallbacks[locale] = [...chain];
395
+ }
358
396
  const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
359
397
  const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
360
398
  const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
@@ -384,6 +422,7 @@ function resolveConfig(input, baseDir) {
384
422
  defaultLocale,
385
423
  localeRouting,
386
424
  localePresets: raw.localePresets,
425
+ localeFallbacks,
387
426
  translate: raw.translate,
388
427
  types
389
428
  };
@@ -399,6 +438,12 @@ init_cjs_shims();
399
438
 
400
439
  // src/core/introspect-schema.ts
401
440
  init_cjs_shims();
441
+ function getArrayElement(schema) {
442
+ if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
443
+ return schema.element;
444
+ }
445
+ return null;
446
+ }
402
447
  function introspectSchema(schema, prefix = []) {
403
448
  const relation = getRelationTarget(schema);
404
449
  if (relation && prefix.length > 0) {
@@ -412,7 +457,10 @@ function introspectSchema(schema, prefix = []) {
412
457
  }
413
458
  ];
414
459
  }
415
- const base = unwrapSchema(schema);
460
+ if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
461
+ return [{ path: prefix, kind: "translatable" }];
462
+ }
463
+ const base = peelOptionalWrappers(schema);
416
464
  if (base instanceof Object && "shape" in base) {
417
465
  const shape = base.shape;
418
466
  const fields = [];
@@ -421,9 +469,9 @@ function introspectSchema(schema, prefix = []) {
421
469
  }
422
470
  return fields;
423
471
  }
424
- if (base instanceof Object && "element" in base) {
425
- const element = base.element;
426
- const elementBase = unwrapSchema(element);
472
+ const element = getArrayElement(base);
473
+ if (element) {
474
+ const elementBase = peelOptionalWrappers(element);
427
475
  if (elementBase instanceof Object && "shape" in elementBase) {
428
476
  const shape = elementBase.shape;
429
477
  const fields = [];
@@ -435,15 +483,48 @@ function introspectSchema(schema, prefix = []) {
435
483
  }
436
484
  return [{ path: prefix, kind: getFieldKind(schema) }];
437
485
  }
438
- function extractByPaths(data, paths) {
439
- const out = {};
486
+ function buildPathTrie(paths) {
487
+ const root = { leaf: false, children: /* @__PURE__ */ new Map() };
440
488
  for (const path15 of paths) {
441
- const value = getAtPath(data, path15);
489
+ let node = root;
490
+ for (const segment of path15) {
491
+ let child = node.children.get(segment);
492
+ if (!child) {
493
+ child = { leaf: false, children: /* @__PURE__ */ new Map() };
494
+ node.children.set(segment, child);
495
+ }
496
+ node = child;
497
+ }
498
+ node.leaf = true;
499
+ }
500
+ return root;
501
+ }
502
+ function extractNode(data, trie) {
503
+ if (trie.leaf) return data;
504
+ const star = trie.children.get("*");
505
+ if (star) {
506
+ if (!Array.isArray(data)) return void 0;
507
+ return data.map(
508
+ (item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
509
+ );
510
+ }
511
+ if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
512
+ const record = data;
513
+ const out = {};
514
+ let any = false;
515
+ for (const [key, child] of trie.children) {
516
+ const value = extractNode(record[key], child);
442
517
  if (value !== void 0) {
443
- setAtPath(out, path15.filter((p) => p !== "*"), value);
518
+ out[key] = value;
519
+ any = true;
444
520
  }
445
521
  }
446
- return out;
522
+ return any ? out : void 0;
523
+ }
524
+ function extractByPaths(data, paths) {
525
+ if (paths.length === 0) return {};
526
+ const extracted = extractNode(data, buildPathTrie(paths));
527
+ return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
447
528
  }
448
529
  function pickTranslatable(data, schema) {
449
530
  const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
@@ -458,32 +539,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
458
539
  const translatable = pickTranslatable(localeData, schema);
459
540
  return deepMerge(structural, translatable);
460
541
  }
461
- function getAtPath(obj, path15) {
462
- let current = obj;
463
- for (const segment of path15) {
464
- if (segment === "*") {
465
- if (!Array.isArray(current)) return void 0;
466
- return current.map(
467
- (item) => typeof item === "object" && item !== null ? getAtPath(item, path15.slice(path15.indexOf("*") + 1)) : void 0
468
- );
469
- }
470
- if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
471
- current = current[segment];
472
- }
473
- return current;
474
- }
475
- function setAtPath(obj, path15, value) {
476
- if (path15.length === 0) return;
477
- if (path15.length === 1) {
478
- obj[path15[0]] = value;
479
- return;
480
- }
481
- const [head, ...rest] = path15;
482
- if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
483
- obj[head] = {};
484
- }
485
- setAtPath(obj[head], rest, value);
486
- }
487
542
  function deepMerge(base, overlay) {
488
543
  const out = { ...base };
489
544
  for (const [key, value] of Object.entries(overlay)) {
@@ -774,7 +829,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
774
829
  if (i >= body.length || body[i] === ">" || body[i] === "/") break;
775
830
  const attrStart = i;
776
831
  while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
777
- body.slice(attrStart, i);
832
+ const attrName = body.slice(attrStart, i);
833
+ if (!attrName) break;
778
834
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
779
835
  if (body[i] !== "=") {
780
836
  tagOut += body.slice(attrStart, i);
@@ -783,10 +839,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
783
839
  tagOut += body.slice(attrStart, i);
784
840
  tagOut += "=";
785
841
  i += 1;
842
+ const wsStart = i;
786
843
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
844
+ tagOut += body.slice(wsStart, i);
787
845
  const quote = body[i];
788
846
  if (quote !== '"' && quote !== "'") {
789
- continue;
847
+ break;
790
848
  }
791
849
  if (quote === "'") {
792
850
  const valStart2 = i + 1;
@@ -1108,7 +1166,7 @@ function getTranslatablePayload(doc, type) {
1108
1166
 
1109
1167
  // src/i18n/resolve-document.ts
1110
1168
  init_cjs_shims();
1111
- function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
1169
+ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
1112
1170
  const urlBuilder = createUrlBuilder({
1113
1171
  locales: [defaultLocale, locale],
1114
1172
  defaultLocale,
@@ -1122,15 +1180,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1122
1180
  for (const [docLocale, docIdx] of allDocs) {
1123
1181
  const found = docIdx.bySlug.get(slug);
1124
1182
  if (!found) continue;
1125
- const correctSlug = getSlugForLocale(found, docLocale, locale, allDocs, defaultLocale);
1126
- if (correctSlug && correctSlug !== slug) {
1183
+ const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
1184
+ let target;
1185
+ let targetLocale = locale;
1186
+ for (const candidateLocale of [locale, ...fallbackLocales]) {
1187
+ const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
1188
+ if (cand) {
1189
+ target = cand;
1190
+ targetLocale = candidateLocale;
1191
+ break;
1192
+ }
1193
+ }
1194
+ if (target) {
1195
+ if (target.slug === slug) {
1196
+ return { document: target, actualLocale: targetLocale };
1197
+ }
1127
1198
  if (!type.path) {
1128
1199
  return { document: null, actualLocale: locale };
1129
1200
  }
1130
1201
  return {
1131
1202
  document: null,
1132
1203
  actualLocale: locale,
1133
- shouldRedirectTo: urlBuilder.resolvePath(type.path, correctSlug, locale)
1204
+ shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
1134
1205
  };
1135
1206
  }
1136
1207
  if (docLocale === defaultLocale) {
@@ -1161,13 +1232,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1161
1232
  }
1162
1233
  return { document: null, actualLocale: locale };
1163
1234
  }
1164
- function getSlugForLocale(document2, sourceLocale, targetLocale, allDocs, defaultLocale) {
1165
- if (sourceLocale === targetLocale) return document2.slug;
1166
- const englishSlug = sourceLocale === defaultLocale ? document2.slug : document2.enSlug;
1167
- if (!englishSlug) return null;
1168
- if (targetLocale === defaultLocale) return englishSlug;
1169
- return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
1170
- }
1171
1235
 
1172
1236
  // src/create-project.ts
1173
1237
  function comparatorFor(orderBy) {
@@ -1220,7 +1284,8 @@ function buildRuntime(config, type, getRuntime) {
1220
1284
  config.defaultLocale,
1221
1285
  load(),
1222
1286
  type,
1223
- config.localeRouting
1287
+ config.localeRouting,
1288
+ config.localeFallbacks[locale] ?? []
1224
1289
  );
1225
1290
  if (result.document && type.path) {
1226
1291
  return {
@@ -1242,8 +1307,14 @@ function buildRuntime(config, type, getRuntime) {
1242
1307
  const params = [];
1243
1308
  for (const locale of options.locales ?? config.locales) {
1244
1309
  const localeIdx = all.get(locale);
1310
+ const fallbacks = config.localeFallbacks[locale] ?? [];
1245
1311
  for (const doc of enIdx.bySlug.values()) {
1246
- const slug = locale === config.defaultLocale ? doc.slug : localeIdx?.byEnSlug.get(doc.slug)?.slug ?? doc.slug;
1312
+ let slug;
1313
+ if (locale === config.defaultLocale) {
1314
+ slug = doc.slug;
1315
+ } else {
1316
+ slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
1317
+ }
1247
1318
  params.push({ locale, slug });
1248
1319
  }
1249
1320
  }
@@ -1705,7 +1776,7 @@ init_sqlite();
1705
1776
 
1706
1777
  // src/validate/validate-relations.ts
1707
1778
  init_cjs_shims();
1708
- function getAtPath2(obj, fieldPath) {
1779
+ function getAtPath(obj, fieldPath) {
1709
1780
  let current = obj;
1710
1781
  for (const segment of fieldPath) {
1711
1782
  if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
@@ -1750,7 +1821,7 @@ function validateRelations(project) {
1750
1821
  });
1751
1822
  continue;
1752
1823
  }
1753
- const value = getAtPath2(doc.frontmatter, fieldMeta.path);
1824
+ const value = getAtPath(doc.frontmatter, fieldMeta.path);
1754
1825
  if (value === void 0 || value === null || value === "") continue;
1755
1826
  const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
1756
1827
  const fieldLabel = fieldMeta.path.join(".");
@@ -1906,6 +1977,20 @@ function listEnSlugs3(rootDir, contentDir) {
1906
1977
  if (!fs2__default.default.existsSync(dir)) return [];
1907
1978
  return fs2__default.default.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
1908
1979
  }
1980
+ function validateDocumentMdxBody(issues, input) {
1981
+ const preparedBody = prepareTranslatedMdxBody(input.body).body;
1982
+ const mdxValidation = validateMdxBody(preparedBody);
1983
+ if (!mdxValidation.ok) {
1984
+ issues.push({
1985
+ level: "error",
1986
+ contentType: input.contentType,
1987
+ enSlug: input.enSlug,
1988
+ locale: input.locale,
1989
+ field: "body",
1990
+ message: `Invalid MDX: ${mdxValidation.error}`
1991
+ });
1992
+ }
1993
+ }
1909
1994
  function validateProject(config) {
1910
1995
  const issues = [];
1911
1996
  const project = createProject(config);
@@ -1964,6 +2049,12 @@ function validateProject(config) {
1964
2049
  })) {
1965
2050
  issues.push(issue);
1966
2051
  }
2052
+ validateDocumentMdxBody(issues, {
2053
+ contentType: type.id,
2054
+ enSlug,
2055
+ locale: config.defaultLocale,
2056
+ body: enDoc.content
2057
+ });
1967
2058
  for (const locale of config.locales) {
1968
2059
  if (locale === config.defaultLocale) continue;
1969
2060
  const row = getTranslation(db, type.id, enSlug, locale);
@@ -1989,17 +2080,12 @@ function validateProject(config) {
1989
2080
  })) {
1990
2081
  issues.push(issue);
1991
2082
  }
1992
- const mdxValidation = validateMdxBody(preparedBody);
1993
- if (!mdxValidation.ok) {
1994
- issues.push({
1995
- level: "error",
1996
- contentType: type.id,
1997
- enSlug,
1998
- locale,
1999
- field: "body",
2000
- message: `Invalid MDX: ${mdxValidation.error}`
2001
- });
2002
- }
2083
+ validateDocumentMdxBody(issues, {
2084
+ contentType: type.id,
2085
+ enSlug,
2086
+ locale,
2087
+ body: row.body
2088
+ });
2003
2089
  }
2004
2090
  }
2005
2091
  try {
@@ -2119,6 +2205,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
2119
2205
  // src/translate/page-translator.ts
2120
2206
  init_cjs_shims();
2121
2207
 
2208
+ // src/storage/batch-jobs.ts
2209
+ init_cjs_shims();
2210
+ function insertBatchJob(db, input) {
2211
+ const info = db.prepare(
2212
+ `INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
2213
+ VALUES (?, ?, ?, ?, ?)`
2214
+ ).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
2215
+ return Number(info.lastInsertRowid);
2216
+ }
2217
+ function insertBatchItems(db, jobId, items) {
2218
+ const stmt = db.prepare(
2219
+ `INSERT INTO translation_batch_items (
2220
+ job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
2221
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
2222
+ );
2223
+ for (const item of items) {
2224
+ stmt.run(
2225
+ jobId,
2226
+ item.requestIndex,
2227
+ item.contentType,
2228
+ item.enSlug,
2229
+ item.locale,
2230
+ item.enHash,
2231
+ item.snapshotId
2232
+ );
2233
+ }
2234
+ }
2235
+ function listPendingBatchJobs(db) {
2236
+ return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
2237
+ }
2238
+ function listBatchItems(db, jobId) {
2239
+ return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
2240
+ }
2241
+ function listPendingBatchItems(db) {
2242
+ return db.prepare(
2243
+ `SELECT i.* FROM translation_batch_items i
2244
+ JOIN translation_batch_jobs j ON j.id = i.job_id
2245
+ WHERE j.completed_at IS NULL AND i.status = 'pending'
2246
+ ORDER BY i.job_id, i.request_index`
2247
+ ).all();
2248
+ }
2249
+ function claimBatchJobCompletion(db, jobId, state, completedAt) {
2250
+ const info = db.prepare(
2251
+ `UPDATE translation_batch_jobs SET state = ?, completed_at = ?
2252
+ WHERE id = ? AND completed_at IS NULL`
2253
+ ).run(state, completedAt, jobId);
2254
+ return info.changes > 0;
2255
+ }
2256
+ function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
2257
+ db.prepare(
2258
+ `UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
2259
+ ).run(status, error ?? null, jobId, requestIndex);
2260
+ }
2261
+
2262
+ // src/translate/page-translator.ts
2263
+ init_sqlite();
2264
+
2265
+ // src/translate/batch-worklist.ts
2266
+ init_cjs_shims();
2267
+
2122
2268
  // src/history/record-snapshot.ts
2123
2269
  init_cjs_shims();
2124
2270
  init_sqlite();
@@ -2136,9 +2282,111 @@ function recordEnSnapshot(config, input, db) {
2136
2282
  return id;
2137
2283
  }
2138
2284
 
2139
- // src/translate/page-translator.ts
2285
+ // src/translate/batch-worklist.ts
2140
2286
  init_sqlite();
2141
2287
 
2288
+ // src/translate/gemini-batch.ts
2289
+ init_cjs_shims();
2290
+
2291
+ // src/translate/retry.ts
2292
+ init_cjs_shims();
2293
+ var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
2294
+ function statusFromError(error) {
2295
+ if (error instanceof genai.ApiError && typeof error.status === "number") {
2296
+ return error.status;
2297
+ }
2298
+ if (error && typeof error === "object" && "status" in error) {
2299
+ const status = error.status;
2300
+ if (typeof status === "number") return status;
2301
+ }
2302
+ const message = error instanceof Error ? error.message : String(error);
2303
+ const match = message.match(/\b(429|500|502|503|504)\b/);
2304
+ if (match) return Number(match[1]);
2305
+ return void 0;
2306
+ }
2307
+ function isNetworkError(error) {
2308
+ const err = error;
2309
+ const code = typeof err?.code === "string" ? err.code : "";
2310
+ if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
2311
+ return true;
2312
+ }
2313
+ const message = error instanceof Error ? error.message.toLowerCase() : "";
2314
+ return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
2315
+ }
2316
+ function isTransientError(error) {
2317
+ const status = statusFromError(error);
2318
+ if (status !== void 0) return TRANSIENT_STATUS.has(status);
2319
+ return isNetworkError(error);
2320
+ }
2321
+ var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
2322
+ async function withRetry(fn, options = {}) {
2323
+ const attempts = Math.max(1, options.attempts ?? 3);
2324
+ const baseDelayMs = options.baseDelayMs ?? 1e3;
2325
+ const sleep2 = options.sleep ?? defaultSleep;
2326
+ const random = options.random ?? Math.random;
2327
+ let lastError;
2328
+ for (let attempt = 1; attempt <= attempts; attempt++) {
2329
+ try {
2330
+ return await fn();
2331
+ } catch (error) {
2332
+ lastError = error;
2333
+ if (attempt >= attempts || !isTransientError(error)) throw error;
2334
+ const backoff = baseDelayMs * 2 ** (attempt - 1);
2335
+ const delay = backoff + random() * backoff;
2336
+ await sleep2(delay);
2337
+ }
2338
+ }
2339
+ throw lastError;
2340
+ }
2341
+
2342
+ // src/translate/gemini-batch.ts
2343
+ var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
2344
+ var TERMINAL_STATES = /* @__PURE__ */ new Set([
2345
+ "SUCCEEDED",
2346
+ "FAILED",
2347
+ "CANCELLED",
2348
+ "EXPIRED",
2349
+ "PARTIALLY_SUCCEEDED"
2350
+ ]);
2351
+ var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
2352
+ function normalizeBatchState(state) {
2353
+ return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
2354
+ }
2355
+ function isTerminalBatchState(state) {
2356
+ return TERMINAL_STATES.has(normalizeBatchState(state));
2357
+ }
2358
+ function isSuccessfulBatchState(state) {
2359
+ return SUCCESSFUL_STATES.has(normalizeBatchState(state));
2360
+ }
2361
+ function makeClient(apiKey) {
2362
+ const key = apiKey ?? process.env.GEMINI_API_KEY;
2363
+ if (!key) {
2364
+ throw new Error("GEMINI_API_KEY is required for scribe translate");
2365
+ }
2366
+ return new genai.GoogleGenAI({ apiKey: key });
2367
+ }
2368
+ function textFromBatchResponse(response) {
2369
+ if (typeof response.text === "string") return response.text;
2370
+ const parts = response.candidates?.[0]?.content?.parts ?? [];
2371
+ return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
2372
+ }
2373
+ async function createGeminiBatchJob(input) {
2374
+ const ai = makeClient(input.apiKey);
2375
+ const job = await withRetry(
2376
+ () => ai.batches.create({
2377
+ model: input.model,
2378
+ src: input.requests,
2379
+ config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
2380
+ })
2381
+ );
2382
+ if (!job.name) throw new Error("Gemini batch job was created without a name");
2383
+ return job;
2384
+ }
2385
+ async function getGeminiBatchJob(input) {
2386
+ const ai = makeClient(input.apiKey);
2387
+ return withRetry(() => ai.batches.get({ name: input.name }));
2388
+ }
2389
+
2142
2390
  // src/translate/gemini-client.ts
2143
2391
  init_cjs_shims();
2144
2392
 
@@ -2163,16 +2411,55 @@ function normalizeGeminiDisplayName(model) {
2163
2411
  if (alias) return alias[0];
2164
2412
  return name;
2165
2413
  }
2414
+ function resolveThinkingConfig(apiModelId) {
2415
+ const id = stripModelsPrefix(apiModelId).toLowerCase();
2416
+ if (id.startsWith("gemini-3")) return { thinkingLevel: genai.ThinkingLevel.LOW };
2417
+ if (id.includes("2.5")) return { thinkingBudget: 128 };
2418
+ return void 0;
2419
+ }
2166
2420
 
2167
2421
  // src/translate/gemini-client.ts
2168
2422
  var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
2169
2423
  function extractJson(text) {
2170
- const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
2424
+ const trimmed = text.trim();
2425
+ const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
2171
2426
  if (fenced?.[1]) return fenced[1].trim();
2172
- const start = text.indexOf("{");
2173
- const end = text.lastIndexOf("}");
2174
- if (start >= 0 && end > start) return text.slice(start, end + 1);
2175
- return text.trim();
2427
+ const start = trimmed.indexOf("{");
2428
+ const end = trimmed.lastIndexOf("}");
2429
+ if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
2430
+ return trimmed;
2431
+ }
2432
+ function parseGeminiResponse(text) {
2433
+ let parsed;
2434
+ try {
2435
+ parsed = JSON.parse(text.trim());
2436
+ } catch {
2437
+ parsed = JSON.parse(extractJson(text));
2438
+ }
2439
+ if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
2440
+ parsed.frontmatter = {};
2441
+ }
2442
+ return parsed;
2443
+ }
2444
+ function buildGeminiRequestConfig(input) {
2445
+ const thinkingConfig = resolveThinkingConfig(input.apiModelId);
2446
+ return {
2447
+ responseMimeType: "application/json",
2448
+ ...input.responseSchema ? { responseSchema: input.responseSchema } : {},
2449
+ ...thinkingConfig ? { thinkingConfig } : {}
2450
+ };
2451
+ }
2452
+ function usageFromResponse(response) {
2453
+ const meta = response.usageMetadata ?? response.usage_metadata ?? {};
2454
+ const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
2455
+ return {
2456
+ inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
2457
+ // candidatesTokenCount does not include thoughts, but thoughts are billed as
2458
+ // output tokens, so fold them in here for accurate cost accounting.
2459
+ outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
2460
+ thoughtsTokens,
2461
+ totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
2462
+ };
2176
2463
  }
2177
2464
  async function translatePageWithGemini(input) {
2178
2465
  const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
@@ -2184,28 +2471,28 @@ async function translatePageWithGemini(input) {
2184
2471
  );
2185
2472
  const apiModel = resolveGeminiModelId(displayModel);
2186
2473
  const ai = new genai.GoogleGenAI({ apiKey });
2187
- const response = await ai.models.generateContent({
2188
- model: apiModel,
2189
- contents: input.prompt,
2190
- config: {
2191
- responseMimeType: "application/json",
2192
- ...input.responseSchema ? { responseSchema: input.responseSchema } : {}
2193
- }
2194
- });
2474
+ const response = await withRetry(
2475
+ () => ai.models.generateContent({
2476
+ model: apiModel,
2477
+ contents: input.prompt,
2478
+ config: buildGeminiRequestConfig({
2479
+ apiModelId: apiModel,
2480
+ responseSchema: input.responseSchema
2481
+ })
2482
+ })
2483
+ );
2195
2484
  const raw = response.text ?? "";
2196
- const parsed = JSON.parse(extractJson(raw));
2485
+ const parsed = parseGeminiResponse(raw);
2197
2486
  if (!parsed.frontmatter || typeof parsed.body !== "string") {
2198
2487
  throw new Error("Gemini response missing frontmatter/body");
2199
2488
  }
2200
- const usageMetadata = response.usageMetadata;
2201
- const usage = {
2202
- inputTokens: usageMetadata?.promptTokenCount ?? 0,
2203
- outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
2204
- totalTokens: usageMetadata?.totalTokenCount ?? 0
2205
- };
2206
- return { model: displayModel, raw, parsed, usage };
2489
+ return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
2207
2490
  }
2208
2491
 
2492
+ // src/translate/translate-core.ts
2493
+ init_cjs_shims();
2494
+ init_sqlite();
2495
+
2209
2496
  // src/translate/gemini-pricing.ts
2210
2497
  init_cjs_shims();
2211
2498
  var CONTEXT_TIER_TOKENS = 2e5;
@@ -2224,12 +2511,14 @@ function resolveModelPricing(model) {
2224
2511
  const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
2225
2512
  return match?.[1];
2226
2513
  }
2227
- function estimateTranslationCostUsd(model, inputTokens, outputTokens) {
2514
+ var BATCH_DISCOUNT = 0.5;
2515
+ function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
2228
2516
  const pricing = resolveModelPricing(model);
2229
2517
  if (!pricing) return void 0;
2230
2518
  const inputRate = tierRate(inputTokens, pricing.input);
2231
2519
  const outputRate = tierRate(outputTokens, pricing.output);
2232
- return inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate;
2520
+ const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
2521
+ return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
2233
2522
  }
2234
2523
  function formatTokenCount(tokens) {
2235
2524
  if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
@@ -2257,6 +2546,7 @@ var LOCALE_NAMES = {
2257
2546
  "zh-CN": "Simplified Chinese",
2258
2547
  "zh-TW": "Traditional Chinese",
2259
2548
  pt: "Portuguese",
2549
+ "pt-BR": "Brazilian Portuguese",
2260
2550
  ja: "Japanese",
2261
2551
  ru: "Russian",
2262
2552
  it: "Italian",
@@ -2264,29 +2554,29 @@ var LOCALE_NAMES = {
2264
2554
  };
2265
2555
  function defaultLocalizationPrompt(localeName, locale) {
2266
2556
  return [
2267
- `Localize the content below into natural ${localeName} (${locale}).`,
2557
+ `Localize the content above into natural ${localeName} (${locale}).`,
2268
2558
  "Do not translate word-for-word.",
2269
2559
  "Preserve the source tone and brand voice.",
2270
2560
  "Write as if a native speaker authored it for the target market."
2271
2561
  ].join(" ");
2272
2562
  }
2563
+ var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
2564
+ function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
2565
+ if (!hasFrontmatter) {
2566
+ return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
2567
+ }
2568
+ return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
2569
+ }
2273
2570
  function buildPageTranslationPrompt(input) {
2274
2571
  const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
2275
- const prompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2276
- const lines = [
2277
- prompt,
2572
+ const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2573
+ const prefix = [
2574
+ TASK_FRAMING,
2278
2575
  "",
2279
2576
  ...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
2280
2577
  ...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
2281
2578
  "## Rules",
2282
2579
  ...input.resolved.rules.map((rule) => `- ${rule}`),
2283
- ...input.slugStrategy === "localized" ? [
2284
- `- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
2285
- ] : [],
2286
- "",
2287
- "## Output format",
2288
- "Return ONLY valid JSON with keys:",
2289
- input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
2290
2580
  "",
2291
2581
  "## EN translatable frontmatter (JSON)",
2292
2582
  JSON.stringify(input.translatableFrontmatter, null, 2),
@@ -2294,7 +2584,22 @@ function buildPageTranslationPrompt(input) {
2294
2584
  "## EN body (MDX)",
2295
2585
  input.enBody
2296
2586
  ];
2297
- return lines.join("\n");
2587
+ const suffix = [
2588
+ "",
2589
+ "## Target language",
2590
+ localizationPrompt,
2591
+ ...input.slugStrategy === "localized" ? [
2592
+ `The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
2593
+ ] : [],
2594
+ "",
2595
+ "## Output format",
2596
+ "Return ONLY valid JSON with keys:",
2597
+ buildOutputFormatLine(
2598
+ Object.keys(input.translatableFrontmatter).length > 0,
2599
+ input.slugStrategy
2600
+ )
2601
+ ];
2602
+ return [...prefix, ...suffix].join("\n");
2298
2603
  }
2299
2604
 
2300
2605
  // src/translate/response-schema.ts
@@ -2345,9 +2650,8 @@ function buildTranslatableSubschema(schema) {
2345
2650
  function buildGeminiResponseSchema(schema, slugStrategy) {
2346
2651
  try {
2347
2652
  const translatable = buildTranslatableSubschema(schema);
2348
- if (!translatable) return null;
2349
2653
  const responseShape = {
2350
- frontmatter: translatable,
2654
+ ...translatable ? { frontmatter: translatable } : {},
2351
2655
  body: zod.z.string()
2352
2656
  };
2353
2657
  if (slugStrategy === "localized") {
@@ -2428,7 +2732,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
2428
2732
  return { ok: true, frontmatter };
2429
2733
  }
2430
2734
 
2431
- // src/translate/page-translator.ts
2735
+ // src/translate/translate-core.ts
2736
+ function translationItemKey(item) {
2737
+ const contentType = item.contentType ?? item.content_type ?? "";
2738
+ const enSlug = item.enSlug ?? item.en_slug ?? "";
2739
+ return `${contentType}\0${enSlug}\0${item.locale}`;
2740
+ }
2432
2741
  function summarizeResults(results, durationMs) {
2433
2742
  return results.reduce(
2434
2743
  (totals, result) => {
@@ -2464,17 +2773,6 @@ function formatTranslateError(error) {
2464
2773
  }
2465
2774
  return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
2466
2775
  }
2467
- async function runPool(items, concurrency, worker) {
2468
- if (items.length === 0) return;
2469
- let nextIndex = 0;
2470
- const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
2471
- while (nextIndex < items.length) {
2472
- const index = nextIndex++;
2473
- await worker(items[index], index);
2474
- }
2475
- });
2476
- await Promise.all(runners);
2477
- }
2478
2776
  function resolveContextLabel(enDoc, enSlug) {
2479
2777
  const fm = enDoc.frontmatter;
2480
2778
  for (const key of ["title", "name", "h1"]) {
@@ -2483,73 +2781,82 @@ function resolveContextLabel(enDoc, enSlug) {
2483
2781
  }
2484
2782
  return enSlug;
2485
2783
  }
2486
- async function translatePage(config, item, options = {}) {
2487
- const startedAt = Date.now();
2488
- const base = {
2784
+ function baseForItem(item) {
2785
+ return {
2489
2786
  contentType: item.contentType,
2490
2787
  enSlug: item.enSlug,
2491
2788
  locale: item.locale
2492
2789
  };
2493
- try {
2494
- const type = config.types.find((t) => t.id === item.contentType);
2495
- if (!type) throw new Error(`Unknown content type ${item.contentType}`);
2496
- const enDoc = readEnDocument(config, type, item.enSlug);
2497
- if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
2498
- const payload = getTranslatablePayload(enDoc, type);
2499
- const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
2500
- const db = openStore(config, "readonly");
2501
- const existing = getTranslation(db, type.id, item.enSlug, item.locale);
2502
- db.close();
2503
- if (!options.force && existing && existing.en_hash === currentEnHash) {
2504
- return {
2505
- ...base,
2790
+ }
2791
+ function prepareTranslation(config, item, options, startedAt) {
2792
+ const type = config.types.find((t) => t.id === item.contentType);
2793
+ if (!type) throw new Error(`Unknown content type ${item.contentType}`);
2794
+ const enDoc = readEnDocument(config, type, item.enSlug);
2795
+ if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
2796
+ const payload = getTranslatablePayload(enDoc, type);
2797
+ const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
2798
+ const db = openStore(config, "readonly");
2799
+ const existing = getTranslation(db, type.id, item.enSlug, item.locale);
2800
+ db.close();
2801
+ if (!options.force && existing && existing.en_hash === currentEnHash) {
2802
+ return {
2803
+ status: "done",
2804
+ result: {
2805
+ ...baseForItem(item),
2506
2806
  skipped: true,
2507
2807
  reason: "fresh",
2508
2808
  durationMs: Date.now() - startedAt
2509
- };
2510
- }
2511
- const resolvedTranslate = resolveTranslateConfig(config, type);
2512
- const model = options.model ?? resolvedTranslate.model;
2513
- const prompt = buildPageTranslationPrompt({
2514
- resolved: resolvedTranslate,
2515
- targetLocale: item.locale,
2516
- contextLabel: resolveContextLabel(enDoc, item.enSlug),
2517
- translatableFrontmatter: payload.frontmatter,
2518
- enBody: payload.body,
2519
- slugStrategy: type.slugStrategy
2520
- });
2521
- if (options.dryRun) {
2522
- return {
2523
- ...base,
2524
- skipped: false,
2525
- model,
2526
- durationMs: Date.now() - startedAt
2527
- };
2528
- }
2529
- const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
2530
- const result = await translatePageWithGemini({
2531
- prompt,
2809
+ }
2810
+ };
2811
+ }
2812
+ const resolvedTranslate = resolveTranslateConfig(config, type);
2813
+ const model = options.model ?? resolvedTranslate.model;
2814
+ const prompt = buildPageTranslationPrompt({
2815
+ resolved: resolvedTranslate,
2816
+ targetLocale: item.locale,
2817
+ contextLabel: resolveContextLabel(enDoc, item.enSlug),
2818
+ translatableFrontmatter: payload.frontmatter,
2819
+ enBody: payload.body,
2820
+ slugStrategy: type.slugStrategy
2821
+ });
2822
+ const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
2823
+ return {
2824
+ status: "ready",
2825
+ prepared: {
2826
+ item,
2827
+ type,
2828
+ enDoc,
2829
+ payload,
2830
+ currentEnHash,
2831
+ existingSlug: existing?.slug,
2532
2832
  model,
2833
+ prompt,
2533
2834
  responseSchema: responseSchema ?? void 0
2534
- });
2535
- const rawSlug = type.slugStrategy === "localized" ? result.parsed.slug ?? existing?.slug ?? item.enSlug : item.enSlug;
2835
+ }
2836
+ };
2837
+ }
2838
+ function finalizeTranslation(config, prepared, output, options) {
2839
+ const { item, type, enDoc, payload } = prepared;
2840
+ const base = baseForItem(item);
2841
+ try {
2842
+ const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
2536
2843
  const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
2537
2844
  const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
2538
2845
  const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
2539
- const validated = validateTranslatedFrontmatter(enDoc, result.parsed.frontmatter, type.schema);
2846
+ const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
2540
2847
  if (!validated.ok) {
2541
2848
  throw new Error(`Translation validation failed: ${validated.error}`);
2542
2849
  }
2543
2850
  const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
2544
- result.parsed.body
2851
+ output.parsed.body
2545
2852
  );
2546
2853
  const writeDb = openStore(config, "readwrite");
2547
- const snapshotId = recordEnSnapshot(
2854
+ const snapshotId = options.snapshotId ?? recordEnSnapshot(
2548
2855
  config,
2549
2856
  {
2550
2857
  contentType: type.id,
2551
2858
  enSlug: item.enSlug,
2552
- enHash: currentEnHash,
2859
+ enHash: prepared.currentEnHash,
2553
2860
  frontmatter: payload.frontmatter,
2554
2861
  body: payload.body
2555
2862
  },
@@ -2562,24 +2869,25 @@ async function translatePage(config, item, options = {}) {
2562
2869
  slug,
2563
2870
  frontmatter: validated.frontmatter,
2564
2871
  body: translatedBody,
2565
- enHash: currentEnHash,
2872
+ enHash: prepared.currentEnHash,
2566
2873
  translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
2567
- model: result.model,
2874
+ model: output.model,
2568
2875
  snapshotId
2569
2876
  });
2570
2877
  writeDb.close();
2571
2878
  const estimatedCostUsd = estimateTranslationCostUsd(
2572
- normalizeGeminiDisplayName(result.model),
2573
- result.usage.inputTokens,
2574
- result.usage.outputTokens
2879
+ normalizeGeminiDisplayName(output.model),
2880
+ output.usage.inputTokens,
2881
+ output.usage.outputTokens,
2882
+ options.costMode
2575
2883
  );
2576
2884
  return {
2577
2885
  ...base,
2578
2886
  skipped: false,
2579
- model: result.model,
2580
- usage: result.usage,
2887
+ model: output.model,
2888
+ usage: output.usage,
2581
2889
  estimatedCostUsd,
2582
- durationMs: Date.now() - startedAt,
2890
+ durationMs: Date.now() - options.startedAt,
2583
2891
  slugAdjusted,
2584
2892
  mdxAdjusted: mdxAdjusted || void 0
2585
2893
  };
@@ -2589,33 +2897,559 @@ async function translatePage(config, item, options = {}) {
2589
2897
  skipped: false,
2590
2898
  failed: true,
2591
2899
  error: formatTranslateError(error),
2900
+ durationMs: Date.now() - options.startedAt
2901
+ };
2902
+ }
2903
+ }
2904
+ function displayModelFor(prepared) {
2905
+ return normalizeGeminiDisplayName(
2906
+ prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
2907
+ );
2908
+ }
2909
+
2910
+ // src/translate/batch-worklist.ts
2911
+ var MAX_REQUESTS_PER_JOB = 100;
2912
+ var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
2913
+ var INITIAL_POLL_MS = 5e3;
2914
+ var MAX_POLL_MS = 3e4;
2915
+ var POLL_BACKOFF_FACTOR = 1.5;
2916
+ function sleep(ms) {
2917
+ return new Promise((resolve) => setTimeout(resolve, ms));
2918
+ }
2919
+ function planBatchJobs(entries) {
2920
+ const byModel = /* @__PURE__ */ new Map();
2921
+ for (const entry of entries) {
2922
+ const group = byModel.get(entry.apiModel);
2923
+ if (group) group.push(entry);
2924
+ else byModel.set(entry.apiModel, [entry]);
2925
+ }
2926
+ const plans = [];
2927
+ for (const [apiModel, group] of byModel) {
2928
+ let current = [];
2929
+ let currentBytes = 0;
2930
+ for (const entry of group) {
2931
+ const size = Buffer.byteLength(entry.prompt, "utf8");
2932
+ if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
2933
+ plans.push({ apiModel, entries: current });
2934
+ current = [];
2935
+ currentBytes = 0;
2936
+ }
2937
+ current.push(entry);
2938
+ currentBytes += size;
2939
+ }
2940
+ if (current.length > 0) plans.push({ apiModel, entries: current });
2941
+ }
2942
+ return plans;
2943
+ }
2944
+ function readPendingBatchWork(config) {
2945
+ const db = openStore(config, "readwrite");
2946
+ const jobs = listPendingBatchJobs(db);
2947
+ const pendingItems = listPendingBatchItems(db);
2948
+ db.close();
2949
+ const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
2950
+ return { jobs, inFlightKeys, pendingItems };
2951
+ }
2952
+ async function submitBatchJobPlan(config, input) {
2953
+ const { plan, displayModel, jobIndex, jobCount } = input;
2954
+ const job = await createGeminiBatchJob({
2955
+ model: plan.apiModel,
2956
+ requests: plan.entries.map(({ prepared }) => ({
2957
+ contents: prepared.prompt,
2958
+ config: buildGeminiRequestConfig({
2959
+ apiModelId: plan.apiModel,
2960
+ responseSchema: prepared.responseSchema
2961
+ })
2962
+ })),
2963
+ displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
2964
+ });
2965
+ const createdAt = (/* @__PURE__ */ new Date()).toISOString();
2966
+ const db = openStore(config, "readwrite");
2967
+ const jobId = insertBatchJob(db, {
2968
+ jobName: job.name,
2969
+ model: plan.apiModel,
2970
+ displayModel,
2971
+ state: String(job.state ?? "JOB_STATE_PENDING"),
2972
+ createdAt
2973
+ });
2974
+ insertBatchItems(
2975
+ db,
2976
+ jobId,
2977
+ plan.entries.map(({ prepared }, requestIndex) => ({
2978
+ requestIndex,
2979
+ contentType: prepared.item.contentType,
2980
+ enSlug: prepared.item.enSlug,
2981
+ locale: prepared.item.locale,
2982
+ enHash: prepared.currentEnHash,
2983
+ // Snapshot the EN source now so ingestion after a resume does not depend
2984
+ // on the EN files still matching (or existing) on disk.
2985
+ snapshotId: recordEnSnapshot(
2986
+ config,
2987
+ {
2988
+ contentType: prepared.item.contentType,
2989
+ enSlug: prepared.item.enSlug,
2990
+ enHash: prepared.currentEnHash,
2991
+ frontmatter: prepared.payload.frontmatter,
2992
+ body: prepared.payload.body
2993
+ },
2994
+ db
2995
+ )
2996
+ }))
2997
+ );
2998
+ const row = {
2999
+ id: jobId,
3000
+ job_name: job.name,
3001
+ model: plan.apiModel,
3002
+ display_model: displayModel,
3003
+ created_at: createdAt,
3004
+ state: String(job.state ?? "JOB_STATE_PENDING"),
3005
+ completed_at: null
3006
+ };
3007
+ db.close();
3008
+ input.onProgress?.({
3009
+ type: "batch-submitted",
3010
+ name: job.name,
3011
+ count: plan.entries.length,
3012
+ model: displayModel,
3013
+ jobIndex,
3014
+ jobCount,
3015
+ createdAt
3016
+ });
3017
+ return row;
3018
+ }
3019
+ function buildIngestContext(config, db, jobRow, itemRow) {
3020
+ const type = config.types.find((t) => t.id === itemRow.content_type);
3021
+ if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
3022
+ let enDoc = readEnDocument(config, type, itemRow.en_slug);
3023
+ if (!enDoc) {
3024
+ const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
3025
+ if (!snapshot) {
3026
+ throw new Error(
3027
+ `EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
3028
+ );
3029
+ }
3030
+ enDoc = {
3031
+ slug: itemRow.en_slug,
3032
+ enSlug: itemRow.en_slug,
3033
+ locale: config.defaultLocale,
3034
+ noindex: false,
3035
+ frontmatter: JSON.parse(snapshot.frontmatter_json),
3036
+ content: snapshot.body
3037
+ };
3038
+ }
3039
+ const item = {
3040
+ contentType: itemRow.content_type,
3041
+ enSlug: itemRow.en_slug,
3042
+ locale: itemRow.locale,
3043
+ reason: "missing",
3044
+ currentEnHash: itemRow.en_hash
3045
+ };
3046
+ return {
3047
+ item,
3048
+ type,
3049
+ enDoc,
3050
+ // Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
3051
+ payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
3052
+ currentEnHash: itemRow.en_hash,
3053
+ existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
3054
+ model: jobRow.display_model,
3055
+ prompt: "",
3056
+ responseSchema: void 0
3057
+ };
3058
+ }
3059
+ function ingestBatchJob(config, jobRow, batchJob, onResult) {
3060
+ const state = String(batchJob.state ?? "UNKNOWN");
3061
+ const db = openStore(config, "readwrite");
3062
+ if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
3063
+ db.close();
3064
+ return null;
3065
+ }
3066
+ const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
3067
+ const results = [];
3068
+ const failItem = (itemRow, message, startedAt) => {
3069
+ const result = {
3070
+ contentType: itemRow.content_type,
3071
+ enSlug: itemRow.en_slug,
3072
+ locale: itemRow.locale,
3073
+ skipped: false,
3074
+ failed: true,
3075
+ error: formatTranslateError(new Error(message)),
2592
3076
  durationMs: Date.now() - startedAt
2593
3077
  };
3078
+ updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
3079
+ results.push(result);
3080
+ onResult?.(result);
3081
+ };
3082
+ if (isSuccessfulBatchState(state)) {
3083
+ const responses = batchJob.dest?.inlinedResponses ?? [];
3084
+ for (const itemRow of pendingItems) {
3085
+ const startedAt = Date.now();
3086
+ const inlined = responses[itemRow.request_index];
3087
+ if (!inlined || inlined.error || !inlined.response) {
3088
+ failItem(
3089
+ itemRow,
3090
+ inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
3091
+ startedAt
3092
+ );
3093
+ continue;
3094
+ }
3095
+ let result;
3096
+ try {
3097
+ const response = inlined.response;
3098
+ const raw = textFromBatchResponse(response);
3099
+ const parsed = parseGeminiResponse(raw);
3100
+ if (!parsed.frontmatter || typeof parsed.body !== "string") {
3101
+ throw new Error("Gemini response missing frontmatter/body");
3102
+ }
3103
+ const prepared = buildIngestContext(config, db, jobRow, itemRow);
3104
+ result = finalizeTranslation(
3105
+ config,
3106
+ prepared,
3107
+ { model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
3108
+ { costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
3109
+ );
3110
+ } catch (error) {
3111
+ failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
3112
+ continue;
3113
+ }
3114
+ updateBatchItemStatus(
3115
+ db,
3116
+ jobRow.id,
3117
+ itemRow.request_index,
3118
+ result.failed ? "failed" : "done",
3119
+ result.error
3120
+ );
3121
+ results.push(result);
3122
+ onResult?.(result);
3123
+ }
3124
+ } else {
3125
+ const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
3126
+ const startedAt = Date.now();
3127
+ for (const itemRow of pendingItems) {
3128
+ failItem(itemRow, message, startedAt);
3129
+ }
2594
3130
  }
3131
+ db.close();
3132
+ return results;
3133
+ }
3134
+ async function pollAndIngestBatchJobs(config, jobs, options = {}) {
3135
+ const jobCount = options.jobCount ?? jobs.length;
3136
+ const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
3137
+ const loopStartedAt = Date.now();
3138
+ let delay = INITIAL_POLL_MS;
3139
+ let firstRound = true;
3140
+ const elapsedFor = (row) => {
3141
+ const createdAtMs = Date.parse(row.created_at);
3142
+ return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
3143
+ };
3144
+ while (active.length > 0) {
3145
+ if (!firstRound) {
3146
+ await sleep(delay);
3147
+ delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
3148
+ }
3149
+ firstRound = false;
3150
+ for (const tracked of [...active]) {
3151
+ const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
3152
+ const state = normalizeBatchState(batchJob.state);
3153
+ options.onProgress?.({
3154
+ type: "batch-polling",
3155
+ name: tracked.row.job_name,
3156
+ state,
3157
+ elapsedMs: elapsedFor(tracked.row),
3158
+ jobIndex: tracked.jobIndex,
3159
+ jobCount
3160
+ });
3161
+ if (isTerminalBatchState(batchJob.state)) {
3162
+ active.splice(active.indexOf(tracked), 1);
3163
+ const results = ingestBatchJob(
3164
+ config,
3165
+ tracked.row,
3166
+ batchJob,
3167
+ options.onResult
3168
+ );
3169
+ if (results === null) {
3170
+ options.onProgress?.({
3171
+ type: "batch-done",
3172
+ name: tracked.row.job_name,
3173
+ state,
3174
+ model: tracked.row.display_model,
3175
+ count: 0,
3176
+ jobIndex: tracked.jobIndex,
3177
+ jobCount,
3178
+ translated: 0,
3179
+ failed: 0,
3180
+ inputTokens: 0,
3181
+ outputTokens: 0,
3182
+ estimatedCostUsd: 0,
3183
+ elapsedMs: elapsedFor(tracked.row),
3184
+ alreadyIngested: true
3185
+ });
3186
+ continue;
3187
+ }
3188
+ options.onProgress?.({
3189
+ type: "batch-done",
3190
+ name: tracked.row.job_name,
3191
+ state,
3192
+ model: tracked.row.display_model,
3193
+ count: results.length,
3194
+ jobIndex: tracked.jobIndex,
3195
+ jobCount,
3196
+ translated: results.filter((r) => !r.failed && !r.skipped).length,
3197
+ failed: results.filter((r) => r.failed).length,
3198
+ inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
3199
+ outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
3200
+ estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
3201
+ elapsedMs: elapsedFor(tracked.row)
3202
+ });
3203
+ }
3204
+ }
3205
+ }
3206
+ }
3207
+ function failedResultForItem(item, error, startedAt) {
3208
+ return {
3209
+ ...baseForItem(item),
3210
+ skipped: false,
3211
+ failed: true,
3212
+ error: formatTranslateError(error),
3213
+ durationMs: Date.now() - startedAt
3214
+ };
3215
+ }
3216
+
3217
+ // src/translate/page-translator.ts
3218
+ async function runPool(items, concurrency, worker) {
3219
+ if (items.length === 0) return;
3220
+ let nextIndex = 0;
3221
+ const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
3222
+ while (nextIndex < items.length) {
3223
+ const index = nextIndex++;
3224
+ await worker(items[index], index);
3225
+ }
3226
+ });
3227
+ await Promise.all(runners);
2595
3228
  }
2596
3229
  function labelForItem(item) {
2597
3230
  return `${item.contentType}/${item.enSlug}@${item.locale}`;
2598
3231
  }
3232
+ async function translatePage(config, item, options = {}) {
3233
+ const startedAt = Date.now();
3234
+ let outcome;
3235
+ try {
3236
+ outcome = prepareTranslation(config, item, options, startedAt);
3237
+ } catch (error) {
3238
+ return failedResultForItem(item, error, startedAt);
3239
+ }
3240
+ if (outcome.status === "done") return outcome.result;
3241
+ const prepared = outcome.prepared;
3242
+ if (options.dryRun) {
3243
+ return {
3244
+ ...baseForItem(item),
3245
+ skipped: false,
3246
+ model: prepared.model,
3247
+ durationMs: Date.now() - startedAt
3248
+ };
3249
+ }
3250
+ try {
3251
+ const result = await translatePageWithGemini({
3252
+ prompt: prepared.prompt,
3253
+ model: prepared.model,
3254
+ responseSchema: prepared.responseSchema
3255
+ });
3256
+ return finalizeTranslation(
3257
+ config,
3258
+ prepared,
3259
+ { model: result.model, parsed: result.parsed, usage: result.usage },
3260
+ { costMode: "interactive", startedAt }
3261
+ );
3262
+ } catch (error) {
3263
+ return failedResultForItem(item, error, startedAt);
3264
+ }
3265
+ }
3266
+ function createResultCollector(items, onProgress) {
3267
+ const slotByKey = /* @__PURE__ */ new Map();
3268
+ items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
3269
+ const slots = new Array(items.length);
3270
+ const extras = [];
3271
+ return {
3272
+ emit(result) {
3273
+ const slot = slotByKey.get(translationItemKey(result));
3274
+ if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
3275
+ else extras.push(result);
3276
+ onProgress?.({ type: "item-done", result });
3277
+ },
3278
+ assemble() {
3279
+ return [
3280
+ ...slots.filter((result) => result !== void 0),
3281
+ ...extras
3282
+ ];
3283
+ }
3284
+ };
3285
+ }
2599
3286
  async function translateWorklist(config, items, options = {}) {
3287
+ const mode = options.mode ?? "batch";
2600
3288
  const concurrency = Math.max(1, options.concurrency ?? 3);
2601
3289
  const startedAt = Date.now();
2602
- const results = new Array(items.length);
2603
- const active = /* @__PURE__ */ new Set();
3290
+ if (options.dryRun) {
3291
+ options.onProgress?.({
3292
+ type: "start",
3293
+ total: items.length,
3294
+ concurrency,
3295
+ dryRun: true,
3296
+ model: options.model,
3297
+ mode
3298
+ });
3299
+ const results2 = items.map((item) => {
3300
+ const itemStartedAt = Date.now();
3301
+ try {
3302
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3303
+ return outcome.status === "done" ? outcome.result : {
3304
+ ...baseForItem(item),
3305
+ skipped: false,
3306
+ model: outcome.prepared.model,
3307
+ durationMs: Date.now() - itemStartedAt
3308
+ };
3309
+ } catch (error) {
3310
+ return failedResultForItem(item, error, itemStartedAt);
3311
+ }
3312
+ });
3313
+ for (const result of results2) options.onProgress?.({ type: "item-done", result });
3314
+ const totals2 = summarizeResults(results2, Date.now() - startedAt);
3315
+ options.onProgress?.({ type: "done", results: results2, totals: totals2 });
3316
+ return results2;
3317
+ }
3318
+ const pending = readPendingBatchWork(config);
3319
+ const inputKeys = new Set(items.map((item) => translationItemKey(item)));
3320
+ const resumedExtraCount = pending.pendingItems.filter(
3321
+ (item) => !inputKeys.has(translationItemKey(item))
3322
+ ).length;
2604
3323
  options.onProgress?.({
2605
3324
  type: "start",
2606
- total: items.length,
3325
+ total: items.length + resumedExtraCount,
2607
3326
  concurrency,
2608
- dryRun: Boolean(options.dryRun),
2609
- model: options.model
3327
+ dryRun: false,
3328
+ model: options.model,
3329
+ mode
3330
+ });
3331
+ const collector = createResultCollector(items, options.onProgress);
3332
+ const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
3333
+ if (mode === "direct") {
3334
+ const pendingCounts = countPendingItems(config, pending.jobs);
3335
+ emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
3336
+ const active = /* @__PURE__ */ new Set();
3337
+ await Promise.all([
3338
+ runPool(workItems, concurrency, async (item) => {
3339
+ const label = labelForItem(item);
3340
+ active.add(label);
3341
+ options.onProgress?.({ type: "item-start", item, active: [...active] });
3342
+ const result = await translatePage(config, item, options);
3343
+ active.delete(label);
3344
+ collector.emit(result);
3345
+ }),
3346
+ pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
3347
+ onProgress: options.onProgress,
3348
+ onResult: collector.emit
3349
+ }) : Promise.resolve()
3350
+ ]);
3351
+ } else {
3352
+ const entries = [];
3353
+ for (const item of workItems) {
3354
+ const itemStartedAt = Date.now();
3355
+ try {
3356
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3357
+ if (outcome.status === "done") {
3358
+ collector.emit(outcome.result);
3359
+ } else {
3360
+ entries.push({
3361
+ apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
3362
+ prompt: outcome.prepared.prompt,
3363
+ prepared: outcome.prepared
3364
+ });
3365
+ }
3366
+ } catch (error) {
3367
+ collector.emit(failedResultForItem(item, error, itemStartedAt));
3368
+ }
3369
+ }
3370
+ const plans = planBatchJobs(entries);
3371
+ const jobCount = pending.jobs.length + plans.length;
3372
+ const pendingCounts = countPendingItems(config, pending.jobs);
3373
+ emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
3374
+ const submitted = await Promise.all(
3375
+ plans.map(async (plan, planIndex) => {
3376
+ try {
3377
+ return await submitBatchJobPlan(config, {
3378
+ plan,
3379
+ displayModel: normalizeGeminiDisplayName(plan.apiModel),
3380
+ jobIndex: pending.jobs.length + planIndex,
3381
+ jobCount,
3382
+ onProgress: options.onProgress
3383
+ });
3384
+ } catch (error) {
3385
+ for (const entry of plan.entries) {
3386
+ collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
3387
+ }
3388
+ return void 0;
3389
+ }
3390
+ })
3391
+ );
3392
+ const jobsToPoll = [
3393
+ ...pending.jobs,
3394
+ ...submitted.filter((row) => row !== void 0)
3395
+ ];
3396
+ if (jobsToPoll.length > 0) {
3397
+ await pollAndIngestBatchJobs(config, jobsToPoll, {
3398
+ jobCount,
3399
+ onProgress: options.onProgress,
3400
+ onResult: collector.emit
3401
+ });
3402
+ }
3403
+ }
3404
+ const results = collector.assemble();
3405
+ const totals = summarizeResults(results, Date.now() - startedAt);
3406
+ options.onProgress?.({ type: "done", results, totals });
3407
+ return results;
3408
+ }
3409
+ function countPendingItems(config, jobs) {
3410
+ if (jobs.length === 0) return /* @__PURE__ */ new Map();
3411
+ const db = openStore(config, "readonly");
3412
+ const counts = /* @__PURE__ */ new Map();
3413
+ for (const job of jobs) {
3414
+ counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
3415
+ }
3416
+ db.close();
3417
+ return counts;
3418
+ }
3419
+ function emitResumedJobs(jobs, counts, jobCount, onProgress) {
3420
+ jobs.forEach((job, jobIndex) => {
3421
+ onProgress?.({
3422
+ type: "batch-submitted",
3423
+ name: job.job_name,
3424
+ count: counts.get(job.id) ?? 0,
3425
+ model: job.display_model,
3426
+ jobIndex,
3427
+ jobCount,
3428
+ resumed: true,
3429
+ createdAt: job.created_at
3430
+ });
3431
+ });
3432
+ }
3433
+ async function resumeTranslationJobs(config, options = {}) {
3434
+ const startedAt = Date.now();
3435
+ const pending = readPendingBatchWork(config);
3436
+ if (pending.jobs.length === 0) return null;
3437
+ options.onProgress?.({
3438
+ type: "start",
3439
+ total: pending.pendingItems.length,
3440
+ concurrency: 1,
3441
+ dryRun: false,
3442
+ mode: "batch"
2610
3443
  });
2611
- await runPool(items, concurrency, async (item, index) => {
2612
- const label = labelForItem(item);
2613
- active.add(label);
2614
- options.onProgress?.({ type: "item-start", item, active: [...active] });
2615
- const result = await translatePage(config, item, options);
2616
- results[index] = result;
2617
- active.delete(label);
2618
- options.onProgress?.({ type: "item-done", result });
3444
+ const counts = countPendingItems(config, pending.jobs);
3445
+ emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
3446
+ const results = [];
3447
+ await pollAndIngestBatchJobs(config, pending.jobs, {
3448
+ onProgress: options.onProgress,
3449
+ onResult: (result) => {
3450
+ results.push(result);
3451
+ options.onProgress?.({ type: "item-done", result });
3452
+ }
2619
3453
  });
2620
3454
  const totals = summarizeResults(results, Date.now() - startedAt);
2621
3455
  options.onProgress?.({ type: "done", results, totals });
@@ -3233,35 +4067,43 @@ async function promptContentType(config) {
3233
4067
  })
3234
4068
  );
3235
4069
  }
4070
+ var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
4071
+ var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
4072
+ async function promptManualLocales(config) {
4073
+ const locales = nonDefaultLocales(config);
4074
+ if (locales.length <= 1) return {};
4075
+ const picked = await runPrompt(
4076
+ prompts.checkbox({
4077
+ message: "Locales to translate",
4078
+ choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
4079
+ validate: (values) => values.length > 0 || "Pick at least one locale"
4080
+ })
4081
+ );
4082
+ return picked.length === locales.length ? {} : { locale: picked };
4083
+ }
3236
4084
  async function promptLocalePreset(config) {
3237
4085
  const presets = Object.entries(config.localePresets ?? {}).filter(
3238
4086
  (entry) => Array.isArray(entry[1])
3239
4087
  );
3240
4088
  if (presets.length === 0) {
3241
- const locales = nonDefaultLocales(config);
3242
- if (locales.length <= 1) return {};
3243
- const picked = await runPrompt(
3244
- prompts.checkbox({
3245
- message: "Locales to translate",
3246
- choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
3247
- validate: (values) => values.length > 0 || "Pick at least one locale"
3248
- })
3249
- );
3250
- return picked.length === locales.length ? {} : { locale: picked };
4089
+ return promptManualLocales(config);
3251
4090
  }
3252
4091
  const choice = await runPrompt(
3253
4092
  prompts.select({
3254
4093
  message: "Locale preset",
3255
4094
  choices: [
3256
- { name: "All locales", value: void 0 },
4095
+ { name: "All locales", value: ALL_LOCALES },
3257
4096
  ...presets.map(([name, locales]) => ({
3258
4097
  name: `${name} (${locales.join(", ")})`,
3259
4098
  value: name
3260
- }))
4099
+ })),
4100
+ { name: "Choose locales manually\u2026", value: PICK_LOCALES }
3261
4101
  ]
3262
4102
  })
3263
4103
  );
3264
- return choice ? { preset: choice } : {};
4104
+ if (choice === PICK_LOCALES) return promptManualLocales(config);
4105
+ if (choice === ALL_LOCALES) return {};
4106
+ return { preset: choice };
3265
4107
  }
3266
4108
  async function promptStrategy() {
3267
4109
  return runPrompt(
@@ -3275,12 +4117,25 @@ async function promptStrategy() {
3275
4117
  })
3276
4118
  );
3277
4119
  }
4120
+ async function promptMode() {
4121
+ return runPrompt(
4122
+ prompts.select({
4123
+ message: "Translation mode",
4124
+ choices: [
4125
+ { name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
4126
+ { name: "Direct \u2014 immediate results, full price", value: "direct" }
4127
+ ],
4128
+ default: "batch"
4129
+ })
4130
+ );
4131
+ }
3278
4132
  async function promptTranslateSelection(config, flags) {
3279
4133
  const selection = {
3280
4134
  contentType: flags.type,
3281
4135
  preset: flags.preset,
3282
4136
  locale: flags.locale,
3283
- strategy: flags.strategy
4137
+ strategy: flags.strategy,
4138
+ mode: flags.mode
3284
4139
  };
3285
4140
  if (!isInteractive()) return selection;
3286
4141
  if (!selection.contentType) {
@@ -3292,6 +4147,9 @@ async function promptTranslateSelection(config, flags) {
3292
4147
  if (!selection.strategy) {
3293
4148
  selection.strategy = await promptStrategy();
3294
4149
  }
4150
+ if (!selection.mode) {
4151
+ selection.mode = await promptMode();
4152
+ }
3295
4153
  return selection;
3296
4154
  }
3297
4155
 
@@ -3313,6 +4171,34 @@ function progressBar(ratio, width = 28) {
3313
4171
  function labelForResult(result) {
3314
4172
  return `${result.contentType}/${result.enSlug}@${result.locale}`;
3315
4173
  }
4174
+ function shortJobId(name) {
4175
+ const segments = name.split("/");
4176
+ return segments[segments.length - 1] || name;
4177
+ }
4178
+ function formatElapsed(elapsedMs) {
4179
+ return `${Math.round(elapsedMs / 1e3)}s`;
4180
+ }
4181
+ function batchDoneParts(event) {
4182
+ if (event.alreadyIngested) {
4183
+ return [
4184
+ `job ${shortJobId(event.name)}`,
4185
+ event.model ?? "?",
4186
+ "already ingested by another scribe run (results saved there)",
4187
+ formatElapsed(event.elapsedMs)
4188
+ ].join(" \xB7 ");
4189
+ }
4190
+ const parts = [
4191
+ `job ${shortJobId(event.name)}`,
4192
+ event.model ?? "?",
4193
+ `${event.count} request${event.count === 1 ? "" : "s"}`,
4194
+ `${event.translated} translated`
4195
+ ];
4196
+ if (event.failed > 0) parts.push(`${event.failed} failed`);
4197
+ parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
4198
+ parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
4199
+ parts.push(formatElapsed(event.elapsedMs));
4200
+ return parts.join(" \xB7 ");
4201
+ }
3316
4202
  function statusForResult(result, dryRun) {
3317
4203
  if (result.failed) return red("failed");
3318
4204
  if (result.skipped) return dim("skipped");
@@ -3358,6 +4244,25 @@ function createTranslateProgressReporter(options = {}) {
3358
4244
  if (!enabled) {
3359
4245
  return {
3360
4246
  onEvent(event) {
4247
+ if (event.type === "batch-submitted") {
4248
+ const verb = event.resumed ? "Resuming" : "Submitted";
4249
+ console.log(
4250
+ `${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
4251
+ );
4252
+ return;
4253
+ }
4254
+ if (event.type === "batch-polling") {
4255
+ console.log(
4256
+ `batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
4257
+ );
4258
+ return;
4259
+ }
4260
+ if (event.type === "batch-done") {
4261
+ const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
4262
+ if (event.failed > 0 && event.translated === 0) console.error(line);
4263
+ else console.log(line);
4264
+ return;
4265
+ }
3361
4266
  if (event.type === "item-done") {
3362
4267
  const label = labelForResult(event.result);
3363
4268
  const status = statusForResult(event.result, dryRun);
@@ -3388,6 +4293,9 @@ function createTranslateProgressReporter(options = {}) {
3388
4293
  let outputTokens = 0;
3389
4294
  let estimatedCostUsd = 0;
3390
4295
  let active = [];
4296
+ let mode;
4297
+ const batchJobs = /* @__PURE__ */ new Map();
4298
+ let batchElapsedMs = 0;
3391
4299
  const recent = [];
3392
4300
  let renderedLines = 0;
3393
4301
  let cursorHidden = false;
@@ -3403,6 +4311,15 @@ function createTranslateProgressReporter(options = {}) {
3403
4311
  cursorHidden = false;
3404
4312
  }
3405
4313
  }
4314
+ function printPersistent(lines) {
4315
+ if (renderedLines > 0) {
4316
+ process.stdout.write(`\x1B[${renderedLines}A`);
4317
+ }
4318
+ for (const line of lines) {
4319
+ process.stdout.write("\x1B[2K" + line + "\n");
4320
+ }
4321
+ renderedLines = 0;
4322
+ }
3406
4323
  function render() {
3407
4324
  hideCursor();
3408
4325
  if (renderedLines > 0) {
@@ -3410,12 +4327,34 @@ function createTranslateProgressReporter(options = {}) {
3410
4327
  }
3411
4328
  const lines = [];
3412
4329
  const title = dryRun ? "Dry run" : "Translating";
3413
- lines.push(cyan(`${title} ${done}/${total}`) + dim(` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : ""));
4330
+ const isBatch = mode === "batch" || batchJobs.size > 0;
4331
+ lines.push(
4332
+ cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
4333
+ );
3414
4334
  lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
3415
4335
  lines.push(
3416
4336
  dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
3417
4337
  );
3418
- if (active.length > 0) {
4338
+ if (batchJobs.size > 0 && done < total) {
4339
+ const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
4340
+ let running = 0;
4341
+ let finished = 0;
4342
+ let requests = 0;
4343
+ for (const job of batchJobs.values()) {
4344
+ if (terminal.test(job.state)) finished += 1;
4345
+ else running += 1;
4346
+ requests += job.count;
4347
+ }
4348
+ lines.push(
4349
+ dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
4350
+ );
4351
+ for (const [name, job] of batchJobs) {
4352
+ if (terminal.test(job.state)) continue;
4353
+ lines.push(
4354
+ dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
4355
+ );
4356
+ }
4357
+ } else if (active.length > 0) {
3419
4358
  lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
3420
4359
  } else if (done < total) {
3421
4360
  lines.push(dim("Active ") + "starting\u2026");
@@ -3443,12 +4382,58 @@ function createTranslateProgressReporter(options = {}) {
3443
4382
  total = event.total;
3444
4383
  concurrency = event.concurrency;
3445
4384
  model = event.model;
4385
+ mode = event.mode;
3446
4386
  render();
3447
4387
  break;
3448
4388
  case "item-start":
3449
4389
  active = event.active;
3450
4390
  render();
3451
4391
  break;
4392
+ case "batch-submitted": {
4393
+ const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
4394
+ const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
4395
+ batchJobs.set(event.name, {
4396
+ state: event.resumed ? "RESUMED" : "SUBMITTED",
4397
+ count: event.count,
4398
+ model: event.model,
4399
+ createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
4400
+ elapsedMs
4401
+ });
4402
+ const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
4403
+ const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
4404
+ printPersistent([
4405
+ dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
4406
+ ]);
4407
+ render();
4408
+ break;
4409
+ }
4410
+ case "batch-polling": {
4411
+ const job = batchJobs.get(event.name);
4412
+ batchJobs.set(event.name, {
4413
+ state: event.state.replace(/^JOB_STATE_/, ""),
4414
+ count: job?.count ?? 0,
4415
+ model: job?.model,
4416
+ createdAtMs: job?.createdAtMs,
4417
+ elapsedMs: event.elapsedMs
4418
+ });
4419
+ batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
4420
+ render();
4421
+ break;
4422
+ }
4423
+ case "batch-done": {
4424
+ const job = batchJobs.get(event.name);
4425
+ batchJobs.set(event.name, {
4426
+ state: event.state.replace(/^JOB_STATE_/, ""),
4427
+ count: job?.count ?? event.count,
4428
+ model: job?.model ?? event.model,
4429
+ createdAtMs: job?.createdAtMs,
4430
+ elapsedMs: event.elapsedMs
4431
+ });
4432
+ const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
4433
+ printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
4434
+ render();
4435
+ break;
4436
+ }
3452
4437
  case "item-done": {
3453
4438
  done += 1;
3454
4439
  active = active.filter((label) => label !== labelForResult(event.result));
@@ -3571,6 +4556,21 @@ function parseArgs(argv) {
3571
4556
  i++;
3572
4557
  continue;
3573
4558
  }
4559
+ if (arg === "--batch") {
4560
+ options.batch = true;
4561
+ i++;
4562
+ continue;
4563
+ }
4564
+ if (arg === "--direct") {
4565
+ options.direct = true;
4566
+ i++;
4567
+ continue;
4568
+ }
4569
+ if (arg === "--resume") {
4570
+ options.resume = true;
4571
+ i++;
4572
+ continue;
4573
+ }
3574
4574
  if (arg === "--strategy") {
3575
4575
  const value = argv[++i];
3576
4576
  if (value !== "all" && value !== "missing-only") {
@@ -3639,11 +4639,19 @@ Translate flags:
3639
4639
  --locale <code>... Target locale(s); overrides --preset
3640
4640
  --slug <en-slug> Single English document
3641
4641
  --model <id> Gemini model override
3642
- --concurrency <n> Parallel translations (default: 3)
4642
+ --batch Force Gemini Batch API mode (50% token cost; default)
4643
+ --direct Force direct interactive calls (immediate, full price)
4644
+ --resume Only poll/ingest pending batch jobs; submit nothing new
4645
+ --concurrency <n> Parallel translations in --direct mode (default: 3)
3643
4646
  --dry-run List work without writing
3644
4647
  --force Re-translate even when hashes match
3645
4648
  --strategy <mode> all (default) or missing-only
3646
4649
  --no-progress Plain line logging instead of live progress
4650
+
4651
+ Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
4652
+ during polling is safe: the next run (or --resume) picks the jobs back up and
4653
+ ingests their results. In a TTY without --batch/--direct, a prompt asks which
4654
+ mode to use.
3647
4655
  `);
3648
4656
  return;
3649
4657
  }
@@ -3689,11 +4697,29 @@ Translate flags:
3689
4697
  break;
3690
4698
  }
3691
4699
  case "translate": {
4700
+ if (options.batch && options.direct) {
4701
+ throw new Error("--batch and --direct are mutually exclusive");
4702
+ }
4703
+ if (options.resume) {
4704
+ const reporter2 = createTranslateProgressReporter({
4705
+ enabled: !options.noProgress,
4706
+ dryRun: false
4707
+ });
4708
+ const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
4709
+ reporter2.finish();
4710
+ if (results2 === null) {
4711
+ console.log("No pending translation batch jobs.");
4712
+ break;
4713
+ }
4714
+ if (results2.some((result) => result.failed)) process.exitCode = 1;
4715
+ break;
4716
+ }
3692
4717
  const selection = await promptTranslateSelection(config, {
3693
4718
  type: options.type,
3694
4719
  preset: options.preset,
3695
4720
  locale: options.locale,
3696
- strategy: options.strategy
4721
+ strategy: options.strategy,
4722
+ mode: options.batch ? "batch" : options.direct ? "direct" : void 0
3697
4723
  });
3698
4724
  const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
3699
4725
  const worklist = buildWorklist(config, {
@@ -3703,6 +4729,18 @@ Translate flags:
3703
4729
  strategy: selection.strategy ?? "all"
3704
4730
  });
3705
4731
  if (worklist.length === 0) {
4732
+ if (!options.dryRun) {
4733
+ const reporter2 = createTranslateProgressReporter({
4734
+ enabled: !options.noProgress,
4735
+ dryRun: false
4736
+ });
4737
+ const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
4738
+ reporter2.finish();
4739
+ if (resumed !== null) {
4740
+ if (resumed.some((result) => result.failed)) process.exitCode = 1;
4741
+ break;
4742
+ }
4743
+ }
3706
4744
  console.log("Nothing to translate.");
3707
4745
  break;
3708
4746
  }
@@ -3715,6 +4753,7 @@ Translate flags:
3715
4753
  dryRun: options.dryRun,
3716
4754
  force: options.force,
3717
4755
  concurrency: options.concurrency,
4756
+ mode: selection.mode,
3718
4757
  onProgress: reporter.onEvent
3719
4758
  });
3720
4759
  reporter.finish();