scribe-cms 0.0.12 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +21 -6
  2. package/dist/cli/index.cjs +1220 -204
  3. package/dist/cli/index.cjs.map +1 -1
  4. package/dist/cli/index.js +1221 -205
  5. package/dist/cli/index.js.map +1 -1
  6. package/dist/cli/prompt-translate.d.ts +3 -0
  7. package/dist/cli/prompt-translate.d.ts.map +1 -1
  8. package/dist/cli/translate-progress.d.ts.map +1 -1
  9. package/dist/index.cjs +982 -166
  10. package/dist/index.cjs.map +1 -1
  11. package/dist/index.js +983 -168
  12. package/dist/index.js.map +1 -1
  13. package/dist/runtime.cjs +149 -62
  14. package/dist/runtime.cjs.map +1 -1
  15. package/dist/runtime.js +149 -62
  16. package/dist/runtime.js.map +1 -1
  17. package/dist/src/config/resolve-config.d.ts.map +1 -1
  18. package/dist/src/core/introspect-schema.d.ts +7 -1
  19. package/dist/src/core/introspect-schema.d.ts.map +1 -1
  20. package/dist/src/core/types.d.ts +6 -0
  21. package/dist/src/core/types.d.ts.map +1 -1
  22. package/dist/src/create-project.d.ts.map +1 -1
  23. package/dist/src/i18n/resolve-document.d.ts +1 -1
  24. package/dist/src/i18n/resolve-document.d.ts.map +1 -1
  25. package/dist/src/index.d.ts +3 -3
  26. package/dist/src/index.d.ts.map +1 -1
  27. package/dist/src/storage/batch-jobs.d.ts +53 -0
  28. package/dist/src/storage/batch-jobs.d.ts.map +1 -0
  29. package/dist/src/storage/sqlite.d.ts.map +1 -1
  30. package/dist/src/translate/batch-worklist.d.ts +87 -0
  31. package/dist/src/translate/batch-worklist.d.ts.map +1 -0
  32. package/dist/src/translate/gemini-batch.d.ts +26 -0
  33. package/dist/src/translate/gemini-batch.d.ts.map +1 -0
  34. package/dist/src/translate/gemini-client.d.ts +18 -0
  35. package/dist/src/translate/gemini-client.d.ts.map +1 -1
  36. package/dist/src/translate/gemini-models.d.ts +9 -0
  37. package/dist/src/translate/gemini-models.d.ts.map +1 -1
  38. package/dist/src/translate/gemini-pricing.d.ts +2 -1
  39. package/dist/src/translate/gemini-pricing.d.ts.map +1 -1
  40. package/dist/src/translate/page-translator.d.ts +25 -52
  41. package/dist/src/translate/page-translator.d.ts.map +1 -1
  42. package/dist/src/translate/prompts/translation-prompt.d.ts +6 -0
  43. package/dist/src/translate/prompts/translation-prompt.d.ts.map +1 -1
  44. package/dist/src/translate/response-schema.d.ts +8 -1
  45. package/dist/src/translate/response-schema.d.ts.map +1 -1
  46. package/dist/src/translate/retry.d.ts +16 -0
  47. package/dist/src/translate/retry.d.ts.map +1 -0
  48. package/dist/src/translate/sanitize-mdx-jsx.d.ts.map +1 -1
  49. package/dist/src/translate/translate-core.d.ts +155 -0
  50. package/dist/src/translate/translate-core.d.ts.map +1 -0
  51. package/dist/studio/server.cjs +84 -47
  52. package/dist/studio/server.cjs.map +1 -1
  53. package/dist/studio/server.js +84 -47
  54. package/dist/studio/server.js.map +1 -1
  55. package/package.json +1 -1
package/dist/cli/index.js CHANGED
@@ -10,7 +10,7 @@ import remarkParse from 'remark-parse';
10
10
  import { unified } from 'unified';
11
11
  import { createJiti } from 'jiti';
12
12
  import { createHash } from 'crypto';
13
- import { GoogleGenAI } from '@google/genai';
13
+ import { GoogleGenAI, ThinkingLevel, ApiError } from '@google/genai';
14
14
  import dotenv from 'dotenv';
15
15
  import { serve } from '@hono/node-server';
16
16
  import { Hono } from 'hono';
@@ -59,6 +59,8 @@ function addColumnIfMissing(db, table, column, ddlType) {
59
59
  }
60
60
  }
61
61
  function readSchemaVersion(db) {
62
+ const metaTable = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'`).get();
63
+ if (!metaTable) return 0;
62
64
  const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
63
65
  return row ? Number.parseInt(row.value, 10) || 0 : 0;
64
66
  }
@@ -83,7 +85,7 @@ var SCHEMA_VERSION, MIGRATIONS;
83
85
  var init_sqlite = __esm({
84
86
  "src/storage/sqlite.ts"() {
85
87
  init_esm_shims();
86
- SCHEMA_VERSION = 4;
88
+ SCHEMA_VERSION = 5;
87
89
  MIGRATIONS = [
88
90
  `CREATE TABLE IF NOT EXISTS meta (
89
91
  key TEXT PRIMARY KEY,
@@ -131,7 +133,30 @@ var init_sqlite = __esm({
131
133
  UNIQUE (content_type, en_slug, en_hash)
132
134
  )`,
133
135
  `CREATE INDEX IF NOT EXISTS idx_en_snapshots_lookup
134
- ON en_snapshots(content_type, en_slug, created_at DESC)`
136
+ ON en_snapshots(content_type, en_slug, created_at DESC)`,
137
+ `CREATE TABLE IF NOT EXISTS translation_batch_jobs (
138
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
139
+ job_name TEXT NOT NULL UNIQUE,
140
+ model TEXT NOT NULL,
141
+ display_model TEXT NOT NULL,
142
+ created_at TEXT NOT NULL,
143
+ state TEXT NOT NULL,
144
+ completed_at TEXT
145
+ )`,
146
+ `CREATE TABLE IF NOT EXISTS translation_batch_items (
147
+ job_id INTEGER NOT NULL,
148
+ request_index INTEGER NOT NULL,
149
+ content_type TEXT NOT NULL,
150
+ en_slug TEXT NOT NULL,
151
+ locale TEXT NOT NULL,
152
+ en_hash TEXT NOT NULL,
153
+ snapshot_id INTEGER NOT NULL,
154
+ status TEXT NOT NULL,
155
+ error TEXT,
156
+ PRIMARY KEY (job_id, request_index)
157
+ )`,
158
+ `CREATE INDEX IF NOT EXISTS idx_batch_items_status
159
+ ON translation_batch_items(status)`
135
160
  ];
136
161
  }
137
162
  });
@@ -173,26 +198,6 @@ function getRelationTarget(schema) {
173
198
  }
174
199
  return null;
175
200
  }
176
- function unwrapSchema(schema) {
177
- let current = schema;
178
- for (let i = 0; i < 8; i++) {
179
- const anySchema = current;
180
- if (typeof anySchema.unwrap === "function") {
181
- current = anySchema.unwrap();
182
- continue;
183
- }
184
- if (typeof anySchema.removeDefault === "function") {
185
- current = anySchema.removeDefault();
186
- continue;
187
- }
188
- if (anySchema._def?.innerType) {
189
- current = anySchema._def.innerType;
190
- continue;
191
- }
192
- break;
193
- }
194
- return current;
195
- }
196
201
  function peelOptionalWrappers(schema) {
197
202
  let current = schema;
198
203
  for (let i = 0; i < 8; i++) {
@@ -338,6 +343,39 @@ function resolveConfig(input, baseDir) {
338
343
  if (localeRouting.strategy === "search-param" && !localeRouting.param) {
339
344
  throw new Error('scribe config: localeRouting search-param requires a "param" name');
340
345
  }
346
+ const localeFallbacks = {};
347
+ for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
348
+ if (!raw.locales.includes(locale)) {
349
+ throw new Error(
350
+ `scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
351
+ );
352
+ }
353
+ const seen = /* @__PURE__ */ new Set();
354
+ for (const fallback of chain) {
355
+ if (!raw.locales.includes(fallback)) {
356
+ throw new Error(
357
+ `scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
358
+ );
359
+ }
360
+ if (fallback === locale) {
361
+ throw new Error(
362
+ `scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
363
+ );
364
+ }
365
+ if (fallback === defaultLocale) {
366
+ throw new Error(
367
+ `scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
368
+ );
369
+ }
370
+ if (seen.has(fallback)) {
371
+ throw new Error(
372
+ `scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
373
+ );
374
+ }
375
+ seen.add(fallback);
376
+ }
377
+ localeFallbacks[locale] = [...chain];
378
+ }
341
379
  const projectRoot = path4.resolve(baseDir ?? process.cwd(), raw.rootDir);
342
380
  const contentRoot = path4.resolve(projectRoot, raw.contentDir ?? "content");
343
381
  const storePath = path4.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
@@ -367,6 +405,7 @@ function resolveConfig(input, baseDir) {
367
405
  defaultLocale,
368
406
  localeRouting,
369
407
  localePresets: raw.localePresets,
408
+ localeFallbacks,
370
409
  translate: raw.translate,
371
410
  types
372
411
  };
@@ -382,6 +421,12 @@ init_esm_shims();
382
421
 
383
422
  // src/core/introspect-schema.ts
384
423
  init_esm_shims();
424
+ function getArrayElement(schema) {
425
+ if (schema instanceof Object && "element" in schema && schema._def?.type === "array") {
426
+ return schema.element;
427
+ }
428
+ return null;
429
+ }
385
430
  function introspectSchema(schema, prefix = []) {
386
431
  const relation = getRelationTarget(schema);
387
432
  if (relation && prefix.length > 0) {
@@ -395,7 +440,10 @@ function introspectSchema(schema, prefix = []) {
395
440
  }
396
441
  ];
397
442
  }
398
- const base = unwrapSchema(schema);
443
+ if (prefix.length > 0 && getFieldKind(schema) === "translatable") {
444
+ return [{ path: prefix, kind: "translatable" }];
445
+ }
446
+ const base = peelOptionalWrappers(schema);
399
447
  if (base instanceof Object && "shape" in base) {
400
448
  const shape = base.shape;
401
449
  const fields = [];
@@ -404,9 +452,9 @@ function introspectSchema(schema, prefix = []) {
404
452
  }
405
453
  return fields;
406
454
  }
407
- if (base instanceof Object && "element" in base) {
408
- const element = base.element;
409
- const elementBase = unwrapSchema(element);
455
+ const element = getArrayElement(base);
456
+ if (element) {
457
+ const elementBase = peelOptionalWrappers(element);
410
458
  if (elementBase instanceof Object && "shape" in elementBase) {
411
459
  const shape = elementBase.shape;
412
460
  const fields = [];
@@ -418,15 +466,48 @@ function introspectSchema(schema, prefix = []) {
418
466
  }
419
467
  return [{ path: prefix, kind: getFieldKind(schema) }];
420
468
  }
421
- function extractByPaths(data, paths) {
422
- const out = {};
469
+ function buildPathTrie(paths) {
470
+ const root = { leaf: false, children: /* @__PURE__ */ new Map() };
423
471
  for (const path16 of paths) {
424
- const value = getAtPath(data, path16);
472
+ let node = root;
473
+ for (const segment of path16) {
474
+ let child = node.children.get(segment);
475
+ if (!child) {
476
+ child = { leaf: false, children: /* @__PURE__ */ new Map() };
477
+ node.children.set(segment, child);
478
+ }
479
+ node = child;
480
+ }
481
+ node.leaf = true;
482
+ }
483
+ return root;
484
+ }
485
+ function extractNode(data, trie) {
486
+ if (trie.leaf) return data;
487
+ const star = trie.children.get("*");
488
+ if (star) {
489
+ if (!Array.isArray(data)) return void 0;
490
+ return data.map(
491
+ (item) => typeof item === "object" && item !== null ? extractNode(item, star) ?? {} : {}
492
+ );
493
+ }
494
+ if (typeof data !== "object" || data === null || Array.isArray(data)) return void 0;
495
+ const record = data;
496
+ const out = {};
497
+ let any = false;
498
+ for (const [key, child] of trie.children) {
499
+ const value = extractNode(record[key], child);
425
500
  if (value !== void 0) {
426
- setAtPath(out, path16.filter((p) => p !== "*"), value);
501
+ out[key] = value;
502
+ any = true;
427
503
  }
428
504
  }
429
- return out;
505
+ return any ? out : void 0;
506
+ }
507
+ function extractByPaths(data, paths) {
508
+ if (paths.length === 0) return {};
509
+ const extracted = extractNode(data, buildPathTrie(paths));
510
+ return extracted && typeof extracted === "object" && !Array.isArray(extracted) ? extracted : {};
430
511
  }
431
512
  function pickTranslatable(data, schema) {
432
513
  const translatablePaths = introspectSchema(schema).filter((f) => f.kind === "translatable").map((f) => f.path);
@@ -441,32 +522,6 @@ function mergeStructuralOntoLocale(localeData, enData, schema) {
441
522
  const translatable = pickTranslatable(localeData, schema);
442
523
  return deepMerge(structural, translatable);
443
524
  }
444
- function getAtPath(obj, path16) {
445
- let current = obj;
446
- for (const segment of path16) {
447
- if (segment === "*") {
448
- if (!Array.isArray(current)) return void 0;
449
- return current.map(
450
- (item) => typeof item === "object" && item !== null ? getAtPath(item, path16.slice(path16.indexOf("*") + 1)) : void 0
451
- );
452
- }
453
- if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
454
- current = current[segment];
455
- }
456
- return current;
457
- }
458
- function setAtPath(obj, path16, value) {
459
- if (path16.length === 0) return;
460
- if (path16.length === 1) {
461
- obj[path16[0]] = value;
462
- return;
463
- }
464
- const [head, ...rest] = path16;
465
- if (!(head in obj) || typeof obj[head] !== "object" || obj[head] === null) {
466
- obj[head] = {};
467
- }
468
- setAtPath(obj[head], rest, value);
469
- }
470
525
  function deepMerge(base, overlay) {
471
526
  const out = { ...base };
472
527
  for (const [key, value] of Object.entries(overlay)) {
@@ -757,7 +812,8 @@ function sanitizeMdxJsxAttributeQuotes(body) {
757
812
  if (i >= body.length || body[i] === ">" || body[i] === "/") break;
758
813
  const attrStart = i;
759
814
  while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
760
- body.slice(attrStart, i);
815
+ const attrName = body.slice(attrStart, i);
816
+ if (!attrName) break;
761
817
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
762
818
  if (body[i] !== "=") {
763
819
  tagOut += body.slice(attrStart, i);
@@ -766,10 +822,12 @@ function sanitizeMdxJsxAttributeQuotes(body) {
766
822
  tagOut += body.slice(attrStart, i);
767
823
  tagOut += "=";
768
824
  i += 1;
825
+ const wsStart = i;
769
826
  while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
827
+ tagOut += body.slice(wsStart, i);
770
828
  const quote = body[i];
771
829
  if (quote !== '"' && quote !== "'") {
772
- continue;
830
+ break;
773
831
  }
774
832
  if (quote === "'") {
775
833
  const valStart2 = i + 1;
@@ -1091,7 +1149,7 @@ function getTranslatablePayload(doc, type) {
1091
1149
 
1092
1150
  // src/i18n/resolve-document.ts
1093
1151
  init_esm_shims();
1094
- function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting) {
1152
+ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, localeRouting, fallbackLocales = []) {
1095
1153
  const urlBuilder = createUrlBuilder({
1096
1154
  locales: [defaultLocale, locale],
1097
1155
  defaultLocale,
@@ -1105,15 +1163,28 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1105
1163
  for (const [docLocale, docIdx] of allDocs) {
1106
1164
  const found = docIdx.bySlug.get(slug);
1107
1165
  if (!found) continue;
1108
- const correctSlug = getSlugForLocale(found, docLocale, locale, allDocs, defaultLocale);
1109
- if (correctSlug && correctSlug !== slug) {
1166
+ const enSlug = docLocale === defaultLocale ? found.slug : found.enSlug;
1167
+ let target;
1168
+ let targetLocale = locale;
1169
+ for (const candidateLocale of [locale, ...fallbackLocales]) {
1170
+ const cand = candidateLocale === defaultLocale ? allDocs.get(candidateLocale)?.bySlug.get(enSlug) : allDocs.get(candidateLocale)?.byEnSlug.get(enSlug);
1171
+ if (cand) {
1172
+ target = cand;
1173
+ targetLocale = candidateLocale;
1174
+ break;
1175
+ }
1176
+ }
1177
+ if (target) {
1178
+ if (target.slug === slug) {
1179
+ return { document: target, actualLocale: targetLocale };
1180
+ }
1110
1181
  if (!type.path) {
1111
1182
  return { document: null, actualLocale: locale };
1112
1183
  }
1113
1184
  return {
1114
1185
  document: null,
1115
1186
  actualLocale: locale,
1116
- shouldRedirectTo: urlBuilder.resolvePath(type.path, correctSlug, locale)
1187
+ shouldRedirectTo: urlBuilder.resolvePath(type.path, target.slug, locale)
1117
1188
  };
1118
1189
  }
1119
1190
  if (docLocale === defaultLocale) {
@@ -1144,13 +1215,6 @@ function resolveLocalizedDocument(slug, locale, defaultLocale, allDocs, type, lo
1144
1215
  }
1145
1216
  return { document: null, actualLocale: locale };
1146
1217
  }
1147
- function getSlugForLocale(document, sourceLocale, targetLocale, allDocs, defaultLocale) {
1148
- if (sourceLocale === targetLocale) return document.slug;
1149
- const englishSlug = sourceLocale === defaultLocale ? document.slug : document.enSlug;
1150
- if (!englishSlug) return null;
1151
- if (targetLocale === defaultLocale) return englishSlug;
1152
- return allDocs.get(targetLocale)?.byEnSlug.get(englishSlug)?.slug ?? null;
1153
- }
1154
1218
 
1155
1219
  // src/create-project.ts
1156
1220
  function comparatorFor(orderBy) {
@@ -1203,7 +1267,8 @@ function buildRuntime(config, type, getRuntime) {
1203
1267
  config.defaultLocale,
1204
1268
  load(),
1205
1269
  type,
1206
- config.localeRouting
1270
+ config.localeRouting,
1271
+ config.localeFallbacks[locale] ?? []
1207
1272
  );
1208
1273
  if (result.document && type.path) {
1209
1274
  return {
@@ -1225,8 +1290,14 @@ function buildRuntime(config, type, getRuntime) {
1225
1290
  const params = [];
1226
1291
  for (const locale of options.locales ?? config.locales) {
1227
1292
  const localeIdx = all.get(locale);
1293
+ const fallbacks = config.localeFallbacks[locale] ?? [];
1228
1294
  for (const doc of enIdx.bySlug.values()) {
1229
- const slug = locale === config.defaultLocale ? doc.slug : localeIdx?.byEnSlug.get(doc.slug)?.slug ?? doc.slug;
1295
+ let slug;
1296
+ if (locale === config.defaultLocale) {
1297
+ slug = doc.slug;
1298
+ } else {
1299
+ slug = localeIdx?.byEnSlug.get(doc.slug)?.slug ?? fallbacks.map((fb) => all.get(fb)?.byEnSlug.get(doc.slug)?.slug).find((s) => s !== void 0) ?? doc.slug;
1300
+ }
1230
1301
  params.push({ locale, slug });
1231
1302
  }
1232
1303
  }
@@ -1688,7 +1759,7 @@ init_sqlite();
1688
1759
 
1689
1760
  // src/validate/validate-relations.ts
1690
1761
  init_esm_shims();
1691
- function getAtPath2(obj, fieldPath) {
1762
+ function getAtPath(obj, fieldPath) {
1692
1763
  let current = obj;
1693
1764
  for (const segment of fieldPath) {
1694
1765
  if (typeof current !== "object" || current === null || Array.isArray(current)) return void 0;
@@ -1733,7 +1804,7 @@ function validateRelations(project) {
1733
1804
  });
1734
1805
  continue;
1735
1806
  }
1736
- const value = getAtPath2(doc.frontmatter, fieldMeta.path);
1807
+ const value = getAtPath(doc.frontmatter, fieldMeta.path);
1737
1808
  if (value === void 0 || value === null || value === "") continue;
1738
1809
  const targetSlugs = slugIndex.get(targetTypeId) ?? /* @__PURE__ */ new Set();
1739
1810
  const fieldLabel = fieldMeta.path.join(".");
@@ -2117,6 +2188,66 @@ function resolveLocalesFromPreset(config, preset, explicitLocales) {
2117
2188
  // src/translate/page-translator.ts
2118
2189
  init_esm_shims();
2119
2190
 
2191
+ // src/storage/batch-jobs.ts
2192
+ init_esm_shims();
2193
+ function insertBatchJob(db, input) {
2194
+ const info = db.prepare(
2195
+ `INSERT INTO translation_batch_jobs (job_name, model, display_model, created_at, state)
2196
+ VALUES (?, ?, ?, ?, ?)`
2197
+ ).run(input.jobName, input.model, input.displayModel, input.createdAt, input.state);
2198
+ return Number(info.lastInsertRowid);
2199
+ }
2200
+ function insertBatchItems(db, jobId, items) {
2201
+ const stmt = db.prepare(
2202
+ `INSERT INTO translation_batch_items (
2203
+ job_id, request_index, content_type, en_slug, locale, en_hash, snapshot_id, status
2204
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')`
2205
+ );
2206
+ for (const item of items) {
2207
+ stmt.run(
2208
+ jobId,
2209
+ item.requestIndex,
2210
+ item.contentType,
2211
+ item.enSlug,
2212
+ item.locale,
2213
+ item.enHash,
2214
+ item.snapshotId
2215
+ );
2216
+ }
2217
+ }
2218
+ function listPendingBatchJobs(db) {
2219
+ return db.prepare(`SELECT * FROM translation_batch_jobs WHERE completed_at IS NULL ORDER BY id`).all();
2220
+ }
2221
+ function listBatchItems(db, jobId) {
2222
+ return db.prepare(`SELECT * FROM translation_batch_items WHERE job_id = ? ORDER BY request_index`).all(jobId);
2223
+ }
2224
+ function listPendingBatchItems(db) {
2225
+ return db.prepare(
2226
+ `SELECT i.* FROM translation_batch_items i
2227
+ JOIN translation_batch_jobs j ON j.id = i.job_id
2228
+ WHERE j.completed_at IS NULL AND i.status = 'pending'
2229
+ ORDER BY i.job_id, i.request_index`
2230
+ ).all();
2231
+ }
2232
+ function claimBatchJobCompletion(db, jobId, state, completedAt) {
2233
+ const info = db.prepare(
2234
+ `UPDATE translation_batch_jobs SET state = ?, completed_at = ?
2235
+ WHERE id = ? AND completed_at IS NULL`
2236
+ ).run(state, completedAt, jobId);
2237
+ return info.changes > 0;
2238
+ }
2239
+ function updateBatchItemStatus(db, jobId, requestIndex, status, error) {
2240
+ db.prepare(
2241
+ `UPDATE translation_batch_items SET status = ?, error = ? WHERE job_id = ? AND request_index = ?`
2242
+ ).run(status, error ?? null, jobId, requestIndex);
2243
+ }
2244
+
2245
+ // src/translate/page-translator.ts
2246
+ init_sqlite();
2247
+
2248
+ // src/translate/batch-worklist.ts
2249
+ init_esm_shims();
2250
+
2120
2251
  // src/history/record-snapshot.ts
2121
2252
  init_esm_shims();
2122
2253
  init_sqlite();
@@ -2134,9 +2265,111 @@ function recordEnSnapshot(config, input, db) {
2134
2265
  return id;
2135
2266
  }
2136
2267
 
2137
- // src/translate/page-translator.ts
2268
+ // src/translate/batch-worklist.ts
2138
2269
  init_sqlite();
2139
2270
 
2271
+ // src/translate/gemini-batch.ts
2272
+ init_esm_shims();
2273
+
2274
+ // src/translate/retry.ts
2275
+ init_esm_shims();
2276
+ var TRANSIENT_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
2277
+ function statusFromError(error) {
2278
+ if (error instanceof ApiError && typeof error.status === "number") {
2279
+ return error.status;
2280
+ }
2281
+ if (error && typeof error === "object" && "status" in error) {
2282
+ const status = error.status;
2283
+ if (typeof status === "number") return status;
2284
+ }
2285
+ const message = error instanceof Error ? error.message : String(error);
2286
+ const match = message.match(/\b(429|500|502|503|504)\b/);
2287
+ if (match) return Number(match[1]);
2288
+ return void 0;
2289
+ }
2290
+ function isNetworkError(error) {
2291
+ const err = error;
2292
+ const code = typeof err?.code === "string" ? err.code : "";
2293
+ if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "EPIPE" || code === "UND_ERR_SOCKET") {
2294
+ return true;
2295
+ }
2296
+ const message = error instanceof Error ? error.message.toLowerCase() : "";
2297
+ return message.includes("network") || message.includes("fetch failed") || message.includes("socket hang up") || message.includes("terminated");
2298
+ }
2299
+ function isTransientError(error) {
2300
+ const status = statusFromError(error);
2301
+ if (status !== void 0) return TRANSIENT_STATUS.has(status);
2302
+ return isNetworkError(error);
2303
+ }
2304
+ var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
2305
+ async function withRetry(fn, options = {}) {
2306
+ const attempts = Math.max(1, options.attempts ?? 3);
2307
+ const baseDelayMs = options.baseDelayMs ?? 1e3;
2308
+ const sleep2 = options.sleep ?? defaultSleep;
2309
+ const random = options.random ?? Math.random;
2310
+ let lastError;
2311
+ for (let attempt = 1; attempt <= attempts; attempt++) {
2312
+ try {
2313
+ return await fn();
2314
+ } catch (error) {
2315
+ lastError = error;
2316
+ if (attempt >= attempts || !isTransientError(error)) throw error;
2317
+ const backoff = baseDelayMs * 2 ** (attempt - 1);
2318
+ const delay = backoff + random() * backoff;
2319
+ await sleep2(delay);
2320
+ }
2321
+ }
2322
+ throw lastError;
2323
+ }
2324
+
2325
+ // src/translate/gemini-batch.ts
2326
+ var STATE_FAMILY_PREFIX = /^(JOB_STATE_|BATCH_STATE_)/;
2327
+ var TERMINAL_STATES = /* @__PURE__ */ new Set([
2328
+ "SUCCEEDED",
2329
+ "FAILED",
2330
+ "CANCELLED",
2331
+ "EXPIRED",
2332
+ "PARTIALLY_SUCCEEDED"
2333
+ ]);
2334
+ var SUCCESSFUL_STATES = /* @__PURE__ */ new Set(["SUCCEEDED", "PARTIALLY_SUCCEEDED"]);
2335
+ function normalizeBatchState(state) {
2336
+ return String(state ?? "UNKNOWN").replace(STATE_FAMILY_PREFIX, "");
2337
+ }
2338
+ function isTerminalBatchState(state) {
2339
+ return TERMINAL_STATES.has(normalizeBatchState(state));
2340
+ }
2341
+ function isSuccessfulBatchState(state) {
2342
+ return SUCCESSFUL_STATES.has(normalizeBatchState(state));
2343
+ }
2344
+ function makeClient(apiKey) {
2345
+ const key = apiKey ?? process.env.GEMINI_API_KEY;
2346
+ if (!key) {
2347
+ throw new Error("GEMINI_API_KEY is required for scribe translate");
2348
+ }
2349
+ return new GoogleGenAI({ apiKey: key });
2350
+ }
2351
+ function textFromBatchResponse(response) {
2352
+ if (typeof response.text === "string") return response.text;
2353
+ const parts = response.candidates?.[0]?.content?.parts ?? [];
2354
+ return parts.filter((part) => !part.thought && typeof part.text === "string").map((part) => part.text).join("");
2355
+ }
2356
+ async function createGeminiBatchJob(input) {
2357
+ const ai = makeClient(input.apiKey);
2358
+ const job = await withRetry(
2359
+ () => ai.batches.create({
2360
+ model: input.model,
2361
+ src: input.requests,
2362
+ config: { displayName: input.displayName ?? `scribe-translate-${Date.now()}` }
2363
+ })
2364
+ );
2365
+ if (!job.name) throw new Error("Gemini batch job was created without a name");
2366
+ return job;
2367
+ }
2368
+ async function getGeminiBatchJob(input) {
2369
+ const ai = makeClient(input.apiKey);
2370
+ return withRetry(() => ai.batches.get({ name: input.name }));
2371
+ }
2372
+
2140
2373
  // src/translate/gemini-client.ts
2141
2374
  init_esm_shims();
2142
2375
 
@@ -2161,6 +2394,12 @@ function normalizeGeminiDisplayName(model) {
2161
2394
  if (alias) return alias[0];
2162
2395
  return name;
2163
2396
  }
2397
+ function resolveThinkingConfig(apiModelId) {
2398
+ const id = stripModelsPrefix(apiModelId).toLowerCase();
2399
+ if (id.startsWith("gemini-3")) return { thinkingLevel: ThinkingLevel.LOW };
2400
+ if (id.includes("2.5")) return { thinkingBudget: 128 };
2401
+ return void 0;
2402
+ }
2164
2403
 
2165
2404
  // src/translate/gemini-client.ts
2166
2405
  var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
@@ -2174,11 +2413,36 @@ function extractJson(text) {
2174
2413
  return trimmed;
2175
2414
  }
2176
2415
  function parseGeminiResponse(text) {
2416
+ let parsed;
2177
2417
  try {
2178
- return JSON.parse(text.trim());
2418
+ parsed = JSON.parse(text.trim());
2179
2419
  } catch {
2180
- return JSON.parse(extractJson(text));
2420
+ parsed = JSON.parse(extractJson(text));
2181
2421
  }
2422
+ if (parsed !== null && typeof parsed === "object" && parsed.frontmatter === void 0) {
2423
+ parsed.frontmatter = {};
2424
+ }
2425
+ return parsed;
2426
+ }
2427
+ function buildGeminiRequestConfig(input) {
2428
+ const thinkingConfig = resolveThinkingConfig(input.apiModelId);
2429
+ return {
2430
+ responseMimeType: "application/json",
2431
+ ...input.responseSchema ? { responseSchema: input.responseSchema } : {},
2432
+ ...thinkingConfig ? { thinkingConfig } : {}
2433
+ };
2434
+ }
2435
+ function usageFromResponse(response) {
2436
+ const meta = response.usageMetadata ?? response.usage_metadata ?? {};
2437
+ const thoughtsTokens = meta.thoughtsTokenCount ?? meta.thoughts_token_count ?? 0;
2438
+ return {
2439
+ inputTokens: meta.promptTokenCount ?? meta.prompt_token_count ?? 0,
2440
+ // candidatesTokenCount does not include thoughts, but thoughts are billed as
2441
+ // output tokens, so fold them in here for accurate cost accounting.
2442
+ outputTokens: (meta.candidatesTokenCount ?? meta.candidates_token_count ?? 0) + thoughtsTokens,
2443
+ thoughtsTokens,
2444
+ totalTokens: meta.totalTokenCount ?? meta.total_token_count ?? 0
2445
+ };
2182
2446
  }
2183
2447
  async function translatePageWithGemini(input) {
2184
2448
  const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
@@ -2190,28 +2454,28 @@ async function translatePageWithGemini(input) {
2190
2454
  );
2191
2455
  const apiModel = resolveGeminiModelId(displayModel);
2192
2456
  const ai = new GoogleGenAI({ apiKey });
2193
- const response = await ai.models.generateContent({
2194
- model: apiModel,
2195
- contents: input.prompt,
2196
- config: {
2197
- responseMimeType: "application/json",
2198
- ...input.responseSchema ? { responseSchema: input.responseSchema } : {}
2199
- }
2200
- });
2457
+ const response = await withRetry(
2458
+ () => ai.models.generateContent({
2459
+ model: apiModel,
2460
+ contents: input.prompt,
2461
+ config: buildGeminiRequestConfig({
2462
+ apiModelId: apiModel,
2463
+ responseSchema: input.responseSchema
2464
+ })
2465
+ })
2466
+ );
2201
2467
  const raw = response.text ?? "";
2202
2468
  const parsed = parseGeminiResponse(raw);
2203
2469
  if (!parsed.frontmatter || typeof parsed.body !== "string") {
2204
2470
  throw new Error("Gemini response missing frontmatter/body");
2205
2471
  }
2206
- const usageMetadata = response.usageMetadata;
2207
- const usage = {
2208
- inputTokens: usageMetadata?.promptTokenCount ?? 0,
2209
- outputTokens: usageMetadata?.candidatesTokenCount ?? 0,
2210
- totalTokens: usageMetadata?.totalTokenCount ?? 0
2211
- };
2212
- return { model: displayModel, raw, parsed, usage };
2472
+ return { model: displayModel, raw, parsed, usage: usageFromResponse(response) };
2213
2473
  }
2214
2474
 
2475
+ // src/translate/translate-core.ts
2476
+ init_esm_shims();
2477
+ init_sqlite();
2478
+
2215
2479
  // src/translate/gemini-pricing.ts
2216
2480
  init_esm_shims();
2217
2481
  var CONTEXT_TIER_TOKENS = 2e5;
@@ -2230,12 +2494,14 @@ function resolveModelPricing(model) {
2230
2494
  const match = Object.entries(GEMINI_PRICING_USD).find(([key]) => id.includes(key));
2231
2495
  return match?.[1];
2232
2496
  }
2233
- function estimateTranslationCostUsd(model, inputTokens, outputTokens) {
2497
+ var BATCH_DISCOUNT = 0.5;
2498
+ function estimateTranslationCostUsd(model, inputTokens, outputTokens, mode = "interactive") {
2234
2499
  const pricing = resolveModelPricing(model);
2235
2500
  if (!pricing) return void 0;
2236
2501
  const inputRate = tierRate(inputTokens, pricing.input);
2237
2502
  const outputRate = tierRate(outputTokens, pricing.output);
2238
- return inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate;
2503
+ const multiplier = mode === "batch" ? BATCH_DISCOUNT : 1;
2504
+ return (inputTokens / 1e6 * inputRate + outputTokens / 1e6 * outputRate) * multiplier;
2239
2505
  }
2240
2506
  function formatTokenCount(tokens) {
2241
2507
  if (tokens >= 1e6) return `${(tokens / 1e6).toFixed(2)}M`;
@@ -2263,6 +2529,7 @@ var LOCALE_NAMES = {
2263
2529
  "zh-CN": "Simplified Chinese",
2264
2530
  "zh-TW": "Traditional Chinese",
2265
2531
  pt: "Portuguese",
2532
+ "pt-BR": "Brazilian Portuguese",
2266
2533
  ja: "Japanese",
2267
2534
  ru: "Russian",
2268
2535
  it: "Italian",
@@ -2270,29 +2537,29 @@ var LOCALE_NAMES = {
2270
2537
  };
2271
2538
  function defaultLocalizationPrompt(localeName, locale) {
2272
2539
  return [
2273
- `Localize the content below into natural ${localeName} (${locale}).`,
2540
+ `Localize the content above into natural ${localeName} (${locale}).`,
2274
2541
  "Do not translate word-for-word.",
2275
2542
  "Preserve the source tone and brand voice.",
2276
2543
  "Write as if a native speaker authored it for the target market."
2277
2544
  ].join(" ");
2278
2545
  }
2546
+ var TASK_FRAMING = "You are localizing the content below into a target language specified at the end of this prompt.";
2547
+ function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
2548
+ if (!hasFrontmatter) {
2549
+ return slugStrategy === "localized" ? "`body` (string, full MDX body), `slug` (string)." : "`body` (string, full MDX body).";
2550
+ }
2551
+ return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
2552
+ }
2279
2553
  function buildPageTranslationPrompt(input) {
2280
2554
  const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
2281
- const prompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2282
- const lines = [
2283
- prompt,
2555
+ const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
2556
+ const prefix = [
2557
+ TASK_FRAMING,
2284
2558
  "",
2285
2559
  ...input.resolved.context ? ["## Context", input.resolved.context, ""] : [],
2286
2560
  ...input.contextLabel ? [`Document: ${input.contextLabel}`, ""] : [],
2287
2561
  "## Rules",
2288
2562
  ...input.resolved.rules.map((rule) => `- ${rule}`),
2289
- ...input.slugStrategy === "localized" ? [
2290
- `- The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug. Transliterate non-Latin ${localeName} into ASCII Latin.`
2291
- ] : [],
2292
- "",
2293
- "## Output format",
2294
- "Return ONLY valid JSON with keys:",
2295
- input.slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).",
2296
2563
  "",
2297
2564
  "## EN translatable frontmatter (JSON)",
2298
2565
  JSON.stringify(input.translatableFrontmatter, null, 2),
@@ -2300,7 +2567,22 @@ function buildPageTranslationPrompt(input) {
2300
2567
  "## EN body (MDX)",
2301
2568
  input.enBody
2302
2569
  ];
2303
- return lines.join("\n");
2570
+ const suffix = [
2571
+ "",
2572
+ "## Target language",
2573
+ localizationPrompt,
2574
+ ...input.slugStrategy === "localized" ? [
2575
+ `The slug MUST be written in ${localeName}, derived from the ${localeName} title and its meaning \u2014 never the English slug.`
2576
+ ] : [],
2577
+ "",
2578
+ "## Output format",
2579
+ "Return ONLY valid JSON with keys:",
2580
+ buildOutputFormatLine(
2581
+ Object.keys(input.translatableFrontmatter).length > 0,
2582
+ input.slugStrategy
2583
+ )
2584
+ ];
2585
+ return [...prefix, ...suffix].join("\n");
2304
2586
  }
2305
2587
 
2306
2588
  // src/translate/response-schema.ts
@@ -2351,9 +2633,8 @@ function buildTranslatableSubschema(schema) {
2351
2633
  function buildGeminiResponseSchema(schema, slugStrategy) {
2352
2634
  try {
2353
2635
  const translatable = buildTranslatableSubschema(schema);
2354
- if (!translatable) return null;
2355
2636
  const responseShape = {
2356
- frontmatter: translatable,
2637
+ ...translatable ? { frontmatter: translatable } : {},
2357
2638
  body: z.string()
2358
2639
  };
2359
2640
  if (slugStrategy === "localized") {
@@ -2434,7 +2715,12 @@ function validateTranslatedFrontmatter(enDoc, localeFrontmatter, typeSchema) {
2434
2715
  return { ok: true, frontmatter };
2435
2716
  }
2436
2717
 
2437
- // src/translate/page-translator.ts
2718
+ // src/translate/translate-core.ts
2719
+ function translationItemKey(item) {
2720
+ const contentType = item.contentType ?? item.content_type ?? "";
2721
+ const enSlug = item.enSlug ?? item.en_slug ?? "";
2722
+ return `${contentType}\0${enSlug}\0${item.locale}`;
2723
+ }
2438
2724
  function summarizeResults(results, durationMs) {
2439
2725
  return results.reduce(
2440
2726
  (totals, result) => {
@@ -2470,17 +2756,6 @@ function formatTranslateError(error) {
2470
2756
  }
2471
2757
  return message.length > 160 ? `${message.slice(0, 157)}\u2026` : message;
2472
2758
  }
2473
- async function runPool(items, concurrency, worker) {
2474
- if (items.length === 0) return;
2475
- let nextIndex = 0;
2476
- const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
2477
- while (nextIndex < items.length) {
2478
- const index = nextIndex++;
2479
- await worker(items[index], index);
2480
- }
2481
- });
2482
- await Promise.all(runners);
2483
- }
2484
2759
  function resolveContextLabel(enDoc, enSlug) {
2485
2760
  const fm = enDoc.frontmatter;
2486
2761
  for (const key of ["title", "name", "h1"]) {
@@ -2489,73 +2764,82 @@ function resolveContextLabel(enDoc, enSlug) {
2489
2764
  }
2490
2765
  return enSlug;
2491
2766
  }
2492
- async function translatePage(config, item, options = {}) {
2493
- const startedAt = Date.now();
2494
- const base = {
2767
+ function baseForItem(item) {
2768
+ return {
2495
2769
  contentType: item.contentType,
2496
2770
  enSlug: item.enSlug,
2497
2771
  locale: item.locale
2498
2772
  };
2499
- try {
2500
- const type = config.types.find((t) => t.id === item.contentType);
2501
- if (!type) throw new Error(`Unknown content type ${item.contentType}`);
2502
- const enDoc = readEnDocument(config, type, item.enSlug);
2503
- if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
2504
- const payload = getTranslatablePayload(enDoc, type);
2505
- const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
2506
- const db = openStore(config, "readonly");
2507
- const existing = getTranslation(db, type.id, item.enSlug, item.locale);
2508
- db.close();
2509
- if (!options.force && existing && existing.en_hash === currentEnHash) {
2510
- return {
2511
- ...base,
2773
+ }
2774
+ function prepareTranslation(config, item, options, startedAt) {
2775
+ const type = config.types.find((t) => t.id === item.contentType);
2776
+ if (!type) throw new Error(`Unknown content type ${item.contentType}`);
2777
+ const enDoc = readEnDocument(config, type, item.enSlug);
2778
+ if (!enDoc) throw new Error(`EN document not found: ${item.enSlug}`);
2779
+ const payload = getTranslatablePayload(enDoc, type);
2780
+ const currentEnHash = computePageEnHash(payload.frontmatter, payload.body);
2781
+ const db = openStore(config, "readonly");
2782
+ const existing = getTranslation(db, type.id, item.enSlug, item.locale);
2783
+ db.close();
2784
+ if (!options.force && existing && existing.en_hash === currentEnHash) {
2785
+ return {
2786
+ status: "done",
2787
+ result: {
2788
+ ...baseForItem(item),
2512
2789
  skipped: true,
2513
2790
  reason: "fresh",
2514
2791
  durationMs: Date.now() - startedAt
2515
- };
2516
- }
2517
- const resolvedTranslate = resolveTranslateConfig(config, type);
2518
- const model = options.model ?? resolvedTranslate.model;
2519
- const prompt = buildPageTranslationPrompt({
2520
- resolved: resolvedTranslate,
2521
- targetLocale: item.locale,
2522
- contextLabel: resolveContextLabel(enDoc, item.enSlug),
2523
- translatableFrontmatter: payload.frontmatter,
2524
- enBody: payload.body,
2525
- slugStrategy: type.slugStrategy
2526
- });
2527
- if (options.dryRun) {
2528
- return {
2529
- ...base,
2530
- skipped: false,
2531
- model,
2532
- durationMs: Date.now() - startedAt
2533
- };
2534
- }
2535
- const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
2536
- const result = await translatePageWithGemini({
2537
- prompt,
2792
+ }
2793
+ };
2794
+ }
2795
+ const resolvedTranslate = resolveTranslateConfig(config, type);
2796
+ const model = options.model ?? resolvedTranslate.model;
2797
+ const prompt = buildPageTranslationPrompt({
2798
+ resolved: resolvedTranslate,
2799
+ targetLocale: item.locale,
2800
+ contextLabel: resolveContextLabel(enDoc, item.enSlug),
2801
+ translatableFrontmatter: payload.frontmatter,
2802
+ enBody: payload.body,
2803
+ slugStrategy: type.slugStrategy
2804
+ });
2805
+ const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
2806
+ return {
2807
+ status: "ready",
2808
+ prepared: {
2809
+ item,
2810
+ type,
2811
+ enDoc,
2812
+ payload,
2813
+ currentEnHash,
2814
+ existingSlug: existing?.slug,
2538
2815
  model,
2816
+ prompt,
2539
2817
  responseSchema: responseSchema ?? void 0
2540
- });
2541
- const rawSlug = type.slugStrategy === "localized" ? result.parsed.slug ?? existing?.slug ?? item.enSlug : item.enSlug;
2818
+ }
2819
+ };
2820
+ }
2821
+ function finalizeTranslation(config, prepared, output, options) {
2822
+ const { item, type, enDoc, payload } = prepared;
2823
+ const base = baseForItem(item);
2824
+ try {
2825
+ const rawSlug = type.slugStrategy === "localized" ? output.parsed.slug ?? prepared.existingSlug ?? item.enSlug : item.enSlug;
2542
2826
  const localeCodes = config.locales.filter((l) => l !== config.defaultLocale);
2543
2827
  const { slug, stripped, matchedCode } = stripLocaleSuffixFromSlug(rawSlug, localeCodes);
2544
2828
  const slugAdjusted = stripped && matchedCode ? { from: rawSlug, to: slug, matchedCode } : void 0;
2545
- const validated = validateTranslatedFrontmatter(enDoc, result.parsed.frontmatter, type.schema);
2829
+ const validated = validateTranslatedFrontmatter(enDoc, output.parsed.frontmatter, type.schema);
2546
2830
  if (!validated.ok) {
2547
2831
  throw new Error(`Translation validation failed: ${validated.error}`);
2548
2832
  }
2549
2833
  const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
2550
- result.parsed.body
2834
+ output.parsed.body
2551
2835
  );
2552
2836
  const writeDb = openStore(config, "readwrite");
2553
- const snapshotId = recordEnSnapshot(
2837
+ const snapshotId = options.snapshotId ?? recordEnSnapshot(
2554
2838
  config,
2555
2839
  {
2556
2840
  contentType: type.id,
2557
2841
  enSlug: item.enSlug,
2558
- enHash: currentEnHash,
2842
+ enHash: prepared.currentEnHash,
2559
2843
  frontmatter: payload.frontmatter,
2560
2844
  body: payload.body
2561
2845
  },
@@ -2568,24 +2852,25 @@ async function translatePage(config, item, options = {}) {
2568
2852
  slug,
2569
2853
  frontmatter: validated.frontmatter,
2570
2854
  body: translatedBody,
2571
- enHash: currentEnHash,
2855
+ enHash: prepared.currentEnHash,
2572
2856
  translatedAt: (/* @__PURE__ */ new Date()).toISOString(),
2573
- model: result.model,
2857
+ model: output.model,
2574
2858
  snapshotId
2575
2859
  });
2576
2860
  writeDb.close();
2577
2861
  const estimatedCostUsd = estimateTranslationCostUsd(
2578
- normalizeGeminiDisplayName(result.model),
2579
- result.usage.inputTokens,
2580
- result.usage.outputTokens
2862
+ normalizeGeminiDisplayName(output.model),
2863
+ output.usage.inputTokens,
2864
+ output.usage.outputTokens,
2865
+ options.costMode
2581
2866
  );
2582
2867
  return {
2583
2868
  ...base,
2584
2869
  skipped: false,
2585
- model: result.model,
2586
- usage: result.usage,
2870
+ model: output.model,
2871
+ usage: output.usage,
2587
2872
  estimatedCostUsd,
2588
- durationMs: Date.now() - startedAt,
2873
+ durationMs: Date.now() - options.startedAt,
2589
2874
  slugAdjusted,
2590
2875
  mdxAdjusted: mdxAdjusted || void 0
2591
2876
  };
@@ -2595,33 +2880,559 @@ async function translatePage(config, item, options = {}) {
2595
2880
  skipped: false,
2596
2881
  failed: true,
2597
2882
  error: formatTranslateError(error),
2883
+ durationMs: Date.now() - options.startedAt
2884
+ };
2885
+ }
2886
+ }
2887
+ function displayModelFor(prepared) {
2888
+ return normalizeGeminiDisplayName(
2889
+ prepared.model ?? process.env.PROSE_GEMINI_MODEL ?? DEFAULT_GEMINI_MODEL
2890
+ );
2891
+ }
2892
+
2893
+ // src/translate/batch-worklist.ts
2894
+ var MAX_REQUESTS_PER_JOB = 100;
2895
+ var MAX_PROMPT_BYTES_PER_JOB = 15 * 1024 * 1024;
2896
+ var INITIAL_POLL_MS = 5e3;
2897
+ var MAX_POLL_MS = 3e4;
2898
+ var POLL_BACKOFF_FACTOR = 1.5;
2899
+ function sleep(ms) {
2900
+ return new Promise((resolve) => setTimeout(resolve, ms));
2901
+ }
2902
+ function planBatchJobs(entries) {
2903
+ const byModel = /* @__PURE__ */ new Map();
2904
+ for (const entry of entries) {
2905
+ const group = byModel.get(entry.apiModel);
2906
+ if (group) group.push(entry);
2907
+ else byModel.set(entry.apiModel, [entry]);
2908
+ }
2909
+ const plans = [];
2910
+ for (const [apiModel, group] of byModel) {
2911
+ let current = [];
2912
+ let currentBytes = 0;
2913
+ for (const entry of group) {
2914
+ const size = Buffer.byteLength(entry.prompt, "utf8");
2915
+ if (current.length > 0 && (current.length >= MAX_REQUESTS_PER_JOB || currentBytes + size > MAX_PROMPT_BYTES_PER_JOB)) {
2916
+ plans.push({ apiModel, entries: current });
2917
+ current = [];
2918
+ currentBytes = 0;
2919
+ }
2920
+ current.push(entry);
2921
+ currentBytes += size;
2922
+ }
2923
+ if (current.length > 0) plans.push({ apiModel, entries: current });
2924
+ }
2925
+ return plans;
2926
+ }
2927
+ function readPendingBatchWork(config) {
2928
+ const db = openStore(config, "readwrite");
2929
+ const jobs = listPendingBatchJobs(db);
2930
+ const pendingItems = listPendingBatchItems(db);
2931
+ db.close();
2932
+ const inFlightKeys = new Set(pendingItems.map((item) => translationItemKey(item)));
2933
+ return { jobs, inFlightKeys, pendingItems };
2934
+ }
2935
+ async function submitBatchJobPlan(config, input) {
2936
+ const { plan, displayModel, jobIndex, jobCount } = input;
2937
+ const job = await createGeminiBatchJob({
2938
+ model: plan.apiModel,
2939
+ requests: plan.entries.map(({ prepared }) => ({
2940
+ contents: prepared.prompt,
2941
+ config: buildGeminiRequestConfig({
2942
+ apiModelId: plan.apiModel,
2943
+ responseSchema: prepared.responseSchema
2944
+ })
2945
+ })),
2946
+ displayName: `scribe-translate-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${jobIndex + 1}`
2947
+ });
2948
+ const createdAt = (/* @__PURE__ */ new Date()).toISOString();
2949
+ const db = openStore(config, "readwrite");
2950
+ const jobId = insertBatchJob(db, {
2951
+ jobName: job.name,
2952
+ model: plan.apiModel,
2953
+ displayModel,
2954
+ state: String(job.state ?? "JOB_STATE_PENDING"),
2955
+ createdAt
2956
+ });
2957
+ insertBatchItems(
2958
+ db,
2959
+ jobId,
2960
+ plan.entries.map(({ prepared }, requestIndex) => ({
2961
+ requestIndex,
2962
+ contentType: prepared.item.contentType,
2963
+ enSlug: prepared.item.enSlug,
2964
+ locale: prepared.item.locale,
2965
+ enHash: prepared.currentEnHash,
2966
+ // Snapshot the EN source now so ingestion after a resume does not depend
2967
+ // on the EN files still matching (or existing) on disk.
2968
+ snapshotId: recordEnSnapshot(
2969
+ config,
2970
+ {
2971
+ contentType: prepared.item.contentType,
2972
+ enSlug: prepared.item.enSlug,
2973
+ enHash: prepared.currentEnHash,
2974
+ frontmatter: prepared.payload.frontmatter,
2975
+ body: prepared.payload.body
2976
+ },
2977
+ db
2978
+ )
2979
+ }))
2980
+ );
2981
+ const row = {
2982
+ id: jobId,
2983
+ job_name: job.name,
2984
+ model: plan.apiModel,
2985
+ display_model: displayModel,
2986
+ created_at: createdAt,
2987
+ state: String(job.state ?? "JOB_STATE_PENDING"),
2988
+ completed_at: null
2989
+ };
2990
+ db.close();
2991
+ input.onProgress?.({
2992
+ type: "batch-submitted",
2993
+ name: job.name,
2994
+ count: plan.entries.length,
2995
+ model: displayModel,
2996
+ jobIndex,
2997
+ jobCount,
2998
+ createdAt
2999
+ });
3000
+ return row;
3001
+ }
3002
+ function buildIngestContext(config, db, jobRow, itemRow) {
3003
+ const type = config.types.find((t) => t.id === itemRow.content_type);
3004
+ if (!type) throw new Error(`Unknown content type ${itemRow.content_type}`);
3005
+ let enDoc = readEnDocument(config, type, itemRow.en_slug);
3006
+ if (!enDoc) {
3007
+ const snapshot = getEnSnapshot(db, itemRow.snapshot_id);
3008
+ if (!snapshot) {
3009
+ throw new Error(
3010
+ `EN document and snapshot #${itemRow.snapshot_id} not found for ${itemRow.en_slug}`
3011
+ );
3012
+ }
3013
+ enDoc = {
3014
+ slug: itemRow.en_slug,
3015
+ enSlug: itemRow.en_slug,
3016
+ locale: config.defaultLocale,
3017
+ noindex: false,
3018
+ frontmatter: JSON.parse(snapshot.frontmatter_json),
3019
+ content: snapshot.body
3020
+ };
3021
+ }
3022
+ const item = {
3023
+ contentType: itemRow.content_type,
3024
+ enSlug: itemRow.en_slug,
3025
+ locale: itemRow.locale,
3026
+ reason: "missing",
3027
+ currentEnHash: itemRow.en_hash
3028
+ };
3029
+ return {
3030
+ item,
3031
+ type,
3032
+ enDoc,
3033
+ // Unused on ingest: finalizeTranslation receives the pre-recorded snapshotId.
3034
+ payload: { frontmatter: enDoc.frontmatter, body: enDoc.content },
3035
+ currentEnHash: itemRow.en_hash,
3036
+ existingSlug: getTranslation(db, type.id, itemRow.en_slug, itemRow.locale)?.slug,
3037
+ model: jobRow.display_model,
3038
+ prompt: "",
3039
+ responseSchema: void 0
3040
+ };
3041
+ }
3042
+ function ingestBatchJob(config, jobRow, batchJob, onResult) {
3043
+ const state = String(batchJob.state ?? "UNKNOWN");
3044
+ const db = openStore(config, "readwrite");
3045
+ if (!claimBatchJobCompletion(db, jobRow.id, state, (/* @__PURE__ */ new Date()).toISOString())) {
3046
+ db.close();
3047
+ return null;
3048
+ }
3049
+ const pendingItems = listBatchItems(db, jobRow.id).filter((item) => item.status === "pending");
3050
+ const results = [];
3051
+ const failItem = (itemRow, message, startedAt) => {
3052
+ const result = {
3053
+ contentType: itemRow.content_type,
3054
+ enSlug: itemRow.en_slug,
3055
+ locale: itemRow.locale,
3056
+ skipped: false,
3057
+ failed: true,
3058
+ error: formatTranslateError(new Error(message)),
2598
3059
  durationMs: Date.now() - startedAt
2599
3060
  };
3061
+ updateBatchItemStatus(db, jobRow.id, itemRow.request_index, "failed", result.error);
3062
+ results.push(result);
3063
+ onResult?.(result);
3064
+ };
3065
+ if (isSuccessfulBatchState(state)) {
3066
+ const responses = batchJob.dest?.inlinedResponses ?? [];
3067
+ for (const itemRow of pendingItems) {
3068
+ const startedAt = Date.now();
3069
+ const inlined = responses[itemRow.request_index];
3070
+ if (!inlined || inlined.error || !inlined.response) {
3071
+ failItem(
3072
+ itemRow,
3073
+ inlined?.error?.message ?? (inlined ? "Batch response missing content" : "Batch response missing"),
3074
+ startedAt
3075
+ );
3076
+ continue;
3077
+ }
3078
+ let result;
3079
+ try {
3080
+ const response = inlined.response;
3081
+ const raw = textFromBatchResponse(response);
3082
+ const parsed = parseGeminiResponse(raw);
3083
+ if (!parsed.frontmatter || typeof parsed.body !== "string") {
3084
+ throw new Error("Gemini response missing frontmatter/body");
3085
+ }
3086
+ const prepared = buildIngestContext(config, db, jobRow, itemRow);
3087
+ result = finalizeTranslation(
3088
+ config,
3089
+ prepared,
3090
+ { model: jobRow.display_model, parsed, usage: usageFromResponse(response) },
3091
+ { costMode: "batch", startedAt, snapshotId: itemRow.snapshot_id }
3092
+ );
3093
+ } catch (error) {
3094
+ failItem(itemRow, error instanceof Error ? error.message : String(error), startedAt);
3095
+ continue;
3096
+ }
3097
+ updateBatchItemStatus(
3098
+ db,
3099
+ jobRow.id,
3100
+ itemRow.request_index,
3101
+ result.failed ? "failed" : "done",
3102
+ result.error
3103
+ );
3104
+ results.push(result);
3105
+ onResult?.(result);
3106
+ }
3107
+ } else {
3108
+ const message = batchJob.error?.message ?? `Batch job ended in state ${normalizeBatchState(state)}`;
3109
+ const startedAt = Date.now();
3110
+ for (const itemRow of pendingItems) {
3111
+ failItem(itemRow, message, startedAt);
3112
+ }
3113
+ }
3114
+ db.close();
3115
+ return results;
3116
+ }
3117
+ async function pollAndIngestBatchJobs(config, jobs, options = {}) {
3118
+ const jobCount = options.jobCount ?? jobs.length;
3119
+ const active = jobs.map((row, jobIndex) => ({ row, jobIndex }));
3120
+ const loopStartedAt = Date.now();
3121
+ let delay = INITIAL_POLL_MS;
3122
+ let firstRound = true;
3123
+ const elapsedFor = (row) => {
3124
+ const createdAtMs = Date.parse(row.created_at);
3125
+ return Number.isNaN(createdAtMs) ? Date.now() - loopStartedAt : Date.now() - createdAtMs;
3126
+ };
3127
+ while (active.length > 0) {
3128
+ if (!firstRound) {
3129
+ await sleep(delay);
3130
+ delay = Math.min(delay * POLL_BACKOFF_FACTOR, MAX_POLL_MS);
3131
+ }
3132
+ firstRound = false;
3133
+ for (const tracked of [...active]) {
3134
+ const batchJob = await getGeminiBatchJob({ name: tracked.row.job_name });
3135
+ const state = normalizeBatchState(batchJob.state);
3136
+ options.onProgress?.({
3137
+ type: "batch-polling",
3138
+ name: tracked.row.job_name,
3139
+ state,
3140
+ elapsedMs: elapsedFor(tracked.row),
3141
+ jobIndex: tracked.jobIndex,
3142
+ jobCount
3143
+ });
3144
+ if (isTerminalBatchState(batchJob.state)) {
3145
+ active.splice(active.indexOf(tracked), 1);
3146
+ const results = ingestBatchJob(
3147
+ config,
3148
+ tracked.row,
3149
+ batchJob,
3150
+ options.onResult
3151
+ );
3152
+ if (results === null) {
3153
+ options.onProgress?.({
3154
+ type: "batch-done",
3155
+ name: tracked.row.job_name,
3156
+ state,
3157
+ model: tracked.row.display_model,
3158
+ count: 0,
3159
+ jobIndex: tracked.jobIndex,
3160
+ jobCount,
3161
+ translated: 0,
3162
+ failed: 0,
3163
+ inputTokens: 0,
3164
+ outputTokens: 0,
3165
+ estimatedCostUsd: 0,
3166
+ elapsedMs: elapsedFor(tracked.row),
3167
+ alreadyIngested: true
3168
+ });
3169
+ continue;
3170
+ }
3171
+ options.onProgress?.({
3172
+ type: "batch-done",
3173
+ name: tracked.row.job_name,
3174
+ state,
3175
+ model: tracked.row.display_model,
3176
+ count: results.length,
3177
+ jobIndex: tracked.jobIndex,
3178
+ jobCount,
3179
+ translated: results.filter((r) => !r.failed && !r.skipped).length,
3180
+ failed: results.filter((r) => r.failed).length,
3181
+ inputTokens: results.reduce((sum, r) => sum + (r.usage?.inputTokens ?? 0), 0),
3182
+ outputTokens: results.reduce((sum, r) => sum + (r.usage?.outputTokens ?? 0), 0),
3183
+ estimatedCostUsd: results.reduce((sum, r) => sum + (r.estimatedCostUsd ?? 0), 0),
3184
+ elapsedMs: elapsedFor(tracked.row)
3185
+ });
3186
+ }
3187
+ }
2600
3188
  }
2601
3189
  }
3190
+ function failedResultForItem(item, error, startedAt) {
3191
+ return {
3192
+ ...baseForItem(item),
3193
+ skipped: false,
3194
+ failed: true,
3195
+ error: formatTranslateError(error),
3196
+ durationMs: Date.now() - startedAt
3197
+ };
3198
+ }
3199
+
3200
+ // src/translate/page-translator.ts
3201
+ async function runPool(items, concurrency, worker) {
3202
+ if (items.length === 0) return;
3203
+ let nextIndex = 0;
3204
+ const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
3205
+ while (nextIndex < items.length) {
3206
+ const index = nextIndex++;
3207
+ await worker(items[index], index);
3208
+ }
3209
+ });
3210
+ await Promise.all(runners);
3211
+ }
2602
3212
  function labelForItem(item) {
2603
3213
  return `${item.contentType}/${item.enSlug}@${item.locale}`;
2604
3214
  }
3215
+ async function translatePage(config, item, options = {}) {
3216
+ const startedAt = Date.now();
3217
+ let outcome;
3218
+ try {
3219
+ outcome = prepareTranslation(config, item, options, startedAt);
3220
+ } catch (error) {
3221
+ return failedResultForItem(item, error, startedAt);
3222
+ }
3223
+ if (outcome.status === "done") return outcome.result;
3224
+ const prepared = outcome.prepared;
3225
+ if (options.dryRun) {
3226
+ return {
3227
+ ...baseForItem(item),
3228
+ skipped: false,
3229
+ model: prepared.model,
3230
+ durationMs: Date.now() - startedAt
3231
+ };
3232
+ }
3233
+ try {
3234
+ const result = await translatePageWithGemini({
3235
+ prompt: prepared.prompt,
3236
+ model: prepared.model,
3237
+ responseSchema: prepared.responseSchema
3238
+ });
3239
+ return finalizeTranslation(
3240
+ config,
3241
+ prepared,
3242
+ { model: result.model, parsed: result.parsed, usage: result.usage },
3243
+ { costMode: "interactive", startedAt }
3244
+ );
3245
+ } catch (error) {
3246
+ return failedResultForItem(item, error, startedAt);
3247
+ }
3248
+ }
3249
+ function createResultCollector(items, onProgress) {
3250
+ const slotByKey = /* @__PURE__ */ new Map();
3251
+ items.forEach((item, index) => slotByKey.set(translationItemKey(item), index));
3252
+ const slots = new Array(items.length);
3253
+ const extras = [];
3254
+ return {
3255
+ emit(result) {
3256
+ const slot = slotByKey.get(translationItemKey(result));
3257
+ if (slot !== void 0 && slots[slot] === void 0) slots[slot] = result;
3258
+ else extras.push(result);
3259
+ onProgress?.({ type: "item-done", result });
3260
+ },
3261
+ assemble() {
3262
+ return [
3263
+ ...slots.filter((result) => result !== void 0),
3264
+ ...extras
3265
+ ];
3266
+ }
3267
+ };
3268
+ }
2605
3269
  async function translateWorklist(config, items, options = {}) {
3270
+ const mode = options.mode ?? "batch";
2606
3271
  const concurrency = Math.max(1, options.concurrency ?? 3);
2607
3272
  const startedAt = Date.now();
2608
- const results = new Array(items.length);
2609
- const active = /* @__PURE__ */ new Set();
3273
+ if (options.dryRun) {
3274
+ options.onProgress?.({
3275
+ type: "start",
3276
+ total: items.length,
3277
+ concurrency,
3278
+ dryRun: true,
3279
+ model: options.model,
3280
+ mode
3281
+ });
3282
+ const results2 = items.map((item) => {
3283
+ const itemStartedAt = Date.now();
3284
+ try {
3285
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3286
+ return outcome.status === "done" ? outcome.result : {
3287
+ ...baseForItem(item),
3288
+ skipped: false,
3289
+ model: outcome.prepared.model,
3290
+ durationMs: Date.now() - itemStartedAt
3291
+ };
3292
+ } catch (error) {
3293
+ return failedResultForItem(item, error, itemStartedAt);
3294
+ }
3295
+ });
3296
+ for (const result of results2) options.onProgress?.({ type: "item-done", result });
3297
+ const totals2 = summarizeResults(results2, Date.now() - startedAt);
3298
+ options.onProgress?.({ type: "done", results: results2, totals: totals2 });
3299
+ return results2;
3300
+ }
3301
+ const pending = readPendingBatchWork(config);
3302
+ const inputKeys = new Set(items.map((item) => translationItemKey(item)));
3303
+ const resumedExtraCount = pending.pendingItems.filter(
3304
+ (item) => !inputKeys.has(translationItemKey(item))
3305
+ ).length;
2610
3306
  options.onProgress?.({
2611
3307
  type: "start",
2612
- total: items.length,
3308
+ total: items.length + resumedExtraCount,
2613
3309
  concurrency,
2614
- dryRun: Boolean(options.dryRun),
2615
- model: options.model
3310
+ dryRun: false,
3311
+ model: options.model,
3312
+ mode
2616
3313
  });
2617
- await runPool(items, concurrency, async (item, index) => {
2618
- const label = labelForItem(item);
2619
- active.add(label);
2620
- options.onProgress?.({ type: "item-start", item, active: [...active] });
2621
- const result = await translatePage(config, item, options);
2622
- results[index] = result;
2623
- active.delete(label);
2624
- options.onProgress?.({ type: "item-done", result });
3314
+ const collector = createResultCollector(items, options.onProgress);
3315
+ const workItems = items.filter((item) => !pending.inFlightKeys.has(translationItemKey(item)));
3316
+ if (mode === "direct") {
3317
+ const pendingCounts = countPendingItems(config, pending.jobs);
3318
+ emitResumedJobs(pending.jobs, pendingCounts, pending.jobs.length, options.onProgress);
3319
+ const active = /* @__PURE__ */ new Set();
3320
+ await Promise.all([
3321
+ runPool(workItems, concurrency, async (item) => {
3322
+ const label = labelForItem(item);
3323
+ active.add(label);
3324
+ options.onProgress?.({ type: "item-start", item, active: [...active] });
3325
+ const result = await translatePage(config, item, options);
3326
+ active.delete(label);
3327
+ collector.emit(result);
3328
+ }),
3329
+ pending.jobs.length > 0 ? pollAndIngestBatchJobs(config, pending.jobs, {
3330
+ onProgress: options.onProgress,
3331
+ onResult: collector.emit
3332
+ }) : Promise.resolve()
3333
+ ]);
3334
+ } else {
3335
+ const entries = [];
3336
+ for (const item of workItems) {
3337
+ const itemStartedAt = Date.now();
3338
+ try {
3339
+ const outcome = prepareTranslation(config, item, options, itemStartedAt);
3340
+ if (outcome.status === "done") {
3341
+ collector.emit(outcome.result);
3342
+ } else {
3343
+ entries.push({
3344
+ apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
3345
+ prompt: outcome.prepared.prompt,
3346
+ prepared: outcome.prepared
3347
+ });
3348
+ }
3349
+ } catch (error) {
3350
+ collector.emit(failedResultForItem(item, error, itemStartedAt));
3351
+ }
3352
+ }
3353
+ const plans = planBatchJobs(entries);
3354
+ const jobCount = pending.jobs.length + plans.length;
3355
+ const pendingCounts = countPendingItems(config, pending.jobs);
3356
+ emitResumedJobs(pending.jobs, pendingCounts, jobCount, options.onProgress);
3357
+ const submitted = await Promise.all(
3358
+ plans.map(async (plan, planIndex) => {
3359
+ try {
3360
+ return await submitBatchJobPlan(config, {
3361
+ plan,
3362
+ displayModel: normalizeGeminiDisplayName(plan.apiModel),
3363
+ jobIndex: pending.jobs.length + planIndex,
3364
+ jobCount,
3365
+ onProgress: options.onProgress
3366
+ });
3367
+ } catch (error) {
3368
+ for (const entry of plan.entries) {
3369
+ collector.emit(failedResultForItem(entry.prepared.item, error, startedAt));
3370
+ }
3371
+ return void 0;
3372
+ }
3373
+ })
3374
+ );
3375
+ const jobsToPoll = [
3376
+ ...pending.jobs,
3377
+ ...submitted.filter((row) => row !== void 0)
3378
+ ];
3379
+ if (jobsToPoll.length > 0) {
3380
+ await pollAndIngestBatchJobs(config, jobsToPoll, {
3381
+ jobCount,
3382
+ onProgress: options.onProgress,
3383
+ onResult: collector.emit
3384
+ });
3385
+ }
3386
+ }
3387
+ const results = collector.assemble();
3388
+ const totals = summarizeResults(results, Date.now() - startedAt);
3389
+ options.onProgress?.({ type: "done", results, totals });
3390
+ return results;
3391
+ }
3392
+ function countPendingItems(config, jobs) {
3393
+ if (jobs.length === 0) return /* @__PURE__ */ new Map();
3394
+ const db = openStore(config, "readonly");
3395
+ const counts = /* @__PURE__ */ new Map();
3396
+ for (const job of jobs) {
3397
+ counts.set(job.id, listBatchItems(db, job.id).filter((i) => i.status === "pending").length);
3398
+ }
3399
+ db.close();
3400
+ return counts;
3401
+ }
3402
+ function emitResumedJobs(jobs, counts, jobCount, onProgress) {
3403
+ jobs.forEach((job, jobIndex) => {
3404
+ onProgress?.({
3405
+ type: "batch-submitted",
3406
+ name: job.job_name,
3407
+ count: counts.get(job.id) ?? 0,
3408
+ model: job.display_model,
3409
+ jobIndex,
3410
+ jobCount,
3411
+ resumed: true,
3412
+ createdAt: job.created_at
3413
+ });
3414
+ });
3415
+ }
3416
+ async function resumeTranslationJobs(config, options = {}) {
3417
+ const startedAt = Date.now();
3418
+ const pending = readPendingBatchWork(config);
3419
+ if (pending.jobs.length === 0) return null;
3420
+ options.onProgress?.({
3421
+ type: "start",
3422
+ total: pending.pendingItems.length,
3423
+ concurrency: 1,
3424
+ dryRun: false,
3425
+ mode: "batch"
3426
+ });
3427
+ const counts = countPendingItems(config, pending.jobs);
3428
+ emitResumedJobs(pending.jobs, counts, pending.jobs.length, options.onProgress);
3429
+ const results = [];
3430
+ await pollAndIngestBatchJobs(config, pending.jobs, {
3431
+ onProgress: options.onProgress,
3432
+ onResult: (result) => {
3433
+ results.push(result);
3434
+ options.onProgress?.({ type: "item-done", result });
3435
+ }
2625
3436
  });
2626
3437
  const totals = summarizeResults(results, Date.now() - startedAt);
2627
3438
  options.onProgress?.({ type: "done", results, totals });
@@ -3239,35 +4050,43 @@ async function promptContentType(config) {
3239
4050
  })
3240
4051
  );
3241
4052
  }
4053
+ var ALL_LOCALES = /* @__PURE__ */ Symbol("all-locales");
4054
+ var PICK_LOCALES = /* @__PURE__ */ Symbol("pick-locales");
4055
+ async function promptManualLocales(config) {
4056
+ const locales = nonDefaultLocales(config);
4057
+ if (locales.length <= 1) return {};
4058
+ const picked = await runPrompt(
4059
+ checkbox({
4060
+ message: "Locales to translate",
4061
+ choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
4062
+ validate: (values) => values.length > 0 || "Pick at least one locale"
4063
+ })
4064
+ );
4065
+ return picked.length === locales.length ? {} : { locale: picked };
4066
+ }
3242
4067
  async function promptLocalePreset(config) {
3243
4068
  const presets = Object.entries(config.localePresets ?? {}).filter(
3244
4069
  (entry) => Array.isArray(entry[1])
3245
4070
  );
3246
4071
  if (presets.length === 0) {
3247
- const locales = nonDefaultLocales(config);
3248
- if (locales.length <= 1) return {};
3249
- const picked = await runPrompt(
3250
- checkbox({
3251
- message: "Locales to translate",
3252
- choices: locales.map((locale) => ({ name: locale, value: locale, checked: true })),
3253
- validate: (values) => values.length > 0 || "Pick at least one locale"
3254
- })
3255
- );
3256
- return picked.length === locales.length ? {} : { locale: picked };
4072
+ return promptManualLocales(config);
3257
4073
  }
3258
4074
  const choice = await runPrompt(
3259
4075
  select({
3260
4076
  message: "Locale preset",
3261
4077
  choices: [
3262
- { name: "All locales", value: void 0 },
4078
+ { name: "All locales", value: ALL_LOCALES },
3263
4079
  ...presets.map(([name, locales]) => ({
3264
4080
  name: `${name} (${locales.join(", ")})`,
3265
4081
  value: name
3266
- }))
4082
+ })),
4083
+ { name: "Choose locales manually\u2026", value: PICK_LOCALES }
3267
4084
  ]
3268
4085
  })
3269
4086
  );
3270
- return choice ? { preset: choice } : {};
4087
+ if (choice === PICK_LOCALES) return promptManualLocales(config);
4088
+ if (choice === ALL_LOCALES) return {};
4089
+ return { preset: choice };
3271
4090
  }
3272
4091
  async function promptStrategy() {
3273
4092
  return runPrompt(
@@ -3281,12 +4100,25 @@ async function promptStrategy() {
3281
4100
  })
3282
4101
  );
3283
4102
  }
4103
+ async function promptMode() {
4104
+ return runPrompt(
4105
+ select({
4106
+ message: "Translation mode",
4107
+ choices: [
4108
+ { name: "Batch \u2014 50% cheaper, async, resumable (recommended)", value: "batch" },
4109
+ { name: "Direct \u2014 immediate results, full price", value: "direct" }
4110
+ ],
4111
+ default: "batch"
4112
+ })
4113
+ );
4114
+ }
3284
4115
  async function promptTranslateSelection(config, flags) {
3285
4116
  const selection = {
3286
4117
  contentType: flags.type,
3287
4118
  preset: flags.preset,
3288
4119
  locale: flags.locale,
3289
- strategy: flags.strategy
4120
+ strategy: flags.strategy,
4121
+ mode: flags.mode
3290
4122
  };
3291
4123
  if (!isInteractive()) return selection;
3292
4124
  if (!selection.contentType) {
@@ -3298,6 +4130,9 @@ async function promptTranslateSelection(config, flags) {
3298
4130
  if (!selection.strategy) {
3299
4131
  selection.strategy = await promptStrategy();
3300
4132
  }
4133
+ if (!selection.mode) {
4134
+ selection.mode = await promptMode();
4135
+ }
3301
4136
  return selection;
3302
4137
  }
3303
4138
 
@@ -3319,6 +4154,34 @@ function progressBar(ratio, width = 28) {
3319
4154
  function labelForResult(result) {
3320
4155
  return `${result.contentType}/${result.enSlug}@${result.locale}`;
3321
4156
  }
4157
+ function shortJobId(name) {
4158
+ const segments = name.split("/");
4159
+ return segments[segments.length - 1] || name;
4160
+ }
4161
+ function formatElapsed(elapsedMs) {
4162
+ return `${Math.round(elapsedMs / 1e3)}s`;
4163
+ }
4164
+ function batchDoneParts(event) {
4165
+ if (event.alreadyIngested) {
4166
+ return [
4167
+ `job ${shortJobId(event.name)}`,
4168
+ event.model ?? "?",
4169
+ "already ingested by another scribe run (results saved there)",
4170
+ formatElapsed(event.elapsedMs)
4171
+ ].join(" \xB7 ");
4172
+ }
4173
+ const parts = [
4174
+ `job ${shortJobId(event.name)}`,
4175
+ event.model ?? "?",
4176
+ `${event.count} request${event.count === 1 ? "" : "s"}`,
4177
+ `${event.translated} translated`
4178
+ ];
4179
+ if (event.failed > 0) parts.push(`${event.failed} failed`);
4180
+ parts.push(`${formatTokenCount(event.inputTokens)} in / ${formatTokenCount(event.outputTokens)} out`);
4181
+ parts.push(`${formatUsd(event.estimatedCostUsd)} est.`);
4182
+ parts.push(formatElapsed(event.elapsedMs));
4183
+ return parts.join(" \xB7 ");
4184
+ }
3322
4185
  function statusForResult(result, dryRun) {
3323
4186
  if (result.failed) return red("failed");
3324
4187
  if (result.skipped) return dim("skipped");
@@ -3364,6 +4227,25 @@ function createTranslateProgressReporter(options = {}) {
3364
4227
  if (!enabled) {
3365
4228
  return {
3366
4229
  onEvent(event) {
4230
+ if (event.type === "batch-submitted") {
4231
+ const verb = event.resumed ? "Resuming" : "Submitted";
4232
+ console.log(
4233
+ `${verb} batch job ${event.jobIndex + 1}/${event.jobCount} with ${event.count} request${event.count === 1 ? "" : "s"}${event.model ? ` (${event.model})` : ""}: ${event.name}`
4234
+ );
4235
+ return;
4236
+ }
4237
+ if (event.type === "batch-polling") {
4238
+ console.log(
4239
+ `batch ${event.jobIndex + 1}/${event.jobCount} ${shortJobId(event.name)}: ${event.state.replace(/^JOB_STATE_/, "")} (${formatElapsed(event.elapsedMs)} elapsed)`
4240
+ );
4241
+ return;
4242
+ }
4243
+ if (event.type === "batch-done") {
4244
+ const line = `batch ${event.jobIndex + 1}/${event.jobCount} ${event.state} \xB7 ${batchDoneParts(event)}`;
4245
+ if (event.failed > 0 && event.translated === 0) console.error(line);
4246
+ else console.log(line);
4247
+ return;
4248
+ }
3367
4249
  if (event.type === "item-done") {
3368
4250
  const label = labelForResult(event.result);
3369
4251
  const status = statusForResult(event.result, dryRun);
@@ -3394,6 +4276,9 @@ function createTranslateProgressReporter(options = {}) {
3394
4276
  let outputTokens = 0;
3395
4277
  let estimatedCostUsd = 0;
3396
4278
  let active = [];
4279
+ let mode;
4280
+ const batchJobs = /* @__PURE__ */ new Map();
4281
+ let batchElapsedMs = 0;
3397
4282
  const recent = [];
3398
4283
  let renderedLines = 0;
3399
4284
  let cursorHidden = false;
@@ -3409,6 +4294,15 @@ function createTranslateProgressReporter(options = {}) {
3409
4294
  cursorHidden = false;
3410
4295
  }
3411
4296
  }
4297
+ function printPersistent(lines) {
4298
+ if (renderedLines > 0) {
4299
+ process.stdout.write(`\x1B[${renderedLines}A`);
4300
+ }
4301
+ for (const line of lines) {
4302
+ process.stdout.write("\x1B[2K" + line + "\n");
4303
+ }
4304
+ renderedLines = 0;
4305
+ }
3412
4306
  function render() {
3413
4307
  hideCursor();
3414
4308
  if (renderedLines > 0) {
@@ -3416,12 +4310,34 @@ function createTranslateProgressReporter(options = {}) {
3416
4310
  }
3417
4311
  const lines = [];
3418
4312
  const title = dryRun ? "Dry run" : "Translating";
3419
- lines.push(cyan(`${title} ${done}/${total}`) + dim(` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : ""));
4313
+ const isBatch = mode === "batch" || batchJobs.size > 0;
4314
+ lines.push(
4315
+ cyan(`${title} ${done}/${total}`) + dim(isBatch ? " \xB7 batch" : ` \xB7 ${concurrency} parallel`) + (model ? dim(` \xB7 ${model}`) : "")
4316
+ );
3420
4317
  lines.push(progressBar(total === 0 ? 0 : done / total) + dim(` ${Math.round(total === 0 ? 0 : done / total * 100)}%`));
3421
4318
  lines.push(
3422
4319
  dim("Tokens ") + `${formatTokenCount(inputTokens)} in \xB7 ${formatTokenCount(outputTokens)} out` + dim(" \xB7 Cost ") + formatUsd(estimatedCostUsd) + dim(" est.")
3423
4320
  );
3424
- if (active.length > 0) {
4321
+ if (batchJobs.size > 0 && done < total) {
4322
+ const terminal = /^(SUCCEEDED|FAILED|CANCELLED|EXPIRED|PARTIALLY_SUCCEEDED)$/;
4323
+ let running = 0;
4324
+ let finished = 0;
4325
+ let requests = 0;
4326
+ for (const job of batchJobs.values()) {
4327
+ if (terminal.test(job.state)) finished += 1;
4328
+ else running += 1;
4329
+ requests += job.count;
4330
+ }
4331
+ lines.push(
4332
+ dim("Batches ") + `${running} running \xB7 ${finished} done` + dim(` \xB7 ${requests} request${requests === 1 ? "" : "s"} \xB7 ${Math.round(batchElapsedMs / 1e3)}s elapsed`)
4333
+ );
4334
+ for (const [name, job] of batchJobs) {
4335
+ if (terminal.test(job.state)) continue;
4336
+ lines.push(
4337
+ dim(" ") + shortJobId(name) + dim(` \xB7 ${job.model ?? "?"} \xB7 ${job.count} req \xB7 `) + job.state + dim(` \xB7 ${formatElapsed(job.elapsedMs)}`)
4338
+ );
4339
+ }
4340
+ } else if (active.length > 0) {
3425
4341
  lines.push(dim("Active ") + active.slice(0, 3).join(", ") + (active.length > 3 ? dim(` +${active.length - 3}`) : ""));
3426
4342
  } else if (done < total) {
3427
4343
  lines.push(dim("Active ") + "starting\u2026");
@@ -3449,12 +4365,58 @@ function createTranslateProgressReporter(options = {}) {
3449
4365
  total = event.total;
3450
4366
  concurrency = event.concurrency;
3451
4367
  model = event.model;
4368
+ mode = event.mode;
3452
4369
  render();
3453
4370
  break;
3454
4371
  case "item-start":
3455
4372
  active = event.active;
3456
4373
  render();
3457
4374
  break;
4375
+ case "batch-submitted": {
4376
+ const createdAtMs = event.createdAt ? Date.parse(event.createdAt) : void 0;
4377
+ const elapsedMs = createdAtMs && !Number.isNaN(createdAtMs) ? Date.now() - createdAtMs : 0;
4378
+ batchJobs.set(event.name, {
4379
+ state: event.resumed ? "RESUMED" : "SUBMITTED",
4380
+ count: event.count,
4381
+ model: event.model,
4382
+ createdAtMs: Number.isNaN(createdAtMs) ? void 0 : createdAtMs,
4383
+ elapsedMs
4384
+ });
4385
+ const requests = `${event.count} ${event.resumed ? "pending " : ""}request${event.count === 1 ? "" : "s"}`;
4386
+ const origin = event.resumed ? `resumed job (submitted ${formatElapsed(elapsedMs)} ago)` : "submitted job";
4387
+ printPersistent([
4388
+ dim(`${event.resumed ? "\u21BB" : "\u2192"} ${origin} ${shortJobId(event.name)} \xB7 ${event.model ?? "?"} \xB7 ${requests}`)
4389
+ ]);
4390
+ render();
4391
+ break;
4392
+ }
4393
+ case "batch-polling": {
4394
+ const job = batchJobs.get(event.name);
4395
+ batchJobs.set(event.name, {
4396
+ state: event.state.replace(/^JOB_STATE_/, ""),
4397
+ count: job?.count ?? 0,
4398
+ model: job?.model,
4399
+ createdAtMs: job?.createdAtMs,
4400
+ elapsedMs: event.elapsedMs
4401
+ });
4402
+ batchElapsedMs = Math.max(batchElapsedMs, event.elapsedMs);
4403
+ render();
4404
+ break;
4405
+ }
4406
+ case "batch-done": {
4407
+ const job = batchJobs.get(event.name);
4408
+ batchJobs.set(event.name, {
4409
+ state: event.state.replace(/^JOB_STATE_/, ""),
4410
+ count: job?.count ?? event.count,
4411
+ model: job?.model ?? event.model,
4412
+ createdAtMs: job?.createdAtMs,
4413
+ elapsedMs: event.elapsedMs
4414
+ });
4415
+ const marker = event.failed > 0 && event.translated === 0 ? red("\u2717") : green("\u2713");
4416
+ printPersistent([`${marker} ${dim(batchDoneParts(event))}`]);
4417
+ render();
4418
+ break;
4419
+ }
3458
4420
  case "item-done": {
3459
4421
  done += 1;
3460
4422
  active = active.filter((label) => label !== labelForResult(event.result));
@@ -3577,6 +4539,21 @@ function parseArgs(argv) {
3577
4539
  i++;
3578
4540
  continue;
3579
4541
  }
4542
+ if (arg === "--batch") {
4543
+ options.batch = true;
4544
+ i++;
4545
+ continue;
4546
+ }
4547
+ if (arg === "--direct") {
4548
+ options.direct = true;
4549
+ i++;
4550
+ continue;
4551
+ }
4552
+ if (arg === "--resume") {
4553
+ options.resume = true;
4554
+ i++;
4555
+ continue;
4556
+ }
3580
4557
  if (arg === "--strategy") {
3581
4558
  const value = argv[++i];
3582
4559
  if (value !== "all" && value !== "missing-only") {
@@ -3645,11 +4622,19 @@ Translate flags:
3645
4622
  --locale <code>... Target locale(s); overrides --preset
3646
4623
  --slug <en-slug> Single English document
3647
4624
  --model <id> Gemini model override
3648
- --concurrency <n> Parallel translations (default: 3)
4625
+ --batch Force Gemini Batch API mode (50% token cost; default)
4626
+ --direct Force direct interactive calls (immediate, full price)
4627
+ --resume Only poll/ingest pending batch jobs; submit nothing new
4628
+ --concurrency <n> Parallel translations in --direct mode (default: 3)
3649
4629
  --dry-run List work without writing
3650
4630
  --force Re-translate even when hashes match
3651
4631
  --strategy <mode> all (default) or missing-only
3652
4632
  --no-progress Plain line logging instead of live progress
4633
+
4634
+ Batch mode submits jobs upfront and persists them in the store, so Ctrl+C
4635
+ during polling is safe: the next run (or --resume) picks the jobs back up and
4636
+ ingests their results. In a TTY without --batch/--direct, a prompt asks which
4637
+ mode to use.
3653
4638
  `);
3654
4639
  return;
3655
4640
  }
@@ -3695,11 +4680,29 @@ Translate flags:
3695
4680
  break;
3696
4681
  }
3697
4682
  case "translate": {
4683
+ if (options.batch && options.direct) {
4684
+ throw new Error("--batch and --direct are mutually exclusive");
4685
+ }
4686
+ if (options.resume) {
4687
+ const reporter2 = createTranslateProgressReporter({
4688
+ enabled: !options.noProgress,
4689
+ dryRun: false
4690
+ });
4691
+ const results2 = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
4692
+ reporter2.finish();
4693
+ if (results2 === null) {
4694
+ console.log("No pending translation batch jobs.");
4695
+ break;
4696
+ }
4697
+ if (results2.some((result) => result.failed)) process.exitCode = 1;
4698
+ break;
4699
+ }
3698
4700
  const selection = await promptTranslateSelection(config, {
3699
4701
  type: options.type,
3700
4702
  preset: options.preset,
3701
4703
  locale: options.locale,
3702
- strategy: options.strategy
4704
+ strategy: options.strategy,
4705
+ mode: options.batch ? "batch" : options.direct ? "direct" : void 0
3703
4706
  });
3704
4707
  const locales = resolveLocalesFromPreset(config, selection.preset, selection.locale);
3705
4708
  const worklist = buildWorklist(config, {
@@ -3709,6 +4712,18 @@ Translate flags:
3709
4712
  strategy: selection.strategy ?? "all"
3710
4713
  });
3711
4714
  if (worklist.length === 0) {
4715
+ if (!options.dryRun) {
4716
+ const reporter2 = createTranslateProgressReporter({
4717
+ enabled: !options.noProgress,
4718
+ dryRun: false
4719
+ });
4720
+ const resumed = await resumeTranslationJobs(config, { onProgress: reporter2.onEvent });
4721
+ reporter2.finish();
4722
+ if (resumed !== null) {
4723
+ if (resumed.some((result) => result.failed)) process.exitCode = 1;
4724
+ break;
4725
+ }
4726
+ }
3712
4727
  console.log("Nothing to translate.");
3713
4728
  break;
3714
4729
  }
@@ -3721,6 +4736,7 @@ Translate flags:
3721
4736
  dryRun: options.dryRun,
3722
4737
  force: options.force,
3723
4738
  concurrency: options.concurrency,
4739
+ mode: selection.mode,
3724
4740
  onProgress: reporter.onEvent
3725
4741
  });
3726
4742
  reporter.finish();