scribe-cms 0.0.13 → 0.0.15

package/README.md

@@ -40,7 +40,7 @@ export default defineConfig({
   // store: ".scribe/store.sqlite"     (default)
   locales: ["en", "fr"],
   // defaultLocale: "en"              (default)
-  // localeFallbacks: { pt: ["pt-BR"] } (optional: try these locales before the default locale)
+  // localeFallbacks: true            (default: regional variants fall back to their base language, e.g. pt-BR to pt; set false to disable)
   types: [
     defineContentType({
       id: "blog",

package/dist/cli/index.cjs

@@ -360,39 +360,7 @@ function resolveConfig(input, baseDir) {
   if (localeRouting.strategy === "search-param" && !localeRouting.param) {
     throw new Error('scribe config: localeRouting search-param requires a "param" name');
   }
-  const localeFallbacks = {};
-  for (const [locale, chain] of Object.entries(raw.localeFallbacks ?? {})) {
-    if (!raw.locales.includes(locale)) {
-      throw new Error(
-        `scribe config: localeFallbacks key "${locale}" is not in locales [${raw.locales.join(", ")}]`
-      );
-    }
-    const seen = /* @__PURE__ */ new Set();
-    for (const fallback of chain) {
-      if (!raw.locales.includes(fallback)) {
-        throw new Error(
-          `scribe config: localeFallbacks["${locale}"] entry "${fallback}" is not in locales [${raw.locales.join(", ")}]`
-        );
-      }
-      if (fallback === locale) {
-        throw new Error(
-          `scribe config: localeFallbacks["${locale}"] must not contain its own key "${locale}"`
-        );
-      }
-      if (fallback === defaultLocale) {
-        throw new Error(
-          `scribe config: localeFallbacks["${locale}"] must not contain the defaultLocale "${defaultLocale}" \u2014 it is always the final fallback; remove it`
-        );
-      }
-      if (seen.has(fallback)) {
-        throw new Error(
-          `scribe config: localeFallbacks["${locale}"] contains duplicate entry "${fallback}"`
-        );
-      }
-      seen.add(fallback);
-    }
-    localeFallbacks[locale] = [...chain];
-  }
+  const localeFallbacks = raw.localeFallbacks === false ? {} : deriveLocaleFallbacks(raw.locales, defaultLocale);
   const projectRoot = path3__default.default.resolve(baseDir ?? process.cwd(), raw.rootDir);
   const contentRoot = path3__default.default.resolve(projectRoot, raw.contentDir ?? "content");
   const storePath = path3__default.default.resolve(projectRoot, raw.store ?? ".scribe/store.sqlite");
@@ -432,6 +400,25 @@ function resolveConfig(input, baseDir) {
   });
   return config;
 }
+function deriveLocaleFallbacks(locales, defaultLocale) {
+  const byLowercase = new Map(locales.map((l) => [l.toLowerCase(), l]));
+  const fallbacks = {};
+  for (const locale of locales) {
+    const subtags = locale.split("-");
+    if (subtags.length < 2) continue;
+    const chain = [];
+    for (let end = subtags.length - 1; end >= 1; end--) {
+      const prefix = byLowercase.get(subtags.slice(0, end).join("-").toLowerCase());
+      if (prefix && prefix !== locale && prefix !== defaultLocale) {
+        chain.push(prefix);
+      }
+    }
+    if (chain.length > 0) {
+      fallbacks[locale] = chain;
+    }
+  }
+  return fallbacks;
+}
 
 // src/create-project.ts
 init_cjs_shims();
@@ -2567,6 +2554,13 @@ function buildOutputFormatLine(hasFrontmatter, slugStrategy) {
   }
   return slugStrategy === "localized" ? "`frontmatter` (object with translated frontmatter fields), `body` (string, full MDX body), `slug` (string)." : "`frontmatter` (object), `body` (string).";
 }
+function buildRetryContextSection(previousError) {
+  return [
+    "",
+    "## Previous attempt",
+    `A previous attempt at this translation was rejected with the following validation errors: ${previousError}. Produce a corrected translation that fixes these issues while re-checking every schema constraint (required fields, array minimums, maximum lengths).`
+  ];
+}
 function buildPageTranslationPrompt(input) {
   const localeName = LOCALE_NAMES[input.targetLocale] ?? input.targetLocale;
   const localizationPrompt = input.resolved.promptOverride ?? defaultLocalizationPrompt(localeName, input.targetLocale);
@@ -2597,7 +2591,10 @@ function buildPageTranslationPrompt(input) {
     buildOutputFormatLine(
       Object.keys(input.translatableFrontmatter).length > 0,
       input.slugStrategy
-    )
+    ),
+    // Retry context is locale-specific suffix material: it MUST stay after the
+    // EN body so the cacheable prefix is byte-identical with and without it.
+    ...input.previousError ? buildRetryContextSection(input.previousError) : []
   ];
   return [...prefix, ...suffix].join("\n");
 }
@@ -2817,7 +2814,8 @@ function prepareTranslation(config, item, options, startedAt) {
     contextLabel: resolveContextLabel(enDoc, item.enSlug),
     translatableFrontmatter: payload.frontmatter,
     enBody: payload.body,
-    slugStrategy: type.slugStrategy
+    slugStrategy: type.slugStrategy,
+    previousError: item.previousError
   });
   const responseSchema = buildGeminiResponseSchema(type.schema, type.slugStrategy);
   return {
@@ -3229,6 +3227,82 @@ async function runPool(items, concurrency, worker) {
 function labelForItem(item) {
   return `${item.contentType}/${item.enSlug}@${item.locale}`;
 }
+function buildRetryWorklist(failed, originalByKey) {
+  return failed.map((result) => {
+    const key = translationItemKey(result);
+    const original = originalByKey.get(key);
+    return {
+      contentType: result.contentType,
+      enSlug: result.enSlug,
+      locale: result.locale,
+      reason: original?.reason ?? "missing",
+      currentEnHash: original?.currentEnHash ?? "",
+      storedEnHash: original?.storedEnHash,
+      previousError: result.error ?? "Translation failed"
+    };
+  });
+}
+async function runRetryRound(config, retryItems, options) {
+  const startedAt = Date.now();
+  const translateOne = options.translateOne ?? ((item) => translatePage(config, item, { model: options.model, force: true }));
+  if (options.mode === "direct") {
+    const active = /* @__PURE__ */ new Set();
+    await runPool(retryItems, options.concurrency, async (item) => {
+      const label = labelForItem(item);
+      active.add(label);
+      options.onProgress?.({ type: "item-start", item, active: [...active] });
+      const result = await translateOne(item);
+      active.delete(label);
+      options.onResult(result);
+    });
+    return;
+  }
+  const entries = [];
+  for (const item of retryItems) {
+    const itemStartedAt = Date.now();
+    try {
+      const outcome = prepareTranslation(config, item, { model: options.model, force: true }, itemStartedAt);
+      if (outcome.status === "done") {
+        options.onResult(outcome.result);
+      } else {
+        entries.push({
+          apiModel: resolveGeminiModelId(displayModelFor(outcome.prepared)),
+          prompt: outcome.prepared.prompt,
+          prepared: outcome.prepared
+        });
+      }
+    } catch (error) {
+      options.onResult(failedResultForItem(item, error, itemStartedAt));
+    }
+  }
+  const plans = planBatchJobs(entries);
+  const submitted = await Promise.all(
+    plans.map(async (plan, planIndex) => {
+      try {
+        return await submitBatchJobPlan(config, {
+          plan,
+          displayModel: normalizeGeminiDisplayName(plan.apiModel),
+          jobIndex: planIndex,
+          jobCount: plans.length,
+          onProgress: options.onProgress
+        });
+      } catch (error) {
+        for (const entry of plan.entries) {
+          options.onResult(failedResultForItem(entry.prepared.item, error, startedAt));
+        }
+        return void 0;
+      }
+    })
+  );
+  const jobsToPoll = submitted.filter((row) => row !== void 0);
+  if (jobsToPoll.length > 0) {
+    await pollAndIngestBatchJobs(config, jobsToPoll, {
+      jobCount: jobsToPoll.length,
+      onProgress: options.onProgress,
+      onResult: options.onResult
+    });
+  }
+}
 async function translatePage(config, item, options = {}) {
   const startedAt = Date.now();
   let outcome;
@@ -3401,9 +3475,40 @@ async function translateWorklist(config, items, options = {}) {
       });
     }
   }
-  const results = collector.assemble();
+  const mainResults = collector.assemble();
+  const failed = mainResults.filter((result) => result.failed);
+  if (failed.length === 0) {
+    const totals2 = summarizeResults(mainResults, Date.now() - startedAt);
+    options.onProgress?.({ type: "done", results: mainResults, totals: totals2 });
+    return mainResults;
+  }
+  options.onProgress?.({ type: "retry-start", failed });
+  const originalByKey = new Map(
+    items.map((item) => [translationItemKey(item), item])
+  );
+  const retryItems = buildRetryWorklist(failed, originalByKey);
+  const retryByKey = /* @__PURE__ */ new Map();
+  await runRetryRound(config, retryItems, {
+    model: options.model,
+    force: options.force,
+    concurrency,
+    mode,
+    onProgress: options.onProgress,
+    onResult: (result) => {
+      retryByKey.set(translationItemKey(result), result);
+      options.onProgress?.({ type: "item-done", result });
+    }
+  });
+  const results = mainResults.map((result) => {
+    if (!result.failed) return result;
+    return retryByKey.get(translationItemKey(result)) ?? result;
+  });
+  const retriedTranslated = failed.filter((original) => {
+    const retried = retryByKey.get(translationItemKey(original));
+    return retried !== void 0 && !retried.failed && !retried.skipped;
+  }).length;
   const totals = summarizeResults(results, Date.now() - startedAt);
-  options.onProgress?.({ type: "done", results, totals });
+  options.onProgress?.({ type: "done", results, totals, retriedTranslated });
   return results;
 }
 function countPendingItems(config, jobs) {
@@ -4225,10 +4330,11 @@ function detailForResult(result) {
   if (result.error) parts.push(red(result.error));
   return parts.length > 0 ? dim(parts.join(" \xB7 ")) : "";
 }
-function formatSummaryLine(totals, dryRun) {
+function formatSummaryLine(totals, dryRun, retriedTranslated = 0) {
   const action = dryRun ? "Would translate" : "Translated";
+  const base = totals.translated - retriedTranslated;
   const parts = [
-    `${action} ${totals.translated}`,
+    retriedTranslated > 0 ? `${action} ${base} +${retriedTranslated} on retry` : `${action} ${totals.translated}`,
     `skipped ${totals.skipped}`
   ];
   if (totals.failed > 0) parts.push(red(`failed ${totals.failed}`));
@@ -4277,8 +4383,18 @@ function createTranslateProgressReporter(options = {}) {
           }
           return;
         }
+        if (event.type === "retry-start") {
+          for (const result of event.failed) {
+            console.log(`${labelForResult(result)}: ${result.error ?? "failed"}`);
+          }
+          console.log(`Retrying ${event.failed.length} failed items with validation context...`);
+          return;
+        }
         if (event.type === "done") {
-          console.log(formatSummaryLine(event.totals, dryRun));
+          console.log(formatSummaryLine(event.totals, dryRun, event.retriedTranslated));
+          for (const result of event.results.filter((entry) => entry.failed)) {
+            console.error(`${labelForResult(result)}: ${result.error ?? "failed"}`);
+          }
         }
       },
       finish() {
@@ -4445,12 +4561,25 @@ function createTranslateProgressReporter(options = {}) {
           render();
           break;
         }
+        case "retry-start": {
+          total += event.failed.length;
+          printPersistent([
+            ...event.failed.map(
+              (result) => red(`${labelForResult(result)}: ${result.error ?? "failed"}`)
+            ),
+            cyan(`Retrying ${event.failed.length} failed items with validation context...`)
+          ]);
+          render();
+          break;
+        }
         case "done":
           showCursor();
           if (renderedLines > 0) {
             process.stdout.write(`\x1B[${renderedLines}A`);
           }
-          process.stdout.write("\x1B[2K" + green("Done") + " \xB7 " + formatSummaryLine(event.totals, dryRun) + "\n");
+          process.stdout.write(
+            "\x1B[2K" + green("Done") + " \xB7 " + formatSummaryLine(event.totals, dryRun, event.retriedTranslated) + "\n"
+          );
           renderedLines = 0;
           for (const result of event.results.filter((entry) => entry.failed)) {
             process.stdout.write("\x1B[2K" + red(`${labelForResult(result)}: ${result.error ?? "failed"}`) + "\n");