solid-translate 1.4.5 → 1.6.0

package/README.md

@@ -406,9 +406,10 @@ on every pull request.
   "sourceLocale": "en",
   "targetLocales": ["es", "fr", "de"],
   "localesDir": "./src/locales",
-  "provider": "openrouter",
+  "provider": "openrouter",            // openai | openrouter | anthropic | google | openai-compatible
   "model": "openai/gpt-4o-mini",
   "batchSize": 50,
+  "maxAttempts": 3,
   "include": ["src/**/*.tsx", "src/**/*.ts"],
   "files": {
     "json": {
@@ -424,6 +425,37 @@ on every pull request.
 }
 ```
 
+### Any OpenAI-compatible host
+
+Point the CLI at a self-hosted gateway or an inference endpoint that speaks the
+OpenAI chat-completions API:
+
+```json
+{
+  "provider": "openai-compatible",
+  "baseURL": "https://api.heyditto.ai/v1",
+  "apiKeyEnv": "DITTO_TRANSLATE_KEY",
+  "model": "translate"
+}
+```
+
+`apiKeyEnv` names the environment variable that holds the key (default
+`OPENAI_COMPATIBLE_API_KEY`). The `model` is whatever id the host accepts.
+
+### Reliability: retries, splitting and batch size
+
+Every batch is retried with exponential backoff (default `maxAttempts: 3`), and each
+retry asks only for the keys the model has not translated yet. A batch that comes back
+with nothing usable — typically a response truncated mid-JSON — is split in half and
+each half recovers on its own, down to single keys. Only keys that fail every attempt
+are reported, and they are left out of the lock file so the next `translate` run
+retries exactly those.
+
+The output-token ceiling scales with both the number of keys and the amount of source
+text in the batch. If your strings are long paragraphs, a smaller `batchSize` (8–10)
+keeps each response well inside model limits and makes a single bad response cheap to
+retry.
+
 The `[locale]` placeholder is replaced with each target locale. Source files are found by replacing `[locale]` with the source locale.
 
 ### Environment Variables

package/dist/{chunk-36PYT2MX.js → chunk-VAKU3D24.js}

@@ -41,9 +41,31 @@ function collectBatchTranslations(parsed, requestedKeys) {
   }
   return { translations, missing };
 }
-async function translateBatch(model, entries, targetLocale, sourceLocale, systemPrompt, contexts) {
+var DEFAULT_MAX_ATTEMPTS = 3;
+var DEFAULT_BASE_DELAY_MS = 500;
+function batchMaxTokens(entries) {
+  const keyCount = Object.keys(entries).length;
+  let sourceChars = 0;
+  for (const [key, value] of Object.entries(entries)) {
+    sourceChars += key.length + value.length;
+  }
+  return Math.min(32e3, 2e3 + 400 * keyCount + Math.ceil(sourceChars * 1.5));
+}
+function backoffDelay(attempt, baseDelayMs) {
+  const exp = Math.min(3e4, baseDelayMs * 2 ** Math.max(0, attempt - 1));
+  return Math.round(exp * (0.75 + Math.random() * 0.5));
+}
+function defaultSleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function translateBatch(model, entries, targetLocale, sourceLocale, systemPrompt, contexts, options = {}) {
   const keys = Object.keys(entries);
   if (keys.length === 0) return {};
+  const maxAttempts = Math.max(1, options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
+  const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+  const sleep = options.sleep ?? defaultSleep;
+  const log = options.log ?? (() => {
+  });
   const defaultSystem = [
     `You are a professional translator specializing in software localization.`,
     `Translate text from "${sourceLocale}" to "${targetLocale}".`,
@@ -54,57 +76,72 @@ async function translateBatch(model, entries, targetLocale, sourceLocale, system
     `- Do not add or remove content`,
     `- Return natural, idiomatic translations`
   ].join("\n");
-  let contextSection = "";
-  if (contexts && Object.keys(contexts).length > 0) {
-    const contextLines = Object.entries(contexts).filter(([key]) => key in entries).map(([key, ctx]) => `  "${key}": ${ctx}`);
-    if (contextLines.length > 0) {
-      contextSection = [
-        ``,
-        `Context hints for disambiguation:`,
-        ...contextLines,
-        ``
-      ].join("\n");
-    }
-  }
+  const contextSectionFor = (subset) => {
+    if (!contexts || Object.keys(contexts).length === 0) return "";
+    const contextLines = Object.entries(contexts).filter(([key]) => key in subset).map(([key, ctx]) => `  "${key}": ${ctx}`);
+    if (contextLines.length === 0) return "";
+    return [``, `Context hints for disambiguation:`, ...contextLines, ``].join(
+      "\n"
+    );
+  };
   const generateText = await loadGenerateText();
-  const basePrompt = [
-    `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`,
-    `Respond with ONLY a JSON object \u2014 no prose, no code fences \u2014 containing the exact same keys and the translated values.`,
-    contextSection,
-    JSON.stringify(entries, null, 2)
-  ].join("\n");
-  const attempt = async (prompt, keyCount) => {
+  const attempt = async (subset) => {
+    const subsetKeys = Object.keys(subset);
     const { text } = await generateText({
       model,
       system: systemPrompt || defaultSystem,
-      prompt,
-      // Provider-default output ceilings (often 4k tokens) truncate large
-      // batches mid-JSON ("Unterminated string in JSON"). Scale the ceiling
-      // with batch size so the full object always fits.
-      maxTokens: Math.min(32e3, 2e3 + 400 * keyCount)
-    });
-    return collectBatchTranslations(extractJsonObject(text), keys);
-  };
-  let { translations, missing } = await attempt(basePrompt, keys.length);
-  if (missing.length > 0) {
-    const retryEntries = {};
-    for (const key of missing) retryEntries[key] = entries[key];
-    const retry = await attempt(
-      [
+      prompt: [
         `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`,
         `Respond with ONLY a JSON object \u2014 no prose, no code fences \u2014 containing the exact same keys and the translated values.`,
-        contextSection,
-        JSON.stringify(retryEntries, null, 2)
+        contextSectionFor(subset),
+        JSON.stringify(subset, null, 2)
       ].join("\n"),
-      missing.length
-    );
-    translations = { ...translations, ...retry.translations };
-    missing = retry.missing;
-  }
-  if (missing.length > 0) {
-    const sample = missing.slice(0, 3).join('", "');
+      maxTokens: batchMaxTokens(subset)
+    });
+    return collectBatchTranslations(extractJsonObject(text), subsetKeys);
+  };
+  const translations = {};
+  const failed = [];
+  let lastError;
+  const recover = async (pending, depth) => {
+    let remaining = pending;
+    for (let n = 1; n <= maxAttempts && remaining.length > 0; n++) {
+      const subset = {};
+      for (const key of remaining) subset[key] = entries[key];
+      let missing;
+      try {
+        const result = await attempt(subset);
+        Object.assign(translations, result.translations);
+        missing = result.missing;
+      } catch (err) {
+        lastError = err;
+        missing = remaining;
+        log(
+          `translate ${targetLocale}: attempt ${n}/${maxAttempts} for ${remaining.length} key(s) failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+      if (missing.length === 0) return;
+      const progressed = missing.length < remaining.length;
+      remaining = missing;
+      if (!progressed && remaining.length > 1) {
+        const mid = Math.ceil(remaining.length / 2);
+        log(
+          `translate ${targetLocale}: splitting ${remaining.length} key(s) into ${mid} + ${remaining.length - mid}`
+        );
+        await recover(remaining.slice(0, mid), depth + 1);
+        await recover(remaining.slice(mid), depth + 1);
+        return;
+      }
+      if (n < maxAttempts) await sleep(backoffDelay(n, baseDelayMs));
+    }
+    failed.push(...remaining);
+  };
+  await recover(keys, 0);
+  if (failed.length > 0) {
+    const sample = failed.slice(0, 3).join('", "');
+    const cause = lastError instanceof Error ? ` (last error: ${lastError.message})` : "";
     throw new Error(
-      `model returned no translation for ${missing.length} of ${keys.length} keys (e.g. "${sample}")`
+      `model returned no translation for ${failed.length} of ${keys.length} keys after ${maxAttempts} attempts (e.g. "${sample}")${cause}`
     );
   }
   return translations;
@@ -144,6 +181,8 @@ async function translateMarkdown(model, content, targetLocale, sourceLocale, sys
 export {
   extractJsonObject,
   collectBatchTranslations,
+  batchMaxTokens,
+  backoffDelay,
   translateBatch,
   translateMarkdown
 };

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   translateBatch
-} from "./chunk-36PYT2MX.js";
+} from "./chunk-VAKU3D24.js";
 import {
   __commonJS,
   __toESM
@@ -15318,10 +15318,30 @@ async function loadConfig() {
   );
   process.exit(1);
 }
+function resolveOpenAICompatible(config) {
+  const baseURL = (config.baseURL || "").trim().replace(/\/+$/, "");
+  if (!baseURL) {
+    throw new Error(
+      'provider "openai-compatible" requires "baseURL" in the config (e.g. "https://api.heyditto.ai/v1")'
+    );
+  }
+  return { baseURL, apiKeyEnv: config.apiKeyEnv || "OPENAI_COMPATIBLE_API_KEY" };
+}
 async function createModel(config) {
   const provider = config.provider || "openai";
   const modelId = config.model || "gpt-4o-mini";
   try {
+    if (provider === "openai-compatible") {
+      const { baseURL, apiKeyEnv } = resolveOpenAICompatible(config);
+      const apiKey = process.env[apiKeyEnv];
+      if (!apiKey) {
+        console.error(`Missing ${apiKeyEnv} for provider "openai-compatible" (${baseURL}).`);
+        process.exit(1);
+      }
+      const { createOpenAI } = await import("./dist-ZA6JWKFH.js");
+      const compat = createOpenAI({ baseURL, apiKey, compatibility: "compatible" });
+      return compat.chat(modelId);
+    }
     if (provider === "openai" || provider === "openrouter") {
       const { createOpenAI } = await import("./dist-ZA6JWKFH.js");
       if (provider === "openrouter") {
@@ -15351,7 +15371,7 @@ async function createModel(config) {
       return google(modelId);
     }
     console.error(`Unknown provider: ${provider}`);
-    console.error("Supported: openai, openrouter, anthropic, google");
+    console.error("Supported: openai, openrouter, anthropic, google, openai-compatible");
     process.exit(1);
   } catch (err) {
     console.error(
@@ -15570,13 +15590,14 @@ Run \`solid-translate translate\` to refresh ${sourceLocale} \u2192 targets.`
   );
 }
 async function runTranslate() {
-  const { translateBatch: translateBatch2, translateMarkdown: translateMarkdown2 } = await import("./translate-PTFH3PFU.js");
+  const { translateBatch: translateBatch2, translateMarkdown: translateMarkdown2 } = await import("./translate-FXWVMHH2.js");
   const config = await loadConfig();
   const root = process.cwd();
   const sourceLocale = config.sourceLocale || "en";
   const targetLocales = config.targetLocales;
   const localesDir = resolve(config.localesDir || "./src/locales");
   const batchSize = config.batchSize || 50;
+  const maxAttempts = config.maxAttempts || 3;
   if (!targetLocales || targetLocales.length === 0) {
     console.error("No targetLocales configured.");
     process.exit(1);
@@ -15588,7 +15609,8 @@ async function runTranslate() {
     sourceLocale,
     targetLocales,
     batchSize,
-    config.systemPrompt
+    config.systemPrompt,
+    maxAttempts
   );
   if (config.files) {
     const { glob } = await import("glob");
@@ -15628,7 +15650,9 @@ async function runTranslate() {
                   sourceDict,
                   targetLocale,
                   sourceLocale,
-                  config.systemPrompt
+                  config.systemPrompt,
+                  void 0,
+                  { maxAttempts, log: (message) => console.log(message) }
                 );
                 mkdirSync(dirname(actualTarget), { recursive: true });
                 writeFileSync2(
@@ -15672,7 +15696,7 @@ async function runTranslate() {
   }
   console.log("\nTranslation complete.");
 }
-async function translateLocaleFiles(model, localesDir, sourceLocale, targetLocales, batchSize, systemPrompt) {
+async function translateLocaleFiles(model, localesDir, sourceLocale, targetLocales, batchSize, systemPrompt, maxAttempts) {
   const result = await syncLocaleFiles({
     localesDir,
     sourceLocale,
@@ -15684,7 +15708,8 @@ async function translateLocaleFiles(model, localesDir, sourceLocale, targetLocal
       targetLocale,
       sourceLocale,
       systemPrompt,
-      contexts
+      contexts,
+      { maxAttempts, log: (message) => console.log(`  ${message}`) }
     ),
     log: (message) => console.log(message)
   });
@@ -15709,3 +15734,6 @@ main().catch((err) => {
   console.error(err);
   process.exit(1);
 });
+export {
+  resolveOpenAICompatible
+};

package/dist/{translate-PTFH3PFU.js → translate-FXWVMHH2.js}

@@ -1,12 +1,16 @@
 #!/usr/bin/env node
 import {
+  backoffDelay,
+  batchMaxTokens,
   collectBatchTranslations,
   extractJsonObject,
   translateBatch,
   translateMarkdown
-} from "./chunk-36PYT2MX.js";
+} from "./chunk-VAKU3D24.js";
 import "./chunk-FYS2JH42.js";
 export {
+  backoffDelay,
+  batchMaxTokens,
   collectBatchTranslations,
   extractJsonObject,
   translateBatch,

package/dist/vite.js

@@ -14652,9 +14652,31 @@ function collectBatchTranslations(parsed, requestedKeys) {
   }
   return { translations, missing };
 }
-async function translateBatch(model, entries, targetLocale, sourceLocale, systemPrompt, contexts) {
+var DEFAULT_MAX_ATTEMPTS = 3;
+var DEFAULT_BASE_DELAY_MS = 500;
+function batchMaxTokens(entries) {
+  const keyCount = Object.keys(entries).length;
+  let sourceChars = 0;
+  for (const [key, value] of Object.entries(entries)) {
+    sourceChars += key.length + value.length;
+  }
+  return Math.min(32e3, 2e3 + 400 * keyCount + Math.ceil(sourceChars * 1.5));
+}
+function backoffDelay(attempt, baseDelayMs) {
+  const exp = Math.min(3e4, baseDelayMs * 2 ** Math.max(0, attempt - 1));
+  return Math.round(exp * (0.75 + Math.random() * 0.5));
+}
+function defaultSleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
+async function translateBatch(model, entries, targetLocale, sourceLocale, systemPrompt, contexts, options = {}) {
   const keys = Object.keys(entries);
   if (keys.length === 0) return {};
+  const maxAttempts = Math.max(1, options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
+  const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+  const sleep = options.sleep ?? defaultSleep;
+  const log = options.log ?? (() => {
+  });
   const defaultSystem = [
     `You are a professional translator specializing in software localization.`,
     `Translate text from "${sourceLocale}" to "${targetLocale}".`,
@@ -14665,57 +14687,72 @@ async function translateBatch(model, entries, targetLocale, sourceLocale, system
     `- Do not add or remove content`,
     `- Return natural, idiomatic translations`
   ].join("\n");
-  let contextSection = "";
-  if (contexts && Object.keys(contexts).length > 0) {
-    const contextLines = Object.entries(contexts).filter(([key]) => key in entries).map(([key, ctx]) => `  "${key}": ${ctx}`);
-    if (contextLines.length > 0) {
-      contextSection = [
-        ``,
-        `Context hints for disambiguation:`,
-        ...contextLines,
-        ``
-      ].join("\n");
-    }
-  }
+  const contextSectionFor = (subset) => {
+    if (!contexts || Object.keys(contexts).length === 0) return "";
+    const contextLines = Object.entries(contexts).filter(([key]) => key in subset).map(([key, ctx]) => `  "${key}": ${ctx}`);
+    if (contextLines.length === 0) return "";
+    return [``, `Context hints for disambiguation:`, ...contextLines, ``].join(
+      "\n"
+    );
+  };
   const generateText = await loadGenerateText();
-  const basePrompt = [
-    `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`,
-    `Respond with ONLY a JSON object \u2014 no prose, no code fences \u2014 containing the exact same keys and the translated values.`,
-    contextSection,
-    JSON.stringify(entries, null, 2)
-  ].join("\n");
-  const attempt = async (prompt, keyCount) => {
+  const attempt = async (subset) => {
+    const subsetKeys = Object.keys(subset);
     const { text } = await generateText({
       model,
       system: systemPrompt || defaultSystem,
-      prompt,
-      // Provider-default output ceilings (often 4k tokens) truncate large
-      // batches mid-JSON ("Unterminated string in JSON"). Scale the ceiling
-      // with batch size so the full object always fits.
-      maxTokens: Math.min(32e3, 2e3 + 400 * keyCount)
-    });
-    return collectBatchTranslations(extractJsonObject(text), keys);
-  };
-  let { translations, missing } = await attempt(basePrompt, keys.length);
-  if (missing.length > 0) {
-    const retryEntries = {};
-    for (const key of missing) retryEntries[key] = entries[key];
-    const retry = await attempt(
-      [
+      prompt: [
         `Translate each value in this JSON object from "${sourceLocale}" to "${targetLocale}".`,
         `Respond with ONLY a JSON object \u2014 no prose, no code fences \u2014 containing the exact same keys and the translated values.`,
-        contextSection,
-        JSON.stringify(retryEntries, null, 2)
+        contextSectionFor(subset),
+        JSON.stringify(subset, null, 2)
       ].join("\n"),
-      missing.length
-    );
-    translations = { ...translations, ...retry.translations };
-    missing = retry.missing;
-  }
-  if (missing.length > 0) {
-    const sample = missing.slice(0, 3).join('", "');
+      maxTokens: batchMaxTokens(subset)
+    });
+    return collectBatchTranslations(extractJsonObject(text), subsetKeys);
+  };
+  const translations = {};
+  const failed = [];
+  let lastError;
+  const recover = async (pending, depth) => {
+    let remaining = pending;
+    for (let n = 1; n <= maxAttempts && remaining.length > 0; n++) {
+      const subset = {};
+      for (const key of remaining) subset[key] = entries[key];
+      let missing;
+      try {
+        const result = await attempt(subset);
+        Object.assign(translations, result.translations);
+        missing = result.missing;
+      } catch (err) {
+        lastError = err;
+        missing = remaining;
+        log(
+          `translate ${targetLocale}: attempt ${n}/${maxAttempts} for ${remaining.length} key(s) failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+      if (missing.length === 0) return;
+      const progressed = missing.length < remaining.length;
+      remaining = missing;
+      if (!progressed && remaining.length > 1) {
+        const mid = Math.ceil(remaining.length / 2);
+        log(
+          `translate ${targetLocale}: splitting ${remaining.length} key(s) into ${mid} + ${remaining.length - mid}`
+        );
+        await recover(remaining.slice(0, mid), depth + 1);
+        await recover(remaining.slice(mid), depth + 1);
+        return;
+      }
+      if (n < maxAttempts) await sleep(backoffDelay(n, baseDelayMs));
+    }
+    failed.push(...remaining);
+  };
+  await recover(keys, 0);
+  if (failed.length > 0) {
+    const sample = failed.slice(0, 3).join('", "');
+    const cause = lastError instanceof Error ? ` (last error: ${lastError.message})` : "";
     throw new Error(
-      `model returned no translation for ${missing.length} of ${keys.length} keys (e.g. "${sample}")`
+      `model returned no translation for ${failed.length} of ${keys.length} keys after ${maxAttempts} attempts (e.g. "${sample}")${cause}`
     );
   }
   return translations;