transduck 0.5.2 → 0.6.0

package/src/index.ts

@@ -1,22 +1,28 @@
 import { createHash } from 'crypto';
 import { loadConfig, type TransduckConfig } from './config.js';
 import { TranslationStore } from './storage.js';
-import { translate as backendTranslate, translatePlural as backendTranslatePlural } from './backend.js';
+import { SharedStore } from './shared-store.js';
+import { TranslationResult } from './result.js';
+import { translate as backendTranslate, translatePlural as backendTranslatePlural, detectLanguage as backendDetectLanguage } from './backend.js';
 import { validateTranslation, extractPlaceholders } from './validation.js';
 import { getPluralCategory, getPluralCategories, interpolateVars } from './plural.js';
 
 interface State {
   config: TransduckConfig | null;
   store: TranslationStore | null;
+  sharedStore: SharedStore | null;
   targetLang: string | null;
   pendingTranslations: Map<string, Promise<string>>;
+  backgroundKeys: Set<string>;
 }
 
 let state: State = {
   config: null,
   store: null,
+  sharedStore: null,
   targetLang: null,
   pendingTranslations: new Map(),
+  backgroundKeys: new Set(),
 };
 
 function hash(text: string): string {
@@ -29,6 +35,17 @@ export async function initialize(config?: TransduckConfig): Promise<void> {
   await store.initialize();
   state.config = cfg;
   state.store = store;
+
+  if (cfg.sharedUrl) {
+    try {
+      const shared = new SharedStore(cfg.sharedUrl);
+      await shared.initialize();
+      state.sharedStore = shared;
+    } catch (err) {
+      console.warn(`[transduck] Could not connect to shared store: ${(err as Error).message}`);
+      state.sharedStore = null;
+    }
+  }
 }
 
 export function setLanguage(lang: string): void {
@@ -36,18 +53,36 @@ export function setLanguage(lang: string): void {
 }
 
 export function _resetState(): void {
-  state = { config: null, store: null, targetLang: null, pendingTranslations: new Map() };
+  state = {
+    config: null, store: null, sharedStore: null, targetLang: null,
+    pendingTranslations: new Map(), backgroundKeys: new Set(),
+  };
 }
 
 export function _getStore(): TranslationStore | null {
   return state.store;
 }
 
+export function _getSharedStore(): SharedStore | null {
+  return state.sharedStore;
+}
+
+export function _setSharedStore(shared: SharedStore | null): void {
+  state.sharedStore = shared;
+}
+
+export interface AitOptions {
+  context?: string;
+  vars?: Record<string, string | number>;
+  sourceLang?: string;
+  background?: boolean;
+}
+
 export async function ait(
   sourceText: string,
-  context?: string,
+  contextOrOpts?: string | AitOptions,
   vars?: Record<string, string | number>,
-): Promise<string> {
+): Promise<TranslationResult> {
   if (!state.config || !state.store) {
     throw new Error('transduck not initialized. Call initialize() first.');
   }
@@ -55,86 +90,170 @@ export async function ait(
     throw new Error('Target language not set. Call setLanguage() first.');
   }
 
+  // Parse overloaded args: ait(text, context?, vars?) OR ait(text, opts)
+  let context: string | undefined;
+  let resolvedVars: Record<string, string | number> | undefined;
+  let sourceLang: string | undefined;
+  let background = false;
+
+  if (typeof contextOrOpts === 'string') {
+    context = contextOrOpts;
+    resolvedVars = vars;
+  } else if (contextOrOpts != null) {
+    context = contextOrOpts.context;
+    resolvedVars = contextOrOpts.vars;
+    sourceLang = contextOrOpts.sourceLang;
+    background = contextOrOpts.background ?? false;
+  } else {
+    resolvedVars = vars;
+  }
+
   const cfg = state.config;
+  const effectiveSourceLang = sourceLang?.toUpperCase() ?? cfg.sourceLang;
+  const targetLang = state.targetLang;
 
-  if (state.targetLang === cfg.sourceLang) {
-    return interpolateVars(sourceText, vars);
+  if (targetLang === effectiveSourceLang) {
+    return new TranslationResult(
+      interpolateVars(sourceText, resolvedVars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
   }
 
   const projectContextHash = hash(cfg.projectContext);
   const stringContextHash = hash(context ?? '');
-
-  // Cache lookup
-  const cached = await state.store.lookup({
-    sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+  const lookupParams = {
+    sourceText, sourceLang: effectiveSourceLang, targetLang,
     projectContextHash, stringContextHash,
-  });
-  if (cached !== null) return interpolateVars(cached, vars);
+  };
+
+  // Tier 1: local SQLite lookup
+  const cached = await state.store.lookup(lookupParams);
+  if (cached !== null) {
+    return new TranslationResult(
+      interpolateVars(cached, resolvedVars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
+  }
+
+  // Tier 2: shared Postgres lookup
+  if (state.sharedStore) {
+    const sharedCached = await state.sharedStore.lookup(lookupParams);
+    if (sharedCached !== null) {
+      // Propagate to local store
+      await state.store.insert({
+        ...lookupParams, stringContext: context ?? '',
+        translatedText: sharedCached, model: cfg.backendModel, status: 'translated',
+      });
+      return new TranslationResult(
+        interpolateVars(sharedCached, resolvedVars),
+        { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+      );
+    }
+  }
 
   // Read-only mode: skip backend, return source text
   if (cfg.readOnly) {
-    return interpolateVars(sourceText, vars);
+    return new TranslationResult(
+      interpolateVars(sourceText, resolvedVars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
+  }
+
+  // Background mode: return immediately with pending=true
+  if (background) {
+    const bgKey = `${sourceText}|${effectiveSourceLang}|${targetLang}|${projectContextHash}|${stringContextHash}`;
+    if (!state.backgroundKeys.has(bgKey)) {
+      state.backgroundKeys.add(bgKey);
+      _doTranslate(sourceText, effectiveSourceLang, targetLang, projectContextHash, stringContextHash, context ?? '', cfg)
+        .finally(() => state.backgroundKeys.delete(bgKey));
+    }
+    return new TranslationResult(
+      interpolateVars(sourceText, resolvedVars),
+      { pending: true, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
   }
 
   // In-process dedup
-  const lockKey = `${sourceText}|${cfg.sourceLang}|${state.targetLang}|${projectContextHash}|${stringContextHash}`;
+  const lockKey = `${sourceText}|${effectiveSourceLang}|${targetLang}|${projectContextHash}|${stringContextHash}`;
 
   if (state.pendingTranslations.has(lockKey)) {
     const pending = await state.pendingTranslations.get(lockKey)!;
-    return interpolateVars(pending, vars);
+    return new TranslationResult(
+      interpolateVars(pending, resolvedVars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
   }
 
-  const translationPromise = (async () => {
-    // Double-check after getting in
-    const rechecked = await state.store!.lookup({
-      sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
-      projectContextHash, stringContextHash,
-    });
-    if (rechecked !== null) return rechecked;
+  const translationPromise = _doTranslate(sourceText, effectiveSourceLang, targetLang, projectContextHash, stringContextHash, context ?? '', cfg);
 
-    try {
-      const translated = await backendTranslate(
-        sourceText, cfg.sourceLang, state.targetLang!,
-        cfg.projectContext, context ?? null, cfg,
-      );
+  state.pendingTranslations.set(lockKey, translationPromise);
+  const result = await translationPromise;
+  state.pendingTranslations.delete(lockKey);
+  return new TranslationResult(
+    interpolateVars(result, resolvedVars),
+    { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+  );
+}
 
-      if (!validateTranslation(sourceText, translated)) {
-        console.warn(`[transduck] Validation failed for: ${sourceText} -> ${translated}`);
-        await state.store!.insert({
-          sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
-          projectContextHash, stringContextHash, stringContext: context ?? '',
-          translatedText: translated, model: cfg.backendModel, status: 'failed',
-        });
-        return sourceText;
-      }
+async function _doTranslate(
+  sourceText: string,
+  sourceLang: string,
+  targetLang: string,
+  projectContextHash: string,
+  stringContextHash: string,
+  stringContext: string,
+  cfg: TransduckConfig,
+): Promise<string> {
+  // Double-check local cache
+  const rechecked = await state.store!.lookup({
+    sourceText, sourceLang, targetLang, projectContextHash, stringContextHash,
+  });
+  if (rechecked !== null) return rechecked;
 
-      await state.store!.insert({
-        sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
-        projectContextHash, stringContextHash, stringContext: context ?? '',
-        translatedText: translated, model: cfg.backendModel, status: 'translated',
-      });
-      return translated;
-    } catch (err) {
-      console.warn(`[transduck] Backend failed for: ${sourceText}`, err);
+  try {
+    const translated = await backendTranslate(
+      sourceText, sourceLang, targetLang,
+      cfg.projectContext, stringContext || null, cfg,
+    );
+
+    const insertParams = {
+      sourceText, sourceLang, targetLang,
+      projectContextHash, stringContextHash, stringContext,
+      translatedText: translated, model: cfg.backendModel,
+    };
+
+    if (!validateTranslation(sourceText, translated)) {
+      console.warn(`[transduck] Validation failed for: ${sourceText} -> ${translated}`);
+      await state.store!.insert({ ...insertParams, status: 'failed' });
+      if (state.sharedStore) await state.sharedStore.insert({ ...insertParams, status: 'failed' });
       return sourceText;
-    } finally {
-      state.pendingTranslations.delete(lockKey);
     }
-  })();
 
-  state.pendingTranslations.set(lockKey, translationPromise);
-  const result = await translationPromise;
-  return interpolateVars(result, vars);
+    await state.store!.insert({ ...insertParams, status: 'translated' });
+    if (state.sharedStore) await state.sharedStore.insert({ ...insertParams, status: 'translated' });
+    return translated;
+  } catch (err) {
+    console.warn(`[transduck] Backend failed for: ${sourceText}`, err);
+    return sourceText;
+  }
 }
 
 export { createTransDuckHandler } from './handler.js';
+export { TranslationResult } from './result.js';
+export { SharedStore } from './shared-store.js';
+export async function detectLanguage(text: string): Promise<string> {
+  if (!state.config) {
+    throw new Error('transduck not initialized. Call initialize() first.');
+  }
+  return backendDetectLanguage(text, state.config);
+}
 
 export async function aitPlural(
   one: string,
   other: string,
   count: number,
-  opts?: { context?: string; vars?: Record<string, string | number> },
-): Promise<string> {
+  opts?: { context?: string; vars?: Record<string, string | number>; sourceLang?: string; background?: boolean },
+): Promise<TranslationResult> {
   if (!state.config || !state.store) {
     throw new Error('transduck not initialized. Call initialize() first.');
   }
@@ -144,6 +263,8 @@ export async function aitPlural(
 
   const cfg = state.config;
   const context = opts?.context;
+  const effectiveSourceLang = opts?.sourceLang?.toUpperCase() ?? cfg.sourceLang;
+  const background = opts?.background ?? false;
 
   // Build vars with count
   let vars: Record<string, string | number>;
@@ -155,13 +276,18 @@ export async function aitPlural(
     vars = { ...opts.vars };
   }
 
+  const targetLang = state.targetLang;
+
   // Same language, 2-form language: select directly from provided forms
-  if (state.targetLang === cfg.sourceLang) {
-    const categories = getPluralCategories(cfg.sourceLang);
+  if (targetLang === effectiveSourceLang) {
+    const categories = getPluralCategories(effectiveSourceLang);
     if (categories.size <= 2) {
-      const category = getPluralCategory(cfg.sourceLang, count);
+      const category = getPluralCategory(effectiveSourceLang, count);
       const form = category === 'one' ? one : other;
-      return interpolateVars(form, vars);
+      return new TranslationResult(
+        interpolateVars(form, vars),
+        { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+      );
     }
   }
 
@@ -169,31 +295,102 @@ export async function aitPlural(
   const sourceKey = one + '\x00' + other;
   const projectContextHash = hash(cfg.projectContext);
   const stringContextHash = hash(context ?? '');
-
-  // Cache lookup
-  const cached = await state.store.lookupPlural({
-    sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+  const lookupParams = {
+    sourceText: sourceKey, sourceLang: effectiveSourceLang, targetLang,
     projectContextHash, stringContextHash,
-  });
+  };
 
-  const category = getPluralCategory(state.targetLang, count);
+  const category = getPluralCategory(targetLang, count);
+
+  // Tier 1: local cache lookup
+  const cached = await state.store.lookupPlural(lookupParams);
   if (category in cached) {
-    return interpolateVars(cached[category], vars);
+    return new TranslationResult(
+      interpolateVars(cached[category], vars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
+  }
+
+  // Tier 2: shared store lookup
+  if (state.sharedStore) {
+    const sharedCached = await state.sharedStore.lookupPlural(lookupParams);
+    if (category in sharedCached) {
+      // Propagate all forms to local store
+      for (const [cat, text] of Object.entries(sharedCached)) {
+        await state.store.insertPlural({
+          ...lookupParams, stringContext: context ?? '',
+          pluralCategory: cat, translatedText: text,
+          model: cfg.backendModel, status: 'translated',
+        });
+      }
+      return new TranslationResult(
+        interpolateVars(sharedCached[category], vars),
+        { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+      );
+    }
   }
 
   // Read-only mode: skip backend, fall back to source forms
   if (cfg.readOnly) {
-    const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+    const fallbackCategory = getPluralCategory(effectiveSourceLang, count);
+    const fallback = fallbackCategory === 'one' ? one : other;
+    return new TranslationResult(
+      interpolateVars(fallback, vars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
+  }
+
+  // Background mode
+  if (background) {
+    const bgKey = `plural|${sourceKey}|${effectiveSourceLang}|${targetLang}|${projectContextHash}|${stringContextHash}`;
+    if (!state.backgroundKeys.has(bgKey)) {
+      state.backgroundKeys.add(bgKey);
+      _doPluralTranslate(one, other, sourceKey, effectiveSourceLang, targetLang, projectContextHash, stringContextHash, context ?? '', cfg)
+        .finally(() => state.backgroundKeys.delete(bgKey));
+    }
+    const fallbackCategory = getPluralCategory(effectiveSourceLang, count);
     const fallback = fallbackCategory === 'one' ? one : other;
-    return interpolateVars(fallback, vars);
+    return new TranslationResult(
+      interpolateVars(fallback, vars),
+      { pending: true, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
   }
 
   // Cache miss — call backend
+  const translatedText = await _doPluralTranslate(one, other, sourceKey, effectiveSourceLang, targetLang, projectContextHash, stringContextHash, context ?? '', cfg);
+
+  if (translatedText !== null && category in translatedText) {
+    return new TranslationResult(
+      interpolateVars(translatedText[category], vars),
+      { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+    );
+  }
+
+  // Fallback
+  const fallbackCategory = getPluralCategory(effectiveSourceLang, count);
+  const fallback = fallbackCategory === 'one' ? one : other;
+  return new TranslationResult(
+    interpolateVars(fallback, vars),
+    { pending: false, sourceLang: effectiveSourceLang, lang: targetLang },
+  );
+}
+
+async function _doPluralTranslate(
+  one: string,
+  other: string,
+  sourceKey: string,
+  sourceLang: string,
+  targetLang: string,
+  projectContextHash: string,
+  stringContextHash: string,
+  stringContext: string,
+  cfg: TransduckConfig,
+): Promise<Record<string, string> | null> {
   try {
     const forms = await backendTranslatePlural(
       one, other,
-      cfg.sourceLang, state.targetLang,
-      cfg.projectContext, context ?? null, cfg,
+      sourceLang, targetLang,
+      cfg.projectContext, stringContext || null, cfg,
     );
 
     // Validate and store each form
@@ -205,14 +402,16 @@ export async function aitPlural(
 
     if (typeof forms !== 'object' || forms === null || !('other' in forms)) {
       console.warn(`[transduck] Invalid plural response for: ${one} / ${other}`);
-      const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
-      const fallback = fallbackCategory === 'one' ? one : other;
-      return interpolateVars(fallback, vars);
+      return null;
     }
 
+    const lookupParams = {
+      sourceText: sourceKey, sourceLang, targetLang,
+      projectContextHash, stringContextHash,
+    };
+
     for (const [cat, text] of Object.entries(forms)) {
       if (!validCategories.has(cat) || !text) continue;
-      // Validate placeholders
       const translatedPlaceholders = extractPlaceholders(text);
       let allPresent = true;
       for (const p of sourcePlaceholders) {
@@ -222,38 +421,18 @@ export async function aitPlural(
         }
       }
       const status = allPresent ? 'translated' : 'failed';
-      await state.store!.insertPlural({
-        sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
-        projectContextHash, stringContextHash, stringContext: context ?? '',
+      const insertParams = {
+        ...lookupParams, stringContext,
         pluralCategory: cat, translatedText: text,
         model: cfg.backendModel, status,
-      });
-    }
-
-    // Select the right form
-    if (category in forms) {
-      const text = forms[category];
-      const tp = extractPlaceholders(text);
-      let allPresent = true;
-      for (const p of sourcePlaceholders) {
-        if (!tp.has(p)) {
-          allPresent = false;
-          break;
-        }
-      }
-      if (allPresent) {
-        return interpolateVars(text, vars);
-      }
+      };
+      await state.store!.insertPlural(insertParams);
+      if (state.sharedStore) await state.sharedStore.insertPlural(insertParams);
     }
 
-    // Fallback
-    const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
-    const fallback = fallbackCategory === 'one' ? one : other;
-    return interpolateVars(fallback, vars);
+    return forms;
   } catch (err) {
     console.warn(`[transduck] Backend failed for plural: ${one} / ${other}`, err);
-    const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
-    const fallback = fallbackCategory === 'one' ? one : other;
-    return interpolateVars(fallback, vars);
+    return null;
   }
 }

package/src/providers/claude-api.ts

@@ -50,6 +50,22 @@ export async function translate(
   return response.content[0].text.trim();
 }
 
+export async function detectLanguage(
+  text: string,
+  config: TransduckConfig,
+): Promise<string> {
+  const client = await getClient(config);
+  const response = await client.messages.create({
+    model: config.backendModel,
+    max_tokens: 16,
+    temperature: 0.0,
+    system: 'What language is this text written in? Return only the uppercase ISO 639-1 code.',
+    messages: [{ role: 'user', content: text }],
+  });
+
+  return response.content[0].text.trim().toUpperCase();
+}
+
 export async function translatePlural(
   one: string,
   other: string,

package/src/providers/claude-code.ts

@@ -54,6 +54,16 @@ export async function translate(
   return translateWithSdk(prompt);
 }
 
+export async function detectLanguage(
+  text: string,
+  config: TransduckConfig,
+): Promise<string> {
+  ensureToken(config);
+  const prompt = 'What language is this text written in? Return only the uppercase ISO 639-1 code.\n\n' + text;
+  const raw = await translateWithSdk(prompt);
+  return raw.trim().toUpperCase();
+}
+
 export async function translatePlural(
   one: string,
   other: string,

package/src/providers/index.ts

@@ -25,6 +25,12 @@ export interface TranslationProvider {
     config: TransduckConfig,
     _clientOverride?: any,
   ): Promise<Record<string, string>>;
+
+  detectLanguage(
+    text: string,
+    config: TransduckConfig,
+    _clientOverride?: any,
+  ): Promise<string>;
 }
 
 /**

package/src/providers/openai-provider.ts

@@ -35,6 +35,30 @@ export async function translate(
   return response.choices[0].message.content.trim();
 }
 
+export async function detectLanguage(
+  text: string,
+  config: TransduckConfig,
+  _clientOverride?: any,
+): Promise<string> {
+  const apiKey = process.env[config.apiKeyEnv];
+  const client = _clientOverride ?? new OpenAI({
+    apiKey,
+    timeout: config.backendTimeout * 1000,
+    maxRetries: config.backendMaxRetries,
+  });
+
+  const response = await client.chat.completions.create({
+    model: config.backendModel,
+    messages: [
+      { role: 'system', content: 'What language is this text written in? Return only the uppercase ISO 639-1 code.' },
+      { role: 'user', content: text },
+    ],
+    temperature: 0.0,
+  });
+
+  return response.choices[0].message.content.trim().toUpperCase();
+}
+
 export async function translatePlural(
   one: string,
   other: string,

package/src/result.ts

@@ -0,0 +1,30 @@
+/**
+ * A string wrapper that carries translation metadata.
+ *
+ * Works in template literals, concatenation, and equality checks via
+ * toString() and valueOf() overrides, plus Symbol.toPrimitive.
+ */
+export class TranslationResult extends String {
+  readonly pending: boolean;
+  readonly sourceLang: string;
+  readonly lang: string;
+
+  constructor(text: string, opts: { pending: boolean; sourceLang: string; lang: string }) {
+    super(text);
+    this.pending = opts.pending;
+    this.sourceLang = opts.sourceLang;
+    this.lang = opts.lang;
+  }
+
+  override toString(): string {
+    return super.toString();
+  }
+
+  override valueOf(): string {
+    return super.valueOf();
+  }
+
+  [Symbol.toPrimitive](_hint: string): string {
+    return super.valueOf();
+  }
+}

package/src/scanner.ts

@@ -57,10 +57,28 @@ function shouldSkipDir(dirname: string): boolean {
   return false;
 }
 
+/**
+ * Join adjacent string literals so the regex can match multi-line strings.
+ *
+ * Handles:
+ * - Python implicit concatenation: "foo" \n "bar" -> "foobar"
+ * - JS + concatenation: "foo" + \n "bar" -> "foobar"
+ * - Both single and double quotes
+ */
+function normalizeMultilineStrings(content: string): string {
+  // Join adjacent string literals: "..." \s+ "..." or '...' \s+ '...'
+  // Also handles "..." + \s+ "..." for JS
+  return content.replace(
+    /(['"])\s*\+?\s*\n\s*\1/g,
+    '',
+  );
+}
+
 /**
  * Extract translatable strings from file content.
  */
 export function extractStrings(content: string, filename: string): ScanEntry[] {
+  content = normalizeMultilineStrings(content);
   const results: ScanEntry[] = [];
   const ext = extname(filename).toLowerCase();
   const isJs = JS_EXTENSIONS.has(ext);