transduck 0.0.1

package/src/index.ts

@@ -0,0 +1,251 @@
+import { createHash } from 'crypto';
+import { loadConfig, type TransduckConfig } from './config.js';
+import { TranslationStore } from './storage.js';
+import { translate as backendTranslate, translatePlural as backendTranslatePlural } from './backend.js';
+import { validateTranslation, extractPlaceholders } from './validation.js';
+import { getPluralCategory, getPluralCategories, interpolateVars } from './plural.js';
+
+interface State {
+  config: TransduckConfig | null;
+  store: TranslationStore | null;
+  targetLang: string | null;
+  pendingTranslations: Map<string, Promise<string>>;
+}
+
+let state: State = {
+  config: null,
+  store: null,
+  targetLang: null,
+  pendingTranslations: new Map(),
+};
+
+function hash(text: string): string {
+  return createHash('sha256').update(text).digest('hex');
+}
+
+export async function initialize(config?: TransduckConfig): Promise<void> {
+  const cfg = config ?? loadConfig();
+  const store = new TranslationStore(cfg.storagePath);
+  await store.initialize();
+  state.config = cfg;
+  state.store = store;
+}
+
+export function setLanguage(lang: string): void {
+  state.targetLang = lang.toUpperCase();
+}
+
+export function _resetState(): void {
+  state = { config: null, store: null, targetLang: null, pendingTranslations: new Map() };
+}
+
+export function _getStore(): TranslationStore | null {
+  return state.store;
+}
+
+export async function ait(
+  sourceText: string,
+  context?: string,
+  vars?: Record<string, string | number>,
+): Promise<string> {
+  if (!state.config || !state.store) {
+    throw new Error('transduck not initialized. Call initialize() first.');
+  }
+  if (!state.targetLang) {
+    throw new Error('Target language not set. Call setLanguage() first.');
+  }
+
+  const cfg = state.config;
+
+  if (state.targetLang === cfg.sourceLang) {
+    return interpolateVars(sourceText, vars);
+  }
+
+  const projectContextHash = hash(cfg.projectContext);
+  const stringContextHash = hash(context ?? '');
+
+  // Cache lookup
+  const cached = await state.store.lookup({
+    sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+    projectContextHash, stringContextHash,
+  });
+  if (cached !== null) return interpolateVars(cached, vars);
+
+  // In-process dedup
+  const lockKey = `${sourceText}|${cfg.sourceLang}|${state.targetLang}|${projectContextHash}|${stringContextHash}`;
+
+  if (state.pendingTranslations.has(lockKey)) {
+    const pending = await state.pendingTranslations.get(lockKey)!;
+    return interpolateVars(pending, vars);
+  }
+
+  const translationPromise = (async () => {
+    // Double-check after getting in
+    const rechecked = await state.store!.lookup({
+      sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
+      projectContextHash, stringContextHash,
+    });
+    if (rechecked !== null) return rechecked;
+
+    const apiKey = process.env[cfg.apiKeyEnv];
+    try {
+      const translated = await backendTranslate({
+        sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
+        projectContext: cfg.projectContext, stringContext: context ?? null,
+        apiKey: apiKey!, model: cfg.backendModel,
+        timeout: cfg.backendTimeout, maxRetries: cfg.backendMaxRetries,
+      });
+
+      if (!validateTranslation(sourceText, translated)) {
+        console.warn(`[transduck] Validation failed for: ${sourceText} -> ${translated}`);
+        await state.store!.insert({
+          sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
+          projectContextHash, stringContextHash,
+          translatedText: translated, model: cfg.backendModel, status: 'failed',
+        });
+        return sourceText;
+      }
+
+      await state.store!.insert({
+        sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
+        projectContextHash, stringContextHash,
+        translatedText: translated, model: cfg.backendModel, status: 'translated',
+      });
+      return translated;
+    } catch (err) {
+      console.warn(`[transduck] Backend failed for: ${sourceText}`, err);
+      return sourceText;
+    } finally {
+      state.pendingTranslations.delete(lockKey);
+    }
+  })();
+
+  state.pendingTranslations.set(lockKey, translationPromise);
+  const result = await translationPromise;
+  return interpolateVars(result, vars);
+}
+
+export async function aitPlural(
+  one: string,
+  other: string,
+  count: number,
+  opts?: { context?: string; vars?: Record<string, string | number> },
+): Promise<string> {
+  if (!state.config || !state.store) {
+    throw new Error('transduck not initialized. Call initialize() first.');
+  }
+  if (!state.targetLang) {
+    throw new Error('Target language not set. Call setLanguage() first.');
+  }
+
+  const cfg = state.config;
+  const context = opts?.context;
+
+  // Build vars with count
+  let vars: Record<string, string | number>;
+  if (!opts?.vars) {
+    vars = { count };
+  } else if (!('count' in opts.vars)) {
+    vars = { ...opts.vars, count };
+  } else {
+    vars = { ...opts.vars };
+  }
+
+  // Same language, 2-form language: select directly from provided forms
+  if (state.targetLang === cfg.sourceLang) {
+    const categories = getPluralCategories(cfg.sourceLang);
+    if (categories.size <= 2) {
+      const category = getPluralCategory(cfg.sourceLang, count);
+      const form = category === 'one' ? one : other;
+      return interpolateVars(form, vars);
+    }
+  }
+
+  // Build cache key
+  const sourceKey = one + '\x00' + other;
+  const projectContextHash = hash(cfg.projectContext);
+  const stringContextHash = hash(context ?? '');
+
+  // Cache lookup
+  const cached = await state.store.lookupPlural({
+    sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+    projectContextHash, stringContextHash,
+  });
+
+  const category = getPluralCategory(state.targetLang, count);
+  if (category in cached) {
+    return interpolateVars(cached[category], vars);
+  }
+
+  // Cache miss — call backend
+  const apiKey = process.env[cfg.apiKeyEnv];
+  try {
+    const forms = await backendTranslatePlural({
+      one, other,
+      sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+      projectContext: cfg.projectContext, stringContext: context ?? null,
+      apiKey: apiKey!, model: cfg.backendModel,
+      timeout: cfg.backendTimeout, maxRetries: cfg.backendMaxRetries,
+    });
+
+    // Validate and store each form
+    const validCategories = new Set(['zero', 'one', 'two', 'few', 'many', 'other']);
+    const sourcePlaceholders = new Set([
+      ...extractPlaceholders(one),
+      ...extractPlaceholders(other),
+    ]);
+
+    if (typeof forms !== 'object' || forms === null || !('other' in forms)) {
+      console.warn(`[transduck] Invalid plural response for: ${one} / ${other}`);
+      const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+      const fallback = fallbackCategory === 'one' ? one : other;
+      return interpolateVars(fallback, vars);
+    }
+
+    for (const [cat, text] of Object.entries(forms)) {
+      if (!validCategories.has(cat) || !text) continue;
+      // Validate placeholders
+      const translatedPlaceholders = extractPlaceholders(text);
+      let allPresent = true;
+      for (const p of sourcePlaceholders) {
+        if (!translatedPlaceholders.has(p)) {
+          allPresent = false;
+          break;
+        }
+      }
+      const status = allPresent ? 'translated' : 'failed';
+      await state.store!.insertPlural({
+        sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang!,
+        projectContextHash, stringContextHash,
+        pluralCategory: cat, translatedText: text,
+        model: cfg.backendModel, status,
+      });
+    }
+
+    // Select the right form
+    if (category in forms) {
+      const text = forms[category];
+      const tp = extractPlaceholders(text);
+      let allPresent = true;
+      for (const p of sourcePlaceholders) {
+        if (!tp.has(p)) {
+          allPresent = false;
+          break;
+        }
+      }
+      if (allPresent) {
+        return interpolateVars(text, vars);
+      }
+    }
+
+    // Fallback
+    const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+    const fallback = fallbackCategory === 'one' ? one : other;
+    return interpolateVars(fallback, vars);
+  } catch (err) {
+    console.warn(`[transduck] Backend failed for plural: ${one} / ${other}`, err);
+    const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+    const fallback = fallbackCategory === 'one' ? one : other;
+    return interpolateVars(fallback, vars);
+  }
+}

package/src/plural.ts

@@ -0,0 +1,47 @@
+/**
+ * CLDR plural rule helpers and variable interpolation.
+ */
+
+/**
+ * Get the CLDR plural category for a count in the given language.
+ * Uses the built-in Intl.PluralRules API.
+ */
+export function getPluralCategory(lang: string, count: number): string {
+  const rules = new Intl.PluralRules(lang.toLowerCase());
+  return rules.select(count);
+}
+
+/**
+ * Get all CLDR plural categories used by a language.
+ * Since Intl.PluralRules doesn't directly expose this, we probe with test values
+ * that cover all CLDR categories.
+ */
+export function getPluralCategories(lang: string): Set<string> {
+  const rules = new Intl.PluralRules(lang.toLowerCase());
+  // Test values that trigger different categories across languages
+  // Include floats to cover languages where "other" is only triggered by non-integers (e.g., Russian)
+  const testValues = [0, 0.5, 1, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 100, 101, 102];
+  const categories = new Set<string>();
+  for (const n of testValues) {
+    categories.add(rules.select(n));
+  }
+  // 'other' is always a valid CLDR category
+  categories.add('other');
+  return categories;
+}
+
+/**
+ * Safely interpolate variables into a translated string.
+ * Replaces {key} with the corresponding value from vars.
+ */
+export function interpolateVars(
+  text: string,
+  vars?: Record<string, string | number> | null,
+): string {
+  if (!vars) return text;
+  let result = text;
+  for (const [key, value] of Object.entries(vars)) {
+    result = result.replaceAll(`{${key}}`, String(value));
+  }
+  return result;
+}

package/src/storage.ts

@@ -0,0 +1,229 @@
+import { DuckDBInstance, DuckDBResultReader } from '@duckdb/node-api';
+import type { DuckDBConnection } from '@duckdb/node-api';
+
+const SCHEMA_V2 = `
+CREATE TABLE IF NOT EXISTS translations (
+    source_text          TEXT NOT NULL,
+    source_lang          TEXT NOT NULL,
+    target_lang          TEXT NOT NULL,
+    project_context_hash TEXT NOT NULL,
+    string_context_hash  TEXT NOT NULL,
+    plural_category      TEXT NOT NULL DEFAULT '',
+    translated_text      TEXT NOT NULL,
+    model                TEXT NOT NULL,
+    status               TEXT NOT NULL DEFAULT 'translated',
+    created_at           TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(source_text, source_lang, target_lang, project_context_hash, string_context_hash, plural_category)
+);
+`;
+
+const MIGRATION_V1_TO_V2 = `
+CREATE TABLE translations_v2 (
+    source_text          TEXT NOT NULL,
+    source_lang          TEXT NOT NULL,
+    target_lang          TEXT NOT NULL,
+    project_context_hash TEXT NOT NULL,
+    string_context_hash  TEXT NOT NULL,
+    plural_category      TEXT NOT NULL DEFAULT '',
+    translated_text      TEXT NOT NULL,
+    model                TEXT NOT NULL,
+    status               TEXT NOT NULL DEFAULT 'translated',
+    created_at           TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(source_text, source_lang, target_lang, project_context_hash, string_context_hash, plural_category)
+);
+INSERT INTO translations_v2
+    SELECT source_text, source_lang, target_lang, project_context_hash,
+           string_context_hash, '' as plural_category, translated_text,
+           model, status, created_at
+    FROM translations;
+DROP TABLE translations;
+ALTER TABLE translations_v2 RENAME TO translations;
+`;
+
+const CHECK_PLURAL_COLUMN = `
+SELECT column_name FROM information_schema.columns
+WHERE table_name = 'translations' AND column_name = 'plural_category'
+`;
+
+const LOOKUP_SQL = `
+SELECT translated_text FROM translations
+WHERE source_text = $1 AND source_lang = $2 AND target_lang = $3
+  AND project_context_hash = $4 AND string_context_hash = $5
+  AND plural_category = '' AND status = 'translated'
+`;
+
+const INSERT_SQL = `
+INSERT INTO translations
+    (source_text, source_lang, target_lang, project_context_hash,
+     string_context_hash, plural_category, translated_text, model, status)
+VALUES ($1, $2, $3, $4, $5, '', $6, $7, $8)
+ON CONFLICT DO NOTHING
+`;
+
+const LOOKUP_PLURAL_SQL = `
+SELECT plural_category, translated_text FROM translations
+WHERE source_text = $1 AND source_lang = $2 AND target_lang = $3
+  AND project_context_hash = $4 AND string_context_hash = $5
+  AND plural_category != '' AND status = 'translated'
+`;
+
+const INSERT_PLURAL_SQL = `
+INSERT INTO translations
+    (source_text, source_lang, target_lang, project_context_hash,
+     string_context_hash, plural_category, translated_text, model, status)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT DO NOTHING
+`;
+
+export interface LookupParams {
+  sourceText: string;
+  sourceLang: string;
+  targetLang: string;
+  projectContextHash: string;
+  stringContextHash: string;
+}
+
+export interface InsertParams extends LookupParams {
+  translatedText: string;
+  model: string;
+  status: string;
+}
+
+export interface InsertPluralParams extends LookupParams {
+  pluralCategory: string;
+  translatedText: string;
+  model: string;
+  status: string;
+}
+
+interface Stats {
+  totalTranslations: number;
+  totalFailed: number;
+  byLanguage: Record<string, number>;
+}
+
+export class TranslationStore {
+  private dbPath: string;
+  private instance: DuckDBInstance | null = null;
+  private conn: DuckDBConnection | null = null;
+
+  constructor(dbPath: string) {
+    this.dbPath = dbPath;
+  }
+
+  private async getConn(): Promise<DuckDBConnection> {
+    if (!this.conn) throw new Error('Store not initialized');
+    return this.conn;
+  }
+
+  private convertRow(row: Record<string, unknown>): Record<string, unknown> {
+    const converted: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(row)) {
+      if (typeof value === 'bigint') {
+        converted[key] = Number(value);
+      } else {
+        converted[key] = value;
+      }
+    }
+    return converted;
+  }
+
+  async query(sql: string, params: string[] = []): Promise<Record<string, unknown>[]> {
+    const conn = await this.getConn();
+    let result;
+    if (params.length === 0) {
+      result = await conn.run(sql);
+    } else {
+      const stmt = await conn.prepare(sql);
+      for (let i = 0; i < params.length; i++) {
+        stmt.bindVarchar(i + 1, params[i]);
+      }
+      result = await stmt.run();
+    }
+    const reader = new DuckDBResultReader(result);
+    await reader.readAll();
+    return [...reader.getRowObjects()].map(row => this.convertRow(row));
+  }
+
+  async initialize(): Promise<void> {
+    this.instance = await DuckDBInstance.create(this.dbPath);
+    this.conn = await this.instance.connect();
+
+    // Check if table exists
+    const tables = await this.query(
+      "SELECT table_name FROM information_schema.tables WHERE table_name = 'translations'"
+    );
+
+    if (tables.length === 0) {
+      // Fresh database — create v2 schema
+      await this.conn.run(SCHEMA_V2);
+      return;
+    }
+
+    // Table exists — check if it has plural_category (v2)
+    const hasPlural = await this.query(CHECK_PLURAL_COLUMN);
+    if (hasPlural.length === 0) {
+      // Migrate from v1 to v2
+      await this.conn.run(MIGRATION_V1_TO_V2);
+    }
+  }
+
+  async lookup(params: LookupParams): Promise<string | null> {
+    const rows = await this.query(LOOKUP_SQL, [
+      params.sourceText, params.sourceLang, params.targetLang,
+      params.projectContextHash, params.stringContextHash,
+    ]);
+    return rows.length > 0 ? (rows[0].translated_text as string) : null;
+  }
+
+  async insert(params: InsertParams): Promise<void> {
+    await this.query(INSERT_SQL, [
+      params.sourceText, params.sourceLang, params.targetLang,
+      params.projectContextHash, params.stringContextHash,
+      params.translatedText, params.model, params.status,
+    ]);
+  }
+
+  async lookupPlural(params: LookupParams): Promise<Record<string, string>> {
+    const rows = await this.query(LOOKUP_PLURAL_SQL, [
+      params.sourceText, params.sourceLang, params.targetLang,
+      params.projectContextHash, params.stringContextHash,
+    ]);
+    const result: Record<string, string> = {};
+    for (const row of rows) {
+      result[row.plural_category as string] = row.translated_text as string;
+    }
+    return result;
+  }
+
+  async insertPlural(params: InsertPluralParams): Promise<void> {
+    await this.query(INSERT_PLURAL_SQL, [
+      params.sourceText, params.sourceLang, params.targetLang,
+      params.projectContextHash, params.stringContextHash,
+      params.pluralCategory, params.translatedText, params.model, params.status,
+    ]);
+  }
+
+  async stats(): Promise<Stats> {
+    const totalRows = await this.query("SELECT count(*) as c FROM translations WHERE status = 'translated'");
+    const failedRows = await this.query("SELECT count(*) as c FROM translations WHERE status = 'failed'");
+    const langRows = await this.query(
+      "SELECT target_lang, count(*) as c FROM translations WHERE status = 'translated' GROUP BY target_lang"
+    );
+    const byLanguage: Record<string, number> = {};
+    for (const row of langRows) {
+      byLanguage[row.target_lang as string] = Number(row.c);
+    }
+    return {
+      totalTranslations: Number(totalRows[0].c),
+      totalFailed: Number(failedRows[0].c),
+      byLanguage,
+    };
+  }
+
+  close(): void {
+    // @duckdb/node-api handles cleanup via GC
+    this.conn = null;
+    this.instance = null;
+  }
+}

package/src/validation.ts

@@ -0,0 +1,30 @@
+const PLACEHOLDER_PATTERNS = [
+  /\{\{[^}]*\}\}/g,           // {{ count }}
+  /\$\{[^}]+\}/g,             // ${value}
+  /(?<!\{)(?<!\$)\{[^{}]+\}(?!\})/g,  // {name} but not {{x}} or ${x}
+  /%[sd]/g,                    // %s, %d
+];
+
+export function extractPlaceholders(text: string): Set<string> {
+  const result = new Set<string>();
+  for (const pattern of PLACEHOLDER_PATTERNS) {
+    const matches = text.match(pattern);
+    if (matches) {
+      for (const m of matches) result.add(m);
+    }
+  }
+  return result;
+}
+
+export function validateTranslation(sourceText: string, translatedText: string): boolean {
+  if (!translatedText || !translatedText.trim()) return false;
+
+  const sourcePlaceholders = extractPlaceholders(sourceText);
+  if (sourcePlaceholders.size === 0) return true;
+
+  const translatedPlaceholders = extractPlaceholders(translatedText);
+  for (const p of sourcePlaceholders) {
+    if (!translatedPlaceholders.has(p)) return false;
+  }
+  return true;
+}