transduck 0.0.1

package/dist/index.js

@@ -0,0 +1,212 @@
+import { createHash } from 'crypto';
+import { loadConfig } from './config.js';
+import { TranslationStore } from './storage.js';
+import { translate as backendTranslate, translatePlural as backendTranslatePlural } from './backend.js';
+import { validateTranslation, extractPlaceholders } from './validation.js';
+import { getPluralCategory, getPluralCategories, interpolateVars } from './plural.js';
+let state = {
+    config: null,
+    store: null,
+    targetLang: null,
+    pendingTranslations: new Map(),
+};
+function hash(text) {
+    return createHash('sha256').update(text).digest('hex');
+}
+export async function initialize(config) {
+    const cfg = config ?? loadConfig();
+    const store = new TranslationStore(cfg.storagePath);
+    await store.initialize();
+    state.config = cfg;
+    state.store = store;
+}
+export function setLanguage(lang) {
+    state.targetLang = lang.toUpperCase();
+}
+export function _resetState() {
+    state = { config: null, store: null, targetLang: null, pendingTranslations: new Map() };
+}
+export function _getStore() {
+    return state.store;
+}
+export async function ait(sourceText, context, vars) {
+    if (!state.config || !state.store) {
+        throw new Error('transduck not initialized. Call initialize() first.');
+    }
+    if (!state.targetLang) {
+        throw new Error('Target language not set. Call setLanguage() first.');
+    }
+    const cfg = state.config;
+    if (state.targetLang === cfg.sourceLang) {
+        return interpolateVars(sourceText, vars);
+    }
+    const projectContextHash = hash(cfg.projectContext);
+    const stringContextHash = hash(context ?? '');
+    // Cache lookup
+    const cached = await state.store.lookup({
+        sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+        projectContextHash, stringContextHash,
+    });
+    if (cached !== null)
+        return interpolateVars(cached, vars);
+    // In-process dedup
+    const lockKey = `${sourceText}|${cfg.sourceLang}|${state.targetLang}|${projectContextHash}|${stringContextHash}`;
+    if (state.pendingTranslations.has(lockKey)) {
+        const pending = await state.pendingTranslations.get(lockKey);
+        return interpolateVars(pending, vars);
+    }
+    const translationPromise = (async () => {
+        // Double-check after getting in
+        const rechecked = await state.store.lookup({
+            sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+            projectContextHash, stringContextHash,
+        });
+        if (rechecked !== null)
+            return rechecked;
+        const apiKey = process.env[cfg.apiKeyEnv];
+        try {
+            const translated = await backendTranslate({
+                sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+                projectContext: cfg.projectContext, stringContext: context ?? null,
+                apiKey: apiKey, model: cfg.backendModel,
+                timeout: cfg.backendTimeout, maxRetries: cfg.backendMaxRetries,
+            });
+            if (!validateTranslation(sourceText, translated)) {
+                console.warn(`[transduck] Validation failed for: ${sourceText} -> ${translated}`);
+                await state.store.insert({
+                    sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+                    projectContextHash, stringContextHash,
+                    translatedText: translated, model: cfg.backendModel, status: 'failed',
+                });
+                return sourceText;
+            }
+            await state.store.insert({
+                sourceText, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+                projectContextHash, stringContextHash,
+                translatedText: translated, model: cfg.backendModel, status: 'translated',
+            });
+            return translated;
+        }
+        catch (err) {
+            console.warn(`[transduck] Backend failed for: ${sourceText}`, err);
+            return sourceText;
+        }
+        finally {
+            state.pendingTranslations.delete(lockKey);
+        }
+    })();
+    state.pendingTranslations.set(lockKey, translationPromise);
+    const result = await translationPromise;
+    return interpolateVars(result, vars);
+}
+export async function aitPlural(one, other, count, opts) {
+    if (!state.config || !state.store) {
+        throw new Error('transduck not initialized. Call initialize() first.');
+    }
+    if (!state.targetLang) {
+        throw new Error('Target language not set. Call setLanguage() first.');
+    }
+    const cfg = state.config;
+    const context = opts?.context;
+    // Build vars with count
+    let vars;
+    if (!opts?.vars) {
+        vars = { count };
+    }
+    else if (!('count' in opts.vars)) {
+        vars = { ...opts.vars, count };
+    }
+    else {
+        vars = { ...opts.vars };
+    }
+    // Same language, 2-form language: select directly from provided forms
+    if (state.targetLang === cfg.sourceLang) {
+        const categories = getPluralCategories(cfg.sourceLang);
+        if (categories.size <= 2) {
+            const category = getPluralCategory(cfg.sourceLang, count);
+            const form = category === 'one' ? one : other;
+            return interpolateVars(form, vars);
+        }
+    }
+    // Build cache key
+    const sourceKey = one + '\x00' + other;
+    const projectContextHash = hash(cfg.projectContext);
+    const stringContextHash = hash(context ?? '');
+    // Cache lookup
+    const cached = await state.store.lookupPlural({
+        sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+        projectContextHash, stringContextHash,
+    });
+    const category = getPluralCategory(state.targetLang, count);
+    if (category in cached) {
+        return interpolateVars(cached[category], vars);
+    }
+    // Cache miss — call backend
+    const apiKey = process.env[cfg.apiKeyEnv];
+    try {
+        const forms = await backendTranslatePlural({
+            one, other,
+            sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+            projectContext: cfg.projectContext, stringContext: context ?? null,
+            apiKey: apiKey, model: cfg.backendModel,
+            timeout: cfg.backendTimeout, maxRetries: cfg.backendMaxRetries,
+        });
+        // Validate and store each form
+        const validCategories = new Set(['zero', 'one', 'two', 'few', 'many', 'other']);
+        const sourcePlaceholders = new Set([
+            ...extractPlaceholders(one),
+            ...extractPlaceholders(other),
+        ]);
+        if (typeof forms !== 'object' || forms === null || !('other' in forms)) {
+            console.warn(`[transduck] Invalid plural response for: ${one} / ${other}`);
+            const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+            const fallback = fallbackCategory === 'one' ? one : other;
+            return interpolateVars(fallback, vars);
+        }
+        for (const [cat, text] of Object.entries(forms)) {
+            if (!validCategories.has(cat) || !text)
+                continue;
+            // Validate placeholders
+            const translatedPlaceholders = extractPlaceholders(text);
+            let allPresent = true;
+            for (const p of sourcePlaceholders) {
+                if (!translatedPlaceholders.has(p)) {
+                    allPresent = false;
+                    break;
+                }
+            }
+            const status = allPresent ? 'translated' : 'failed';
+            await state.store.insertPlural({
+                sourceText: sourceKey, sourceLang: cfg.sourceLang, targetLang: state.targetLang,
+                projectContextHash, stringContextHash,
+                pluralCategory: cat, translatedText: text,
+                model: cfg.backendModel, status,
+            });
+        }
+        // Select the right form
+        if (category in forms) {
+            const text = forms[category];
+            const tp = extractPlaceholders(text);
+            let allPresent = true;
+            for (const p of sourcePlaceholders) {
+                if (!tp.has(p)) {
+                    allPresent = false;
+                    break;
+                }
+            }
+            if (allPresent) {
+                return interpolateVars(text, vars);
+            }
+        }
+        // Fallback
+        const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+        const fallback = fallbackCategory === 'one' ? one : other;
+        return interpolateVars(fallback, vars);
+    }
+    catch (err) {
+        console.warn(`[transduck] Backend failed for plural: ${one} / ${other}`, err);
+        const fallbackCategory = getPluralCategory(cfg.sourceLang, count);
+        const fallback = fallbackCategory === 'one' ? one : other;
+        return interpolateVars(fallback, vars);
+    }
+}

package/dist/plural.d.ts

@@ -0,0 +1,19 @@
+/**
+ * CLDR plural rule helpers and variable interpolation.
+ */
+/**
+ * Get the CLDR plural category for a count in the given language.
+ * Uses the built-in Intl.PluralRules API.
+ */
+export declare function getPluralCategory(lang: string, count: number): string;
+/**
+ * Get all CLDR plural categories used by a language.
+ * Since Intl.PluralRules doesn't directly expose this, we probe with test values
+ * that cover all CLDR categories.
+ */
+export declare function getPluralCategories(lang: string): Set<string>;
+/**
+ * Safely interpolate variables into a translated string.
+ * Replaces {key} with the corresponding value from vars.
+ */
+export declare function interpolateVars(text: string, vars?: Record<string, string | number> | null): string;

package/dist/plural.js

@@ -0,0 +1,42 @@
+/**
+ * CLDR plural rule helpers and variable interpolation.
+ */
+/**
+ * Get the CLDR plural category for a count in the given language.
+ * Uses the built-in Intl.PluralRules API.
+ */
+export function getPluralCategory(lang, count) {
+    const rules = new Intl.PluralRules(lang.toLowerCase());
+    return rules.select(count);
+}
+/**
+ * Get all CLDR plural categories used by a language.
+ * Since Intl.PluralRules doesn't directly expose this, we probe with test values
+ * that cover all CLDR categories.
+ */
+export function getPluralCategories(lang) {
+    const rules = new Intl.PluralRules(lang.toLowerCase());
+    // Test values that trigger different categories across languages
+    // Include floats to cover languages where "other" is only triggered by non-integers (e.g., Russian)
+    const testValues = [0, 0.5, 1, 1.5, 2, 2.5, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 100, 101, 102];
+    const categories = new Set();
+    for (const n of testValues) {
+        categories.add(rules.select(n));
+    }
+    // 'other' is always a valid CLDR category
+    categories.add('other');
+    return categories;
+}
+/**
+ * Safely interpolate variables into a translated string.
+ * Replaces {key} with the corresponding value from vars.
+ */
+export function interpolateVars(text, vars) {
+    if (!vars)
+        return text;
+    let result = text;
+    for (const [key, value] of Object.entries(vars)) {
+        result = result.replaceAll(`{${key}}`, String(value));
+    }
+    return result;
+}

package/dist/storage.d.ts

@@ -0,0 +1,40 @@
+export interface LookupParams {
+    sourceText: string;
+    sourceLang: string;
+    targetLang: string;
+    projectContextHash: string;
+    stringContextHash: string;
+}
+export interface InsertParams extends LookupParams {
+    translatedText: string;
+    model: string;
+    status: string;
+}
+export interface InsertPluralParams extends LookupParams {
+    pluralCategory: string;
+    translatedText: string;
+    model: string;
+    status: string;
+}
+interface Stats {
+    totalTranslations: number;
+    totalFailed: number;
+    byLanguage: Record<string, number>;
+}
+export declare class TranslationStore {
+    private dbPath;
+    private instance;
+    private conn;
+    constructor(dbPath: string);
+    private getConn;
+    private convertRow;
+    query(sql: string, params?: string[]): Promise<Record<string, unknown>[]>;
+    initialize(): Promise<void>;
+    lookup(params: LookupParams): Promise<string | null>;
+    insert(params: InsertParams): Promise<void>;
+    lookupPlural(params: LookupParams): Promise<Record<string, string>>;
+    insertPlural(params: InsertPluralParams): Promise<void>;
+    stats(): Promise<Stats>;
+    close(): void;
+}
+export {};

package/dist/storage.js

@@ -0,0 +1,178 @@
+import { DuckDBInstance, DuckDBResultReader } from '@duckdb/node-api';
+const SCHEMA_V2 = `
+CREATE TABLE IF NOT EXISTS translations (
+    source_text          TEXT NOT NULL,
+    source_lang          TEXT NOT NULL,
+    target_lang          TEXT NOT NULL,
+    project_context_hash TEXT NOT NULL,
+    string_context_hash  TEXT NOT NULL,
+    plural_category      TEXT NOT NULL DEFAULT '',
+    translated_text      TEXT NOT NULL,
+    model                TEXT NOT NULL,
+    status               TEXT NOT NULL DEFAULT 'translated',
+    created_at           TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(source_text, source_lang, target_lang, project_context_hash, string_context_hash, plural_category)
+);
+`;
+const MIGRATION_V1_TO_V2 = `
+CREATE TABLE translations_v2 (
+    source_text          TEXT NOT NULL,
+    source_lang          TEXT NOT NULL,
+    target_lang          TEXT NOT NULL,
+    project_context_hash TEXT NOT NULL,
+    string_context_hash  TEXT NOT NULL,
+    plural_category      TEXT NOT NULL DEFAULT '',
+    translated_text      TEXT NOT NULL,
+    model                TEXT NOT NULL,
+    status               TEXT NOT NULL DEFAULT 'translated',
+    created_at           TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(source_text, source_lang, target_lang, project_context_hash, string_context_hash, plural_category)
+);
+INSERT INTO translations_v2
+    SELECT source_text, source_lang, target_lang, project_context_hash,
+           string_context_hash, '' as plural_category, translated_text,
+           model, status, created_at
+    FROM translations;
+DROP TABLE translations;
+ALTER TABLE translations_v2 RENAME TO translations;
+`;
+const CHECK_PLURAL_COLUMN = `
+SELECT column_name FROM information_schema.columns
+WHERE table_name = 'translations' AND column_name = 'plural_category'
+`;
+const LOOKUP_SQL = `
+SELECT translated_text FROM translations
+WHERE source_text = $1 AND source_lang = $2 AND target_lang = $3
+  AND project_context_hash = $4 AND string_context_hash = $5
+  AND plural_category = '' AND status = 'translated'
+`;
+const INSERT_SQL = `
+INSERT INTO translations
+    (source_text, source_lang, target_lang, project_context_hash,
+     string_context_hash, plural_category, translated_text, model, status)
+VALUES ($1, $2, $3, $4, $5, '', $6, $7, $8)
+ON CONFLICT DO NOTHING
+`;
+const LOOKUP_PLURAL_SQL = `
+SELECT plural_category, translated_text FROM translations
+WHERE source_text = $1 AND source_lang = $2 AND target_lang = $3
+  AND project_context_hash = $4 AND string_context_hash = $5
+  AND plural_category != '' AND status = 'translated'
+`;
+const INSERT_PLURAL_SQL = `
+INSERT INTO translations
+    (source_text, source_lang, target_lang, project_context_hash,
+     string_context_hash, plural_category, translated_text, model, status)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ON CONFLICT DO NOTHING
+`;
+export class TranslationStore {
+    dbPath;
+    instance = null;
+    conn = null;
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    async getConn() {
+        if (!this.conn)
+            throw new Error('Store not initialized');
+        return this.conn;
+    }
+    convertRow(row) {
+        const converted = {};
+        for (const [key, value] of Object.entries(row)) {
+            if (typeof value === 'bigint') {
+                converted[key] = Number(value);
+            }
+            else {
+                converted[key] = value;
+            }
+        }
+        return converted;
+    }
+    async query(sql, params = []) {
+        const conn = await this.getConn();
+        let result;
+        if (params.length === 0) {
+            result = await conn.run(sql);
+        }
+        else {
+            const stmt = await conn.prepare(sql);
+            for (let i = 0; i < params.length; i++) {
+                stmt.bindVarchar(i + 1, params[i]);
+            }
+            result = await stmt.run();
+        }
+        const reader = new DuckDBResultReader(result);
+        await reader.readAll();
+        return [...reader.getRowObjects()].map(row => this.convertRow(row));
+    }
+    async initialize() {
+        this.instance = await DuckDBInstance.create(this.dbPath);
+        this.conn = await this.instance.connect();
+        // Check if table exists
+        const tables = await this.query("SELECT table_name FROM information_schema.tables WHERE table_name = 'translations'");
+        if (tables.length === 0) {
+            // Fresh database — create v2 schema
+            await this.conn.run(SCHEMA_V2);
+            return;
+        }
+        // Table exists — check if it has plural_category (v2)
+        const hasPlural = await this.query(CHECK_PLURAL_COLUMN);
+        if (hasPlural.length === 0) {
+            // Migrate from v1 to v2
+            await this.conn.run(MIGRATION_V1_TO_V2);
+        }
+    }
+    async lookup(params) {
+        const rows = await this.query(LOOKUP_SQL, [
+            params.sourceText, params.sourceLang, params.targetLang,
+            params.projectContextHash, params.stringContextHash,
+        ]);
+        return rows.length > 0 ? rows[0].translated_text : null;
+    }
+    async insert(params) {
+        await this.query(INSERT_SQL, [
+            params.sourceText, params.sourceLang, params.targetLang,
+            params.projectContextHash, params.stringContextHash,
+            params.translatedText, params.model, params.status,
+        ]);
+    }
+    async lookupPlural(params) {
+        const rows = await this.query(LOOKUP_PLURAL_SQL, [
+            params.sourceText, params.sourceLang, params.targetLang,
+            params.projectContextHash, params.stringContextHash,
+        ]);
+        const result = {};
+        for (const row of rows) {
+            result[row.plural_category] = row.translated_text;
+        }
+        return result;
+    }
+    async insertPlural(params) {
+        await this.query(INSERT_PLURAL_SQL, [
+            params.sourceText, params.sourceLang, params.targetLang,
+            params.projectContextHash, params.stringContextHash,
+            params.pluralCategory, params.translatedText, params.model, params.status,
+        ]);
+    }
+    async stats() {
+        const totalRows = await this.query("SELECT count(*) as c FROM translations WHERE status = 'translated'");
+        const failedRows = await this.query("SELECT count(*) as c FROM translations WHERE status = 'failed'");
+        const langRows = await this.query("SELECT target_lang, count(*) as c FROM translations WHERE status = 'translated' GROUP BY target_lang");
+        const byLanguage = {};
+        for (const row of langRows) {
+            byLanguage[row.target_lang] = Number(row.c);
+        }
+        return {
+            totalTranslations: Number(totalRows[0].c),
+            totalFailed: Number(failedRows[0].c),
+            byLanguage,
+        };
+    }
+    close() {
+        // @duckdb/node-api handles cleanup via GC
+        this.conn = null;
+        this.instance = null;
+    }
+}

package/dist/validation.d.ts

@@ -0,0 +1,2 @@
+export declare function extractPlaceholders(text: string): Set<string>;
+export declare function validateTranslation(sourceText: string, translatedText: string): boolean;

package/dist/validation.js

@@ -0,0 +1,30 @@
+const PLACEHOLDER_PATTERNS = [
+    /\{\{[^}]*\}\}/g, // {{ count }}
+    /\$\{[^}]+\}/g, // ${value}
+    /(?<!\{)(?<!\$)\{[^{}]+\}(?!\})/g, // {name} but not {{x}} or ${x}
+    /%[sd]/g, // %s, %d
+];
+export function extractPlaceholders(text) {
+    const result = new Set();
+    for (const pattern of PLACEHOLDER_PATTERNS) {
+        const matches = text.match(pattern);
+        if (matches) {
+            for (const m of matches)
+                result.add(m);
+        }
+    }
+    return result;
+}
+export function validateTranslation(sourceText, translatedText) {
+    if (!translatedText || !translatedText.trim())
+        return false;
+    const sourcePlaceholders = extractPlaceholders(sourceText);
+    if (sourcePlaceholders.size === 0)
+        return true;
+    const translatedPlaceholders = extractPlaceholders(translatedText);
+    for (const p of sourcePlaceholders) {
+        if (!translatedPlaceholders.has(p))
+            return false;
+    }
+    return true;
+}

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "transduck",
+  "version": "0.0.1",
+  "description": "AI-native translation tool using source text as keys",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "transduck": "./dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@duckdb/node-api": "^1.5.0-r.1",
+    "commander": "^12.0.0",
+    "duckdb": "^1.0.0",
+    "openai": "^4.0.0",
+    "yaml": "^2.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^2.0.0"
+  }
+}