transduck 0.0.4 → 0.1.1

package/src/scanner.ts

@@ -0,0 +1,215 @@
+/**
+ * Source code scanner for TransDuck ait() and ait_plural()/aitPlural() calls.
+ */
+
+import { readdirSync, readFileSync, statSync } from 'fs';
+import { join, extname } from 'path';
+
+// --- Types ---
+
+export interface ScanEntry {
+  text?: string;
+  context?: string | null;
+  plural?: true;
+  one?: string;
+  other?: string;
+  line?: number;
+  files?: string[];
+}
+
+// --- Regex patterns ---
+
+// ait("text") or ait("text", context="ctx") — Python/template keyword style
+const AIT_KEYWORD_CTX = /ait\s*\(\s*(['"])(.*?)\1(?:\s*,\s*context\s*=\s*(['"])(.*?)\3)?/g;
+
+// ait("text") or ait("text", "ctx") — JS positional style
+const AIT_POSITIONAL_CTX = /ait\s*\(\s*(['"])(.*?)\1(?:\s*,\s*(['"])(.*?)\3)?/g;
+
+// ait_plural("one", "other") or aitPlural("one", "other")
+const AIT_PLURAL = /ait(?:_p|P)lural\s*\(\s*(['"])(.*?)\1\s*,\s*(['"])(.*?)\3/g;
+
+// {% ait "text" %} or {% ait "text" context="ctx" %}
+const DJANGO_TAG = /\{%\s*ait\s+(['"])(.*?)\1(?:\s+context=(['"])(.*?)\3)?\s*%\}/g;
+
+// t("text") or t("text", "ctx") — only matched in files with transduck/react import
+const T_POSITIONAL = /(?<![a-zA-Z_.$])t\s*\(\s*(['"])(.*?)\1(?:\s*,\s*(['"])(.*?)\3)?/g;
+
+// tPlural("one", "other") — only matched in files with transduck/react import
+const T_PLURAL = /(?<![a-zA-Z_.$])tPlural\s*\(\s*(['"])(.*?)\1\s*,\s*(['"])(.*?)\3/g;
+
+const HAS_TRANSDUCK_REACT = /from\s+['"]transduck\/react['"]/;
+
+// File extensions that use JS-style positional context
+const JS_EXTENSIONS = new Set(['.js', '.ts', '.tsx', '.jsx']);
+
+// File extensions that may contain Django template tags
+const TEMPLATE_EXTENSIONS = new Set(['.html', '.jinja', '.jinja2']);
+
+// Supported file extensions for scanning
+const SCAN_EXTENSIONS = new Set(['.py', '.js', '.ts', '.tsx', '.jsx', '.html', '.jinja', '.jinja2']);
+
+// Directories to skip
+const SKIP_DIRS = new Set(['node_modules', '.venv', 'venv', '__pycache__', '.git', 'dist', 'build', '.next']);
+
+function shouldSkipDir(dirname: string): boolean {
+  if (SKIP_DIRS.has(dirname)) return true;
+  if (dirname.includes('egg-info')) return true;
+  return false;
+}
+
+/**
+ * Extract translatable strings from file content.
+ */
+export function extractStrings(content: string, filename: string): ScanEntry[] {
+  const results: ScanEntry[] = [];
+  const ext = extname(filename).toLowerCase();
+  const isJs = JS_EXTENSIONS.has(ext);
+  const isTemplate = TEMPLATE_EXTENSIONS.has(ext);
+
+  // Track positions of plural matches so we don't double-match them as ait()
+  const pluralSpans: Array<[number, number]> = [];
+
+  // 1. Find all plural calls
+  const pluralRegex = new RegExp(AIT_PLURAL.source, 'g');
+  let match: RegExpExecArray | null;
+  while ((match = pluralRegex.exec(content)) !== null) {
+    const one = match[2];
+    const other = match[4];
+    const lineNum = content.slice(0, match.index).split('\n').length;
+    results.push({
+      plural: true,
+      one,
+      other,
+      context: null,
+      line: lineNum,
+    });
+    pluralSpans.push([match.index, match.index + match[0].length]);
+  }
+
+  // 2. Find Django template tags (only in template files)
+  if (isTemplate) {
+    const djangoRegex = new RegExp(DJANGO_TAG.source, 'g');
+    while ((match = djangoRegex.exec(content)) !== null) {
+      const text = match[2];
+      const context = match[4] || null;
+      const lineNum = content.slice(0, match.index).split('\n').length;
+      results.push({ text, context, line: lineNum });
+    }
+  }
+
+  // 3. Find ait() calls
+  const pattern = isJs ? AIT_POSITIONAL_CTX : AIT_KEYWORD_CTX;
+  const aitRegex = new RegExp(pattern.source, 'g');
+  while ((match = aitRegex.exec(content)) !== null) {
+    // Skip if this is part of a plural match
+    const pos = match.index;
+    if (pluralSpans.some(([start, end]) => pos >= start && pos < end)) {
+      continue;
+    }
+
+    // Check that this is specifically "ait(" not "ait_plural(" or "aitPlural("
+    const prefixStart = Math.max(0, pos - 10);
+    const prefix = content.slice(prefixStart, pos + 4);
+    if (prefix.toLowerCase().includes('plural') || prefix.includes('Plural')) {
+      continue;
+    }
+
+    const text = match[2];
+    const context = match[4] || null;
+    const lineNum = content.slice(0, pos).split('\n').length;
+    results.push({ text, context, line: lineNum });
+  }
+
+  // 4. t() and tPlural() — only in files that import from transduck/react
+  if (HAS_TRANSDUCK_REACT.test(content)) {
+    // t() calls
+    for (const m of content.matchAll(new RegExp(T_POSITIONAL.source, 'g'))) {
+      const pos = m.index!;
+      // Skip if overlaps with plural spans
+      if (pluralSpans.some(([start, end]) => pos >= start && pos < end)) continue;
+      const text = m[2];
+      const context = m[4] || null;
+      const lineNum = content.slice(0, pos).split('\n').length;
+      results.push({ text, context, line: lineNum });
+    }
+    // tPlural() calls
+    for (const m of content.matchAll(new RegExp(T_PLURAL.source, 'g'))) {
+      const one = m[2];
+      const other = m[4];
+      const lineNum = content.slice(0, m.index!).split('\n').length;
+      results.push({ plural: true, one, other, context: null, line: lineNum });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Walk directories and extract all translatable strings.
+ * Returns deduplicated list of entries with 'files' field listing all locations.
+ */
+export function scanDirectory(dirs: string[]): ScanEntry[] {
+  const rawMatches = new Map<string, ScanEntry>();
+
+  for (const scanDir of dirs) {
+    walkDir(scanDir, rawMatches);
+  }
+
+  return Array.from(rawMatches.values());
+}
+
+function walkDir(dir: string, rawMatches: Map<string, ScanEntry>): void {
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return;
+  }
+
+  for (const name of entries) {
+    const fullPath = join(dir, name);
+    let stat;
+    try {
+      stat = statSync(fullPath);
+    } catch {
+      continue;
+    }
+
+    if (stat.isDirectory()) {
+      if (!shouldSkipDir(name)) {
+        walkDir(fullPath, rawMatches);
+      }
+    } else if (stat.isFile()) {
+      const ext = extname(name).toLowerCase();
+      if (!SCAN_EXTENSIONS.has(ext)) continue;
+
+      let content: string;
+      try {
+        content = readFileSync(fullPath, 'utf-8');
+      } catch {
+        continue;
+      }
+
+      const entries = extractStrings(content, name);
+      for (const entry of entries) {
+        // Build dedup key
+        let key: string;
+        if (entry.plural) {
+          key = `plural:${entry.one}\x00${entry.other}\x00${entry.context ?? ''}`;
+        } else {
+          key = `text:${entry.text}\x00${entry.context ?? ''}`;
+        }
+
+        const fileLoc = `${fullPath}:${entry.line}`;
+
+        if (rawMatches.has(key)) {
+          rawMatches.get(key)!.files!.push(fileLoc);
+        } else {
+          const resultEntry: ScanEntry = { ...entry, files: [fileLoc] };
+          delete resultEntry.line;
+          rawMatches.set(key, resultEntry);
+        }
+      }
+    }
+  }
+}

package/tests/cli.test.ts

@@ -1,11 +1,11 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { writeFileSync } from 'fs';
+import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs';
 import { join } from 'path';
 import { mkdtempSync } from 'fs';
 import { tmpdir } from 'os';
 import { createHash } from 'crypto';
 
-import { runInit, runStats, runTranslate, runTranslatePlural, runWarm } from '../src/cli.js';
+import { runInit, runStats, runTranslate, runTranslatePlural, runWarm, runScan } from '../src/cli.js';
 import { TranslationStore } from '../src/storage.js';
 
 function hash(text: string): string {
@@ -159,4 +159,65 @@ describe('CLI functions', () => {
     // Since plural forms already exist, should be skipped
     expect(output).toContain('Skipped: 1');
   });
+
+  // --- scan ---
+
+  it('scan finds strings', async () => {
+    const configPath = writeConfig(tmpDir);
+    // Create source files to scan
+    const srcDir = join(tmpDir, 'src');
+    mkdirSync(srcDir);
+    writeFileSync(join(srcDir, 'app.py'), 'from transduck import ait\nait("Hello")\nait("World", context="greeting")\n');
+    const output = await runScan({
+      dirs: [srcDir],
+      configPath,
+    });
+    expect(output).toContain('Hello');
+    expect(output).toContain('World');
+    expect(output).toContain('2');
+  });
+
+  it('scan output JSON', async () => {
+    const configPath = writeConfig(tmpDir);
+    const srcDir = join(tmpDir, 'src');
+    mkdirSync(srcDir);
+    writeFileSync(join(srcDir, 'app.py'), 'ait("Hello")\n');
+    const outputFile = join(tmpDir, 'strings.json');
+    const output = await runScan({
+      dirs: [srcDir],
+      outputPath: outputFile,
+      configPath,
+    });
+    expect(output).toContain('Wrote');
+    expect(existsSync(outputFile)).toBe(true);
+    const data = JSON.parse(readFileSync(outputFile, 'utf-8'));
+    expect(data).toHaveLength(1);
+    expect(data[0].text).toBe('Hello');
+  });
+
+  it('scan with warm', async () => {
+    const configPath = writeConfig(tmpDir);
+    const srcDir = join(tmpDir, 'src');
+    mkdirSync(srcDir);
+    writeFileSync(join(srcDir, 'app.py'), 'ait("Hello")\n');
+
+    // Pre-populate the DB so warm skips
+    const dbPath = join(tmpDir, 'translations.duckdb');
+    const store = new TranslationStore(dbPath);
+    await store.initialize();
+    await store.insert({
+      sourceText: 'Hello', sourceLang: 'EN', targetLang: 'DE',
+      projectContextHash: hash('A test site'), stringContextHash: hash(''),
+      translatedText: 'Hallo', model: 'gpt-4.1-mini', status: 'translated',
+    });
+    store.close();
+
+    const output = await runScan({
+      dirs: [srcDir],
+      warm: true,
+      langs: ['DE'],
+      configPath,
+    });
+    expect(output).toContain('Skipped: 1');
+  });
 });

package/tests/handler.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { mkdtempSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+// We test the handler logic directly, not via HTTP
+import { handleTranslationRequest, _resetHandlerStore } from '../src/handler.js';
+
+function makeTmpDir(): string {
+  return mkdtempSync(join(tmpdir(), 'transduck-handler-test-'));
+}
+
+function writeConfig(dir: string): string {
+  const configPath = join(dir, 'transduck.yaml');
+  writeFileSync(configPath, `
+project:
+  name: test-project
+  context: "A test site"
+languages:
+  source: EN
+  targets:
+    - DE
+storage:
+  path: ./translations.duckdb
+backend:
+  api_key_env: OPENAI_API_KEY
+  model: gpt-4.1-mini
+  timeout_seconds: 10
+  max_retries: 2
+`);
+  return configPath;
+}
+
+describe('handleTranslationRequest', () => {
+  let tmpDir: string;
+  let configPath: string;
+
+  beforeEach(() => {
+    _resetHandlerStore();
+    tmpDir = makeTmpDir();
+    configPath = writeConfig(tmpDir);
+    process.env.OPENAI_API_KEY = 'test-key';
+  });
+
+  it('returns translations for strings from cache', async () => {
+    // Pre-populate cache via storage directly
+    const { TranslationStore } = await import('../src/storage.js');
+    const { createHash } = await import('crypto');
+    const hash = (t: string) => createHash('sha256').update(t).digest('hex');
+
+    const store = new TranslationStore(join(tmpDir, 'translations.duckdb'));
+    await store.initialize();
+    await store.insert({
+      sourceText: 'Hello', sourceLang: 'EN', targetLang: 'DE',
+      projectContextHash: hash('A test site'), stringContextHash: hash(''),
+      translatedText: 'Hallo', model: 'gpt-4.1-mini', status: 'translated',
+    });
+    store.close();
+
+    const result = await handleTranslationRequest({
+      language: 'DE',
+      strings: [{ text: 'Hello' }],
+      plurals: [],
+    }, configPath);
+
+    expect(result.translations['Hello||']).toBe('Hallo');
+  });
+
+  it('returns empty translations for uncached strings without API key', async () => {
+    delete process.env.OPENAI_API_KEY;
+    const result = await handleTranslationRequest({
+      language: 'DE',
+      strings: [{ text: 'Unknown' }],
+      plurals: [],
+    }, configPath);
+
+    // Without API key, backend will fail, should return source text
+    expect(result.translations['Unknown||']).toBe('Unknown');
+  });
+
+  it('handles context in cache key', async () => {
+    const { TranslationStore } = await import('../src/storage.js');
+    const { createHash } = await import('crypto');
+    const hash = (t: string) => createHash('sha256').update(t).digest('hex');
+
+    const store = new TranslationStore(join(tmpDir, 'translations.duckdb'));
+    await store.initialize();
+    await store.insert({
+      sourceText: 'Book', sourceLang: 'EN', targetLang: 'DE',
+      projectContextHash: hash('A test site'), stringContextHash: hash('Hotel booking'),
+      translatedText: 'Buchen', model: 'gpt-4.1-mini', status: 'translated',
+    });
+    store.close();
+
+    const result = await handleTranslationRequest({
+      language: 'DE',
+      strings: [{ text: 'Book', context: 'Hotel booking' }],
+      plurals: [],
+    }, configPath);
+
+    expect(result.translations['Book||Hotel booking']).toBe('Buchen');
+  });
+
+  it('returns plural forms from cache', async () => {
+    const { TranslationStore } = await import('../src/storage.js');
+    const { createHash } = await import('crypto');
+    const hash = (t: string) => createHash('sha256').update(t).digest('hex');
+
+    const store = new TranslationStore(join(tmpDir, 'translations.duckdb'));
+    await store.initialize();
+    const sourceKey = '{count} item\x00{count} items';
+    await store.insertPlural({
+      sourceText: sourceKey, sourceLang: 'EN', targetLang: 'DE',
+      projectContextHash: hash('A test site'), stringContextHash: hash(''),
+      pluralCategory: 'one', translatedText: '{count} Artikel',
+      model: 'gpt-4.1-mini', status: 'translated',
+    });
+    await store.insertPlural({
+      sourceText: sourceKey, sourceLang: 'EN', targetLang: 'DE',
+      projectContextHash: hash('A test site'), stringContextHash: hash(''),
+      pluralCategory: 'other', translatedText: '{count} Artikel',
+      model: 'gpt-4.1-mini', status: 'translated',
+    });
+    store.close();
+
+    const result = await handleTranslationRequest({
+      language: 'DE',
+      strings: [],
+      plurals: [{ one: '{count} item', other: '{count} items' }],
+    }, configPath);
+
+    const pluralKey = '{count} item\x00{count} items||';
+    expect(result.plurals[pluralKey]).toBeDefined();
+    expect(result.plurals[pluralKey].one).toBe('{count} Artikel');
+  });
+});

package/tests/react-provider.test.tsx

@@ -0,0 +1,162 @@
+import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
+import React from 'react';
+import { render, screen, cleanup, waitFor } from '@testing-library/react';
+import { TransDuckProvider, t, ait, tPlural, aitPlural, useTransDuck, _resetReactState } from '../src/react/index.js';
+
+// Mock fetch globally
+const mockFetch = vi.fn();
+global.fetch = mockFetch;
+
+// Mock localStorage
+const localStorageStore: Record<string, string> = {};
+const mockLocalStorage = {
+  getItem: vi.fn((key: string) => localStorageStore[key] ?? null),
+  setItem: vi.fn((key: string, value: string) => { localStorageStore[key] = value; }),
+  removeItem: vi.fn((key: string) => { delete localStorageStore[key]; }),
+  clear: vi.fn(() => { for (const k in localStorageStore) delete localStorageStore[k]; }),
+};
+Object.defineProperty(globalThis, 'localStorage', { value: mockLocalStorage, writable: true });
+
+describe('TransDuckProvider + t()', () => {
+  beforeEach(() => {
+    _resetReactState();
+    mockFetch.mockReset();
+    mockLocalStorage.clear();
+  });
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  it('t() returns source text before translation loads', () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({ translations: { 'Hello||': 'Hallo' }, plurals: {} }),
+    });
+
+    function TestComp() {
+      useTransDuck();
+      return <span data-testid="text">{t('Hello')}</span>;
+    }
+
+    render(
+      <TransDuckProvider language="DE">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    // Before fetch resolves, shows source text
+    expect(screen.getByTestId('text').textContent).toBe('Hello');
+  });
+
+  it('t() and ait() are the same function', () => {
+    expect(t).toBe(ait);
+  });
+
+  it('tPlural() and aitPlural() are the same function', () => {
+    expect(tPlural).toBe(aitPlural);
+  });
+
+  it('t() returns source text when language matches source', () => {
+    function TestComp() {
+      useTransDuck();
+      return <span data-testid="text">{t('Hello')}</span>;
+    }
+
+    render(
+      <TransDuckProvider language="EN" sourceLang="EN">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    expect(screen.getByTestId('text').textContent).toBe('Hello');
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+
+  it('t() interpolates vars', () => {
+    function TestComp() {
+      useTransDuck();
+      return <span data-testid="text">{t('Hello {name}', undefined, { name: 'Tim' })}</span>;
+    }
+
+    render(
+      <TransDuckProvider language="EN" sourceLang="EN">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    expect(screen.getByTestId('text').textContent).toBe('Hello Tim');
+  });
+
+  it('t() batches and fetches translations after render', async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({
+        translations: { 'Hello||': 'Hallo', 'World||': 'Welt' },
+        plurals: {},
+      }),
+    });
+
+    function TestComp() {
+      useTransDuck();
+      return (
+        <div>
+          <span data-testid="a">{t('Hello')}</span>
+          <span data-testid="b">{t('World')}</span>
+        </div>
+      );
+    }
+
+    render(
+      <TransDuckProvider language="DE">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    // After useEffect fires and fetch resolves
+    await waitFor(() => {
+      expect(screen.getByTestId('a').textContent).toBe('Hallo');
+      expect(screen.getByTestId('b').textContent).toBe('Welt');
+    });
+
+    // Should have made exactly one fetch call (batched)
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+
+  it('t() does not refetch cached strings', async () => {
+    mockFetch.mockResolvedValue({
+      ok: true,
+      json: async () => ({
+        translations: { 'Hello||': 'Hallo' },
+        plurals: {},
+      }),
+    });
+
+    function TestComp() {
+      useTransDuck();
+      return <span data-testid="text">{t('Hello')}</span>;
+    }
+
+    const { rerender } = render(
+      <TransDuckProvider language="DE">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    await waitFor(() => {
+      expect(screen.getByTestId('text').textContent).toBe('Hallo');
+    });
+
+    mockFetch.mockClear();
+
+    // Re-render — should not fetch again
+    rerender(
+      <TransDuckProvider language="DE">
+        <TestComp />
+      </TransDuckProvider>
+    );
+
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(screen.getByTestId('text').textContent).toBe('Hallo');
+  });
+});