translatronx 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1121 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1599 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +444 -0
- package/dist/index.js +1441 -0
- package/dist/index.js.map +1 -0
- package/package.json +83 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,1599 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import chalk4 from 'chalk';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { cosmiconfig } from 'cosmiconfig';
|
|
6
|
+
import { pathToFileURL } from 'url';
|
|
7
|
+
import Database from 'better-sqlite3';
|
|
8
|
+
import { writeFileSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync } from 'fs';
|
|
9
|
+
import { dirname, join } from 'path';
|
|
10
|
+
import fg from 'fast-glob';
|
|
11
|
+
import { createHash } from 'crypto';
|
|
12
|
+
import OpenAI from 'openai';
|
|
13
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
14
|
+
import { Groq } from 'groq-sdk';
|
|
15
|
+
import ora from 'ora';
|
|
16
|
+
|
|
17
|
+
var ProviderTypeSchema = z.enum([
|
|
18
|
+
"openai",
|
|
19
|
+
"anthropic",
|
|
20
|
+
"groq",
|
|
21
|
+
"local",
|
|
22
|
+
"azure-openai",
|
|
23
|
+
"openrouter"
|
|
24
|
+
]);
|
|
25
|
+
var ProviderConfigSchema = z.object({
|
|
26
|
+
name: z.string().min(1, "Provider name is required"),
|
|
27
|
+
type: ProviderTypeSchema,
|
|
28
|
+
apiKey: z.string().optional(),
|
|
29
|
+
baseUrl: z.string().url().optional(),
|
|
30
|
+
model: z.string().min(1, "Model name is required"),
|
|
31
|
+
temperature: z.number().min(0).max(2).default(0.3),
|
|
32
|
+
maxRetries: z.number().int().min(0).default(3),
|
|
33
|
+
fallback: z.string().optional()
|
|
34
|
+
});
|
|
35
|
+
var ExtractorConfigSchema = z.object({
|
|
36
|
+
type: z.enum(["json", "typescript", "custom"]),
|
|
37
|
+
pattern: z.string().or(z.array(z.string())),
|
|
38
|
+
keyPrefix: z.string().optional(),
|
|
39
|
+
exclude: z.array(z.string()).optional()
|
|
40
|
+
});
|
|
41
|
+
var PromptConfigSchema = z.object({
|
|
42
|
+
/** @deprecated Use customContext instead. This field will be removed in v2.0 */
|
|
43
|
+
systemPrompt: z.string().optional(),
|
|
44
|
+
/** User-configurable prompt content (array for better readability, will be joined with newlines) */
|
|
45
|
+
userPrompt: z.array(z.string()).optional(),
|
|
46
|
+
customContext: z.string().optional(),
|
|
47
|
+
formatting: z.enum(["formal", "casual", "technical"]).optional(),
|
|
48
|
+
glossary: z.record(z.string(), z.string()).optional(),
|
|
49
|
+
brandVoice: z.string().optional()
|
|
50
|
+
}).optional();
|
|
51
|
+
var ValidationConfigSchema = z.object({
|
|
52
|
+
preservePlaceholders: z.boolean().default(true),
|
|
53
|
+
maxLengthRatio: z.number().min(0).default(3),
|
|
54
|
+
preventSourceLeakage: z.boolean().default(true),
|
|
55
|
+
brandNames: z.array(z.string()).optional(),
|
|
56
|
+
customRules: z.array(z.any()).optional()
|
|
57
|
+
});
|
|
58
|
+
var OutputConfigSchema = z.object({
|
|
59
|
+
dir: z.string().default("./locales"),
|
|
60
|
+
format: z.enum(["json", "yaml", "typescript"]).default("json"),
|
|
61
|
+
flat: z.boolean().default(false),
|
|
62
|
+
indent: z.number().int().min(0).max(8).default(2),
|
|
63
|
+
// File naming pattern: {shortCode}.json, {language}.translation.json, or custom
|
|
64
|
+
fileNaming: z.string().default("{shortCode}.json"),
|
|
65
|
+
// Allow source and target files in same directory
|
|
66
|
+
allowSameFolder: z.boolean().default(false)
|
|
67
|
+
});
|
|
68
|
+
var AdvancedConfigSchema = z.object({
|
|
69
|
+
batchSize: z.number().int().min(1).default(20),
|
|
70
|
+
concurrency: z.number().int().min(1).max(10).default(3),
|
|
71
|
+
cacheDir: z.string().default("./.translatronx"),
|
|
72
|
+
ledgerPath: z.string().default("./.translatronx/ledger.sqlite"),
|
|
73
|
+
verbose: z.boolean().default(false)
|
|
74
|
+
}).optional();
|
|
75
|
+
var TargetLanguageSchema = z.object({
|
|
76
|
+
language: z.string().min(1, "Language name is required"),
|
|
77
|
+
shortCode: z.string().min(2, "Language short code is required")
|
|
78
|
+
});
|
|
79
|
+
var translatronxConfigSchema = z.object({
|
|
80
|
+
sourceLanguage: z.string().min(2, "Source language code is required"),
|
|
81
|
+
targetLanguages: z.array(TargetLanguageSchema).min(1, "At least one target language is required"),
|
|
82
|
+
extractors: z.array(ExtractorConfigSchema).min(1, "At least one extractor is required"),
|
|
83
|
+
providers: z.array(ProviderConfigSchema).min(1, "At least one provider is required"),
|
|
84
|
+
validation: ValidationConfigSchema.default({}),
|
|
85
|
+
output: OutputConfigSchema.default({}),
|
|
86
|
+
prompts: PromptConfigSchema,
|
|
87
|
+
advanced: AdvancedConfigSchema
|
|
88
|
+
});
|
|
89
|
+
function validateConfig(config) {
|
|
90
|
+
return translatronxConfigSchema.parse(config);
|
|
91
|
+
}
|
|
92
|
+
var MODULE_NAME = "translatronx";
|
|
93
|
+
async function loadConfig(searchFrom) {
|
|
94
|
+
const explorer = cosmiconfig(MODULE_NAME, {
|
|
95
|
+
searchPlaces: [
|
|
96
|
+
"translatronx.config.ts",
|
|
97
|
+
"translatronx.config.js",
|
|
98
|
+
"translatronx.config.json",
|
|
99
|
+
`.${MODULE_NAME}rc`,
|
|
100
|
+
`.${MODULE_NAME}rc.json`,
|
|
101
|
+
`.${MODULE_NAME}rc.ts`,
|
|
102
|
+
`.${MODULE_NAME}rc.js`
|
|
103
|
+
],
|
|
104
|
+
loaders: {
|
|
105
|
+
".ts": async (filepath) => {
|
|
106
|
+
try {
|
|
107
|
+
const fileUrl = pathToFileURL(filepath).href;
|
|
108
|
+
const module = await import(fileUrl);
|
|
109
|
+
return module.default || module;
|
|
110
|
+
} catch (error) {
|
|
111
|
+
throw new Error(`Failed to load TypeScript config from ${filepath}: ${error}`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
try {
|
|
117
|
+
const result = await explorer.search(searchFrom);
|
|
118
|
+
if (!result || !result.config) {
|
|
119
|
+
throw new Error(
|
|
120
|
+
`No ${MODULE_NAME} configuration found. Run 'translatronx init' to create one.`
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
const config = validateConfig(result.config);
|
|
125
|
+
return config;
|
|
126
|
+
} catch (error) {
|
|
127
|
+
console.error(chalk4.red("\u274C Configuration validation failed:"));
|
|
128
|
+
if (error.errors) {
|
|
129
|
+
error.errors.forEach((err) => {
|
|
130
|
+
console.error(chalk4.yellow(` - ${err.path.join(".")}: ${err.message}`));
|
|
131
|
+
});
|
|
132
|
+
} else {
|
|
133
|
+
console.error(chalk4.yellow(` ${error.message}`));
|
|
134
|
+
}
|
|
135
|
+
throw new Error("Invalid configuration");
|
|
136
|
+
}
|
|
137
|
+
} catch (error) {
|
|
138
|
+
if (error.message.includes("No translatronx configuration found")) {
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
console.error(chalk4.red("\u274C Failed to load configuration:"));
|
|
142
|
+
console.error(chalk4.yellow(` ${error.message}`));
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function getDefaultConfig() {
|
|
147
|
+
return {
|
|
148
|
+
sourceLanguage: "en",
|
|
149
|
+
targetLanguages: [
|
|
150
|
+
{ language: "Spanish", shortCode: "es" },
|
|
151
|
+
{ language: "French", shortCode: "fr" },
|
|
152
|
+
{ language: "German", shortCode: "de" }
|
|
153
|
+
],
|
|
154
|
+
extractors: [
|
|
155
|
+
{
|
|
156
|
+
type: "json",
|
|
157
|
+
pattern: "src/locales/en/**/*.json"
|
|
158
|
+
}
|
|
159
|
+
],
|
|
160
|
+
providers: [
|
|
161
|
+
{
|
|
162
|
+
name: "primary",
|
|
163
|
+
type: "openai",
|
|
164
|
+
model: "gpt-4-turbo-preview",
|
|
165
|
+
temperature: 0.3,
|
|
166
|
+
maxRetries: 3
|
|
167
|
+
}
|
|
168
|
+
],
|
|
169
|
+
validation: {
|
|
170
|
+
preservePlaceholders: true,
|
|
171
|
+
maxLengthRatio: 3,
|
|
172
|
+
preventSourceLeakage: true
|
|
173
|
+
},
|
|
174
|
+
output: {
|
|
175
|
+
dir: "./locales",
|
|
176
|
+
format: "json",
|
|
177
|
+
flat: false,
|
|
178
|
+
indent: 2,
|
|
179
|
+
fileNaming: "{shortCode}.json",
|
|
180
|
+
allowSameFolder: false
|
|
181
|
+
},
|
|
182
|
+
// Optional: Customize translation prompts
|
|
183
|
+
// prompts: {
|
|
184
|
+
// userPrompt: [
|
|
185
|
+
// 'Please translate the following strings.',
|
|
186
|
+
// 'Maintain a professional and friendly tone.',
|
|
187
|
+
// 'Use gender-neutral language where possible.',
|
|
188
|
+
// ],
|
|
189
|
+
// customContext: 'This is a mobile banking application.',
|
|
190
|
+
// formatting: 'formal',
|
|
191
|
+
// brandVoice: 'Professional, trustworthy, and approachable',
|
|
192
|
+
// glossary: {
|
|
193
|
+
// 'Account': 'Cuenta',
|
|
194
|
+
// 'Balance': 'Saldo',
|
|
195
|
+
// },
|
|
196
|
+
// },
|
|
197
|
+
advanced: {
|
|
198
|
+
batchSize: 20,
|
|
199
|
+
concurrency: 3,
|
|
200
|
+
cacheDir: "./.translatronx",
|
|
201
|
+
ledgerPath: "./.translatronx/ledger.sqlite",
|
|
202
|
+
verbose: false
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
var translatronxLedger = class {
|
|
207
|
+
db;
|
|
208
|
+
constructor(ledgerPath) {
|
|
209
|
+
const dir = dirname(ledgerPath);
|
|
210
|
+
if (!existsSync(dir)) {
|
|
211
|
+
mkdirSync(dir, { recursive: true });
|
|
212
|
+
}
|
|
213
|
+
this.db = new Database(ledgerPath);
|
|
214
|
+
this.initializeSchema();
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Initialize database schema
|
|
218
|
+
*/
|
|
219
|
+
initializeSchema() {
|
|
220
|
+
this.db.exec(`
|
|
221
|
+
-- Source hashes table for change detection
|
|
222
|
+
CREATE TABLE IF NOT EXISTS source_hashes (
|
|
223
|
+
key_path TEXT PRIMARY KEY,
|
|
224
|
+
value_hash TEXT NOT NULL,
|
|
225
|
+
context_sig TEXT,
|
|
226
|
+
last_seen_run TEXT,
|
|
227
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
228
|
+
);
|
|
229
|
+
|
|
230
|
+
CREATE INDEX IF NOT EXISTS idx_hash_lookup
|
|
231
|
+
ON source_hashes(value_hash, context_sig);
|
|
232
|
+
|
|
233
|
+
-- Sync status table for tracking translation state
|
|
234
|
+
CREATE TABLE IF NOT EXISTS sync_status (
|
|
235
|
+
key_path TEXT,
|
|
236
|
+
lang_code TEXT,
|
|
237
|
+
target_hash TEXT,
|
|
238
|
+
status TEXT CHECK(status IN ('CLEAN','DIRTY','FAILED','MANUAL','SKIPPED')),
|
|
239
|
+
model_fingerprint TEXT,
|
|
240
|
+
prompt_version INTEGER,
|
|
241
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
242
|
+
PRIMARY KEY (key_path, lang_code)
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
CREATE INDEX IF NOT EXISTS idx_status_lookup
|
|
246
|
+
ON sync_status(status, lang_code);
|
|
247
|
+
|
|
248
|
+
-- Run history table for audit trail
|
|
249
|
+
CREATE TABLE IF NOT EXISTS run_history (
|
|
250
|
+
run_id TEXT PRIMARY KEY,
|
|
251
|
+
started_at TIMESTAMP,
|
|
252
|
+
finished_at TIMESTAMP,
|
|
253
|
+
model_used TEXT,
|
|
254
|
+
tokens_in INTEGER,
|
|
255
|
+
tokens_out INTEGER,
|
|
256
|
+
cost_estimate_usd REAL,
|
|
257
|
+
config_hash TEXT
|
|
258
|
+
);
|
|
259
|
+
`);
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Get source hash for a key path
|
|
263
|
+
*/
|
|
264
|
+
getSourceHash(keyPath) {
|
|
265
|
+
const stmt = this.db.prepare(`
|
|
266
|
+
SELECT value_hash FROM source_hashes WHERE key_path = ?
|
|
267
|
+
`);
|
|
268
|
+
const row = stmt.get(keyPath);
|
|
269
|
+
return row?.value_hash || null;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Update source hash for a key path
|
|
273
|
+
*/
|
|
274
|
+
updateSourceHash(keyPath, valueHash, contextSig, runId) {
|
|
275
|
+
const stmt = this.db.prepare(`
|
|
276
|
+
INSERT INTO source_hashes (key_path, value_hash, context_sig, last_seen_run)
|
|
277
|
+
VALUES (?, ?, ?, ?)
|
|
278
|
+
ON CONFLICT(key_path) DO UPDATE SET
|
|
279
|
+
value_hash = excluded.value_hash,
|
|
280
|
+
context_sig = excluded.context_sig,
|
|
281
|
+
last_seen_run = excluded.last_seen_run,
|
|
282
|
+
updated_at = CURRENT_TIMESTAMP
|
|
283
|
+
`);
|
|
284
|
+
stmt.run(keyPath, valueHash, contextSig || null, runId || null);
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Get sync status for a key-language pair
|
|
288
|
+
*/
|
|
289
|
+
getSyncStatus(keyPath, langCode) {
|
|
290
|
+
const stmt = this.db.prepare(`
|
|
291
|
+
SELECT * FROM sync_status WHERE key_path = ? AND lang_code = ?
|
|
292
|
+
`);
|
|
293
|
+
const row = stmt.get(keyPath, langCode);
|
|
294
|
+
return row || null;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Update sync status for a key-language pair
|
|
298
|
+
*/
|
|
299
|
+
updateSyncStatus(keyPath, langCode, targetHash, status, modelFingerprint, promptVersion) {
|
|
300
|
+
const stmt = this.db.prepare(`
|
|
301
|
+
INSERT INTO sync_status (key_path, lang_code, target_hash, status, model_fingerprint, prompt_version)
|
|
302
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
303
|
+
ON CONFLICT(key_path, lang_code) DO UPDATE SET
|
|
304
|
+
target_hash = excluded.target_hash,
|
|
305
|
+
status = excluded.status,
|
|
306
|
+
model_fingerprint = excluded.model_fingerprint,
|
|
307
|
+
prompt_version = excluded.prompt_version,
|
|
308
|
+
updated_at = CURRENT_TIMESTAMP
|
|
309
|
+
`);
|
|
310
|
+
stmt.run(keyPath, langCode, targetHash, status, modelFingerprint || null, promptVersion || null);
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Begin a new run and return run ID
|
|
314
|
+
*/
|
|
315
|
+
startRun(modelUsed, configHash) {
|
|
316
|
+
const runId = `run_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
317
|
+
const stmt = this.db.prepare(`
|
|
318
|
+
INSERT INTO run_history (run_id, started_at, model_used, config_hash)
|
|
319
|
+
VALUES (?, CURRENT_TIMESTAMP, ?, ?)
|
|
320
|
+
`);
|
|
321
|
+
stmt.run(runId, modelUsed, configHash);
|
|
322
|
+
return runId;
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Complete a run with stats
|
|
326
|
+
*/
|
|
327
|
+
completeRun(runId, tokensIn, tokensOut, costEstimate) {
|
|
328
|
+
const stmt = this.db.prepare(`
|
|
329
|
+
UPDATE run_history
|
|
330
|
+
SET finished_at = CURRENT_TIMESTAMP,
|
|
331
|
+
tokens_in = ?,
|
|
332
|
+
tokens_out = ?,
|
|
333
|
+
cost_estimate_usd = ?
|
|
334
|
+
WHERE run_id = ?
|
|
335
|
+
`);
|
|
336
|
+
stmt.run(tokensIn, tokensOut, costEstimate, runId);
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Get all dirty/new translations for a language
|
|
340
|
+
*/
|
|
341
|
+
getDirtyKeys(langCode) {
|
|
342
|
+
const stmt = this.db.prepare(`
|
|
343
|
+
SELECT key_path FROM sync_status
|
|
344
|
+
WHERE lang_code = ? AND status IN ('DIRTY', 'FAILED')
|
|
345
|
+
`);
|
|
346
|
+
const rows = stmt.all(langCode);
|
|
347
|
+
return rows.map((r) => r.key_path);
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Clean up old run history (keep last N runs)
|
|
351
|
+
*/
|
|
352
|
+
cleanupHistory(keepLast = 100) {
|
|
353
|
+
this.db.exec(`
|
|
354
|
+
DELETE FROM run_history
|
|
355
|
+
WHERE run_id NOT IN (
|
|
356
|
+
SELECT run_id FROM run_history
|
|
357
|
+
ORDER BY started_at DESC
|
|
358
|
+
LIMIT ${keepLast}
|
|
359
|
+
)
|
|
360
|
+
`);
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Get the most recent run record
|
|
364
|
+
*/
|
|
365
|
+
getLatestRun() {
|
|
366
|
+
const stmt = this.db.prepare(`
|
|
367
|
+
SELECT * FROM run_history ORDER BY started_at DESC LIMIT 1
|
|
368
|
+
`);
|
|
369
|
+
return stmt.get() || null;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Get aggregate statistics for the project
|
|
373
|
+
*/
|
|
374
|
+
getProjectStats() {
|
|
375
|
+
const totalKeys = this.db.prepare(`SELECT COUNT(DISTINCT key_path) as count FROM source_hashes`).get();
|
|
376
|
+
const manualCount = this.db.prepare(`SELECT COUNT(*) as count FROM sync_status WHERE status = 'MANUAL'`).get();
|
|
377
|
+
return {
|
|
378
|
+
total_keys: totalKeys.count,
|
|
379
|
+
manual_count: manualCount.count
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Get status counts per language
|
|
384
|
+
*/
|
|
385
|
+
getLanguageStats(langCode) {
|
|
386
|
+
const translated = this.db.prepare(`
|
|
387
|
+
SELECT COUNT(*) as count FROM sync_status
|
|
388
|
+
WHERE lang_code = ? AND status = 'CLEAN'
|
|
389
|
+
`).get(langCode);
|
|
390
|
+
const failed = this.db.prepare(`
|
|
391
|
+
SELECT COUNT(*) as count FROM sync_status
|
|
392
|
+
WHERE lang_code = ? AND status IN ('FAILED', 'DIRTY')
|
|
393
|
+
`).get(langCode);
|
|
394
|
+
return {
|
|
395
|
+
translated: translated.count,
|
|
396
|
+
failed: failed.count
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Get failed translation items for retry
|
|
401
|
+
*/
|
|
402
|
+
getFailedItems(langCode, _batchId) {
|
|
403
|
+
let query = `
|
|
404
|
+
SELECT ss.*, sh.value_hash
|
|
405
|
+
FROM sync_status ss
|
|
406
|
+
LEFT JOIN source_hashes sh ON ss.key_path = sh.key_path
|
|
407
|
+
WHERE ss.status = 'FAILED'
|
|
408
|
+
`;
|
|
409
|
+
const params = [];
|
|
410
|
+
if (langCode) {
|
|
411
|
+
query += ` AND ss.lang_code = ?`;
|
|
412
|
+
params.push(langCode);
|
|
413
|
+
}
|
|
414
|
+
const stmt = this.db.prepare(query);
|
|
415
|
+
return stmt.all(...params);
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Close database connection
|
|
419
|
+
*/
|
|
420
|
+
close() {
|
|
421
|
+
this.db.close();
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Execute a transaction
|
|
425
|
+
*/
|
|
426
|
+
transaction(fn) {
|
|
427
|
+
return this.db.transaction(fn)();
|
|
428
|
+
}
|
|
429
|
+
};
|
|
430
|
+
function computeHash(content) {
|
|
431
|
+
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
432
|
+
}
|
|
433
|
+
function extractPlaceholders(text) {
|
|
434
|
+
const patterns = [
|
|
435
|
+
/\{([^}]+)\}/g,
|
|
436
|
+
// {var}
|
|
437
|
+
/\{\{([^}]+)\}\}/g,
|
|
438
|
+
// {{var}}
|
|
439
|
+
/%[sdif]/g,
|
|
440
|
+
// %s, %d, %i, %f
|
|
441
|
+
/\$\{([^}]+)\}/g,
|
|
442
|
+
// ${var}
|
|
443
|
+
/\$\d+/g
|
|
444
|
+
// $1, $2, etc.
|
|
445
|
+
];
|
|
446
|
+
const placeholders = /* @__PURE__ */ new Set();
|
|
447
|
+
for (const pattern of patterns) {
|
|
448
|
+
const matches = text.matchAll(pattern);
|
|
449
|
+
for (const match of matches) {
|
|
450
|
+
placeholders.add(match[0]);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return Array.from(placeholders).sort();
|
|
454
|
+
}
|
|
455
|
+
function generateUnitId(keyPath, sourceFile) {
|
|
456
|
+
const combined = `${sourceFile}:${keyPath}`;
|
|
457
|
+
return computeHash(combined).substring(0, 16);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// src/extractors/json-extractor.ts
|
|
461
|
+
var JsonExtractor = class {
|
|
462
|
+
schemaVersion = 1;
|
|
463
|
+
/**
|
|
464
|
+
* Extract translatable strings from JSON files
|
|
465
|
+
*/
|
|
466
|
+
async extract(_sourceFiles, config) {
|
|
467
|
+
const units = [];
|
|
468
|
+
const patterns = Array.isArray(config.pattern) ? config.pattern : [config.pattern];
|
|
469
|
+
const files = await fg(patterns, {
|
|
470
|
+
ignore: config.exclude || ["**/node_modules/**", "**/dist/**"],
|
|
471
|
+
absolute: true
|
|
472
|
+
});
|
|
473
|
+
for (const filePath of files) {
|
|
474
|
+
try {
|
|
475
|
+
const content = readFileSync(filePath, "utf-8");
|
|
476
|
+
const data = JSON.parse(content);
|
|
477
|
+
const fileUnits = this.extractFromObject(data, [], filePath, config.keyPrefix);
|
|
478
|
+
units.push(...fileUnits);
|
|
479
|
+
} catch (error) {
|
|
480
|
+
console.error(`Failed to extract from ${filePath}:`, error);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return units;
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Recursively extract strings from a JSON object
|
|
487
|
+
*/
|
|
488
|
+
extractFromObject(obj, keyPath, sourceFile, keyPrefix) {
|
|
489
|
+
const units = [];
|
|
490
|
+
if (typeof obj === "string") {
|
|
491
|
+
const fullKeyPath = keyPrefix ? `${keyPrefix}.${keyPath.join(".")}` : keyPath.join(".");
|
|
492
|
+
units.push(this.createSourceUnit(fullKeyPath, obj, sourceFile));
|
|
493
|
+
} else if (Array.isArray(obj)) {
|
|
494
|
+
obj.forEach((item, index) => {
|
|
495
|
+
units.push(...this.extractFromObject(item, [...keyPath, index.toString()], sourceFile, keyPrefix));
|
|
496
|
+
});
|
|
497
|
+
} else if (obj && typeof obj === "object") {
|
|
498
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
499
|
+
units.push(...this.extractFromObject(value, [...keyPath, key], sourceFile, keyPrefix));
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
return units;
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Create a source unit from extracted string
|
|
506
|
+
*/
|
|
507
|
+
createSourceUnit(keyPath, sourceText, sourceFile) {
|
|
508
|
+
const placeholders = extractPlaceholders(sourceText);
|
|
509
|
+
const sourceHash = computeHash(sourceText);
|
|
510
|
+
const unitId = generateUnitId(keyPath, sourceFile);
|
|
511
|
+
return {
|
|
512
|
+
unitId,
|
|
513
|
+
keyPath,
|
|
514
|
+
sourceText,
|
|
515
|
+
sourceHash,
|
|
516
|
+
placeholders,
|
|
517
|
+
sourceFile,
|
|
518
|
+
schemaVersion: this.schemaVersion
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
};
|
|
522
|
+
|
|
523
|
+
// src/planner/index.ts
|
|
524
|
+
var IncrementalTranslationPlanner = class {
|
|
525
|
+
constructor(ledger) {
|
|
526
|
+
this.ledger = ledger;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Create a translation plan based on change detection
|
|
530
|
+
*/
|
|
531
|
+
async createPlan(sourceUnits, targetLanguages) {
|
|
532
|
+
const batches = [];
|
|
533
|
+
let totalUnits = 0;
|
|
534
|
+
for (const lang of targetLanguages) {
|
|
535
|
+
const unitsNeedingTranslation = this.detectChanges(sourceUnits, lang.shortCode);
|
|
536
|
+
if (unitsNeedingTranslation.length === 0) {
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
const langBatches = this.createBatches(unitsNeedingTranslation, lang.shortCode);
|
|
540
|
+
batches.push(...langBatches);
|
|
541
|
+
totalUnits += unitsNeedingTranslation.length;
|
|
542
|
+
}
|
|
543
|
+
const estimatedCost = this.estimateCost(totalUnits);
|
|
544
|
+
return {
|
|
545
|
+
batches,
|
|
546
|
+
totalUnits,
|
|
547
|
+
estimatedCost
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Detect which source units have changed and need translation
|
|
552
|
+
*/
|
|
553
|
+
detectChanges(sourceUnits, targetLang) {
|
|
554
|
+
const needsTranslation = [];
|
|
555
|
+
for (const unit of sourceUnits) {
|
|
556
|
+
const storedHash = this.ledger.getSourceHash(unit.keyPath);
|
|
557
|
+
const sourceChanged = storedHash !== unit.sourceHash;
|
|
558
|
+
const syncStatus = this.ledger.getSyncStatus(unit.keyPath, targetLang);
|
|
559
|
+
if (!syncStatus) {
|
|
560
|
+
needsTranslation.push(unit);
|
|
561
|
+
} else if (syncStatus.status === "MANUAL") {
|
|
562
|
+
continue;
|
|
563
|
+
} else if (sourceChanged) {
|
|
564
|
+
needsTranslation.push(unit);
|
|
565
|
+
} else if (syncStatus.status === "FAILED" || syncStatus.status === "DIRTY") {
|
|
566
|
+
needsTranslation.push(unit);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
return needsTranslation;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Create batches from units for efficient LLM processing
|
|
573
|
+
*/
|
|
574
|
+
createBatches(units, targetLang, batchSize = 20) {
|
|
575
|
+
const batches = [];
|
|
576
|
+
for (let i = 0; i < units.length; i += batchSize) {
|
|
577
|
+
const batchUnits = units.slice(i, i + batchSize);
|
|
578
|
+
const batchId = `batch_${targetLang}_${Date.now()}_${i}`;
|
|
579
|
+
const contentHash = computeHash(
|
|
580
|
+
batchUnits.map((u) => u.sourceHash).join("_")
|
|
581
|
+
);
|
|
582
|
+
batches.push({
|
|
583
|
+
batchId,
|
|
584
|
+
sourceUnits: batchUnits,
|
|
585
|
+
targetLanguage: targetLang,
|
|
586
|
+
deduplicationKey: contentHash
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
return batches;
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Estimate cost for translation
|
|
593
|
+
* Rough estimate: 50 input tokens + 50 output tokens per unit
|
|
594
|
+
* OpenAI pricing: ~$0.01 per 1K tokens
|
|
595
|
+
*/
|
|
596
|
+
estimateCost(totalUnits) {
|
|
597
|
+
const tokensPerUnit = 100;
|
|
598
|
+
const totalTokens = totalUnits * tokensPerUnit;
|
|
599
|
+
const costPer1kTokens = 0.01;
|
|
600
|
+
return totalTokens / 1e3 * costPer1kTokens;
|
|
601
|
+
}
|
|
602
|
+
};
|
|
603
|
+
|
|
604
|
+
// src/providers/base-provider.ts
|
|
605
|
+
var BaseLLMProvider = class {
|
|
606
|
+
constructor(config) {
|
|
607
|
+
this.config = config;
|
|
608
|
+
this.maxRetries = config.maxRetries ?? 3;
|
|
609
|
+
this.temperature = config.temperature ?? 0.3;
|
|
610
|
+
}
|
|
611
|
+
maxRetries;
|
|
612
|
+
temperature;
|
|
613
|
+
/**
|
|
614
|
+
* Retry logic with exponential backoff
|
|
615
|
+
*/
|
|
616
|
+
async retryWithBackoff(fn, attempt = 0) {
|
|
617
|
+
try {
|
|
618
|
+
return await fn();
|
|
619
|
+
} catch (error) {
|
|
620
|
+
if (attempt >= this.maxRetries) {
|
|
621
|
+
throw error;
|
|
622
|
+
}
|
|
623
|
+
const delay = Math.pow(2, attempt) * 1e3;
|
|
624
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
625
|
+
return this.retryWithBackoff(fn, attempt + 1);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Build prompt for translation batch
|
|
630
|
+
*/
|
|
631
|
+
buildTranslationPrompt(batch) {
|
|
632
|
+
const items = batch.sourceUnits.map((unit, idx) => {
|
|
633
|
+
return `${idx + 1}. [${unit.keyPath}]: "${unit.sourceText}"`;
|
|
634
|
+
}).join("\n");
|
|
635
|
+
return `Translate the following strings to ${batch.targetLanguage}. Preserve all placeholders exactly as they appear. Return a JSON array with the same order.
|
|
636
|
+
|
|
637
|
+
${items}`;
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Clean LLM output to extract just the JSON part
|
|
641
|
+
* Handles <think> tags, markdown, and extra text
|
|
642
|
+
*/
|
|
643
|
+
cleanJsonOutput(text) {
|
|
644
|
+
let clean = text.replace(/<think>[\s\S]*?<\/think>/g, "").trim();
|
|
645
|
+
const codeBlockMatch = clean.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
|
|
646
|
+
if (codeBlockMatch) {
|
|
647
|
+
return codeBlockMatch[1];
|
|
648
|
+
}
|
|
649
|
+
const start = clean.indexOf("[");
|
|
650
|
+
const end = clean.lastIndexOf("]");
|
|
651
|
+
if (start !== -1 && end !== -1 && end > start) {
|
|
652
|
+
return clean.substring(start, end + 1);
|
|
653
|
+
}
|
|
654
|
+
return clean;
|
|
655
|
+
}
|
|
656
|
+
/**
|
|
657
|
+
* Parse translation response into results
|
|
658
|
+
*/
|
|
659
|
+
parseTranslationResponse(batch, responseText) {
|
|
660
|
+
try {
|
|
661
|
+
const jsonString = this.cleanJsonOutput(responseText);
|
|
662
|
+
const translations = JSON.parse(jsonString);
|
|
663
|
+
if (!Array.isArray(translations)) {
|
|
664
|
+
throw new Error("Response is not an array");
|
|
665
|
+
}
|
|
666
|
+
return batch.sourceUnits.map((unit, idx) => ({
|
|
667
|
+
unitId: unit.unitId,
|
|
668
|
+
translatedText: translations[idx] || "",
|
|
669
|
+
confidence: 1
|
|
670
|
+
}));
|
|
671
|
+
} catch (error) {
|
|
672
|
+
console.error("Failed to parse response as JSON array:", error);
|
|
673
|
+
console.debug("Raw response:", responseText);
|
|
674
|
+
throw new Error("Invalid translation response format");
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
|
|
679
|
+
// src/providers/openai-provider.ts
|
|
680
|
+
var OpenAIProvider = class extends BaseLLMProvider {
|
|
681
|
+
client;
|
|
682
|
+
constructor(config) {
|
|
683
|
+
super(config);
|
|
684
|
+
this.client = new OpenAI({
|
|
685
|
+
apiKey: config.apiKey,
|
|
686
|
+
baseURL: config.baseUrl
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
async translate(batch, prompt) {
|
|
690
|
+
return this.retryWithBackoff(async () => {
|
|
691
|
+
const userPrompt = this.buildTranslationPrompt(batch);
|
|
692
|
+
const response = await this.client.chat.completions.create({
|
|
693
|
+
model: this.config.model,
|
|
694
|
+
temperature: this.temperature,
|
|
695
|
+
messages: [
|
|
696
|
+
{ role: "system", content: prompt.system },
|
|
697
|
+
{ role: "user", content: userPrompt }
|
|
698
|
+
],
|
|
699
|
+
response_format: { type: "json_object" }
|
|
700
|
+
});
|
|
701
|
+
const content = response.choices[0]?.message?.content || "[]";
|
|
702
|
+
const jsonMatch = content.match(/\[[\s\S]*\]/);
|
|
703
|
+
const jsonString = jsonMatch ? jsonMatch[0] : content;
|
|
704
|
+
return this.parseTranslationResponse(batch, jsonString);
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
getModelFingerprint() {
|
|
708
|
+
return `openai:${this.config.model}`;
|
|
709
|
+
}
|
|
710
|
+
estimateCost(batch) {
|
|
711
|
+
const avgCharsPerUnit = batch.sourceUnits.reduce((sum, u) => sum + u.sourceText.length, 0) / batch.sourceUnits.length;
|
|
712
|
+
const estimatedInputTokens = batch.sourceUnits.length * (avgCharsPerUnit / 4);
|
|
713
|
+
const estimatedOutputTokens = estimatedInputTokens;
|
|
714
|
+
const inputCost = estimatedInputTokens / 1e3 * 0.03;
|
|
715
|
+
const outputCost = estimatedOutputTokens / 1e3 * 0.06;
|
|
716
|
+
return inputCost + outputCost;
|
|
717
|
+
}
|
|
718
|
+
};
|
|
719
|
+
var AnthropicProvider = class extends BaseLLMProvider {
|
|
720
|
+
client;
|
|
721
|
+
constructor(config) {
|
|
722
|
+
super(config);
|
|
723
|
+
this.client = new Anthropic({
|
|
724
|
+
apiKey: config.apiKey
|
|
725
|
+
});
|
|
726
|
+
}
|
|
727
|
+
async translate(batch, prompt) {
|
|
728
|
+
return this.retryWithBackoff(async () => {
|
|
729
|
+
const userPrompt = this.buildTranslationPrompt(batch);
|
|
730
|
+
const response = await this.client.messages.create({
|
|
731
|
+
model: this.config.model,
|
|
732
|
+
max_tokens: prompt.maxTokens || 4096,
|
|
733
|
+
temperature: this.temperature,
|
|
734
|
+
system: prompt.system,
|
|
735
|
+
messages: [
|
|
736
|
+
{ role: "user", content: userPrompt }
|
|
737
|
+
]
|
|
738
|
+
});
|
|
739
|
+
const content = response.content[0];
|
|
740
|
+
if (content.type !== "text") {
|
|
741
|
+
throw new Error("Expected text response from Anthropic");
|
|
742
|
+
}
|
|
743
|
+
const jsonMatch = content.text.match(/\[[\s\S]*\]/);
|
|
744
|
+
const jsonString = jsonMatch ? jsonMatch[0] : content.text;
|
|
745
|
+
return this.parseTranslationResponse(batch, jsonString);
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
getModelFingerprint() {
|
|
749
|
+
return `anthropic:${this.config.model}`;
|
|
750
|
+
}
|
|
751
|
+
estimateCost(batch) {
|
|
752
|
+
const avgCharsPerUnit = batch.sourceUnits.reduce((sum, u) => sum + u.sourceText.length, 0) / batch.sourceUnits.length;
|
|
753
|
+
const estimatedInputTokens = batch.sourceUnits.length * (avgCharsPerUnit / 4);
|
|
754
|
+
const estimatedOutputTokens = estimatedInputTokens;
|
|
755
|
+
const inputCost = estimatedInputTokens / 1e3 * 0.015;
|
|
756
|
+
const outputCost = estimatedOutputTokens / 1e3 * 0.075;
|
|
757
|
+
return inputCost + outputCost;
|
|
758
|
+
}
|
|
759
|
+
};
|
|
760
|
+
var GroqProvider = class extends BaseLLMProvider {
|
|
761
|
+
client;
|
|
762
|
+
constructor(config) {
|
|
763
|
+
super(config);
|
|
764
|
+
this.client = new Groq({
|
|
765
|
+
apiKey: config.apiKey
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
async translate(batch, prompt) {
|
|
769
|
+
return this.retryWithBackoff(async () => {
|
|
770
|
+
const userPrompt = this.buildTranslationPrompt(batch);
|
|
771
|
+
const response = await this.client.chat.completions.create({
|
|
772
|
+
model: this.config.model,
|
|
773
|
+
temperature: this.temperature,
|
|
774
|
+
messages: [
|
|
775
|
+
{ role: "system", content: prompt.system },
|
|
776
|
+
{ role: "user", content: userPrompt }
|
|
777
|
+
],
|
|
778
|
+
response_format: { type: "json_object" }
|
|
779
|
+
});
|
|
780
|
+
const content = response.choices[0]?.message?.content || "[]";
|
|
781
|
+
const jsonMatch = content.match(/\[[\s\S]*\]/);
|
|
782
|
+
const jsonString = jsonMatch ? jsonMatch[0] : content;
|
|
783
|
+
return this.parseTranslationResponse(batch, jsonString);
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
getModelFingerprint() {
|
|
787
|
+
return `groq:${this.config.model}`;
|
|
788
|
+
}
|
|
789
|
+
estimateCost(batch) {
|
|
790
|
+
const avgCharsPerUnit = batch.sourceUnits.reduce((sum, u) => sum + u.sourceText.length, 0) / batch.sourceUnits.length;
|
|
791
|
+
const estimatedInputTokens = batch.sourceUnits.length * (avgCharsPerUnit / 4);
|
|
792
|
+
const estimatedOutputTokens = estimatedInputTokens;
|
|
793
|
+
const inputCost = estimatedInputTokens / 1e3 * 1e-4;
|
|
794
|
+
const outputCost = estimatedOutputTokens / 1e3 * 2e-4;
|
|
795
|
+
return inputCost + outputCost;
|
|
796
|
+
}
|
|
797
|
+
};
|
|
798
|
+
|
|
799
|
+
// src/providers/index.ts
|
|
800
|
+
var ProviderFactory = class {
|
|
801
|
+
/**
|
|
802
|
+
* Create an LLM provider from configuration
|
|
803
|
+
*/
|
|
804
|
+
static createProvider(config) {
|
|
805
|
+
switch (config.type) {
|
|
806
|
+
case "openai":
|
|
807
|
+
if (!config.apiKey) {
|
|
808
|
+
throw new Error("OpenAI API key is required");
|
|
809
|
+
}
|
|
810
|
+
return new OpenAIProvider({
|
|
811
|
+
apiKey: config.apiKey,
|
|
812
|
+
model: config.model,
|
|
813
|
+
temperature: config.temperature,
|
|
814
|
+
maxRetries: config.maxRetries,
|
|
815
|
+
baseUrl: config.baseUrl
|
|
816
|
+
});
|
|
817
|
+
case "anthropic":
|
|
818
|
+
if (!config.apiKey) {
|
|
819
|
+
throw new Error("Anthropic API key is required");
|
|
820
|
+
}
|
|
821
|
+
return new AnthropicProvider({
|
|
822
|
+
apiKey: config.apiKey,
|
|
823
|
+
model: config.model,
|
|
824
|
+
temperature: config.temperature,
|
|
825
|
+
maxRetries: config.maxRetries
|
|
826
|
+
});
|
|
827
|
+
case "groq":
|
|
828
|
+
if (!config.apiKey) {
|
|
829
|
+
throw new Error("Groq API key is required");
|
|
830
|
+
}
|
|
831
|
+
return new GroqProvider({
|
|
832
|
+
apiKey: config.apiKey,
|
|
833
|
+
model: config.model,
|
|
834
|
+
temperature: config.temperature,
|
|
835
|
+
maxRetries: config.maxRetries
|
|
836
|
+
});
|
|
837
|
+
case "local":
|
|
838
|
+
return new OpenAIProvider({
|
|
839
|
+
apiKey: config.apiKey || "not-needed",
|
|
840
|
+
model: config.model,
|
|
841
|
+
temperature: config.temperature,
|
|
842
|
+
maxRetries: config.maxRetries,
|
|
843
|
+
baseUrl: config.baseUrl || "http://localhost:11434/v1"
|
|
844
|
+
});
|
|
845
|
+
case "azure-openai":
|
|
846
|
+
if (!config.apiKey) {
|
|
847
|
+
throw new Error("Azure OpenAI API key is required");
|
|
848
|
+
}
|
|
849
|
+
return new OpenAIProvider({
|
|
850
|
+
apiKey: config.apiKey,
|
|
851
|
+
model: config.model,
|
|
852
|
+
temperature: config.temperature,
|
|
853
|
+
maxRetries: config.maxRetries,
|
|
854
|
+
baseUrl: config.baseUrl
|
|
855
|
+
// Azure endpoint
|
|
856
|
+
});
|
|
857
|
+
case "openrouter":
|
|
858
|
+
if (!config.apiKey) {
|
|
859
|
+
throw new Error("OpenRouter API key is required");
|
|
860
|
+
}
|
|
861
|
+
return new OpenAIProvider({
|
|
862
|
+
apiKey: config.apiKey,
|
|
863
|
+
model: config.model,
|
|
864
|
+
temperature: config.temperature,
|
|
865
|
+
maxRetries: config.maxRetries,
|
|
866
|
+
baseUrl: config.baseUrl || "https://openrouter.ai/api/v1"
|
|
867
|
+
});
|
|
868
|
+
default:
|
|
869
|
+
throw new Error(`Unknown provider type: ${config.type}`);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
};
|
|
873
|
+
|
|
874
|
+
// src/validation/index.ts
|
|
875
|
+
var TranslationValidationPipeline = class {
|
|
876
|
+
constructor(config = {}) {
|
|
877
|
+
this.config = config;
|
|
878
|
+
}
|
|
879
|
+
async validate(result, sourceUnit) {
|
|
880
|
+
const errors = [];
|
|
881
|
+
const warnings = [];
|
|
882
|
+
let confidence = 1;
|
|
883
|
+
if (!result.translatedText || result.translatedText.trim().length === 0) {
|
|
884
|
+
errors.push({
|
|
885
|
+
type: "EMPTY_TRANSLATION",
|
|
886
|
+
message: "Translation is empty"
|
|
887
|
+
});
|
|
888
|
+
return { isValid: false, errors, warnings, confidence: 0 };
|
|
889
|
+
}
|
|
890
|
+
if (this.config.preservePlaceholders !== false) {
|
|
891
|
+
const placeholderErrors = this.validatePlaceholders(sourceUnit, result);
|
|
892
|
+
errors.push(...placeholderErrors);
|
|
893
|
+
if (placeholderErrors.length > 0) {
|
|
894
|
+
confidence *= 0.5;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
const semanticWarnings = this.validateSemantics(sourceUnit, result);
|
|
898
|
+
warnings.push(...semanticWarnings);
|
|
899
|
+
if (semanticWarnings.length > 0) {
|
|
900
|
+
confidence *= 0.8;
|
|
901
|
+
}
|
|
902
|
+
if (this.config.preventSourceLeakage !== false) {
|
|
903
|
+
const leakageDetected = this.detectSourceLeakage(sourceUnit, result);
|
|
904
|
+
if (leakageDetected) {
|
|
905
|
+
errors.push({
|
|
906
|
+
type: "SOURCE_LEAKAGE",
|
|
907
|
+
message: "Translation appears to be in source language"
|
|
908
|
+
});
|
|
909
|
+
confidence *= 0.3;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
if (this.config.brandNames && this.config.brandNames.length > 0) {
|
|
913
|
+
const brandWarnings = this.validateBrandNames(sourceUnit, result);
|
|
914
|
+
warnings.push(...brandWarnings);
|
|
915
|
+
}
|
|
916
|
+
const isValid = errors.length === 0;
|
|
917
|
+
return { isValid, errors, warnings, confidence };
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* Validate that all placeholders are preserved
|
|
921
|
+
*/
|
|
922
|
+
validatePlaceholders(sourceUnit, result) {
|
|
923
|
+
const errors = [];
|
|
924
|
+
const sourcePlaceholders = new Set(sourceUnit.placeholders);
|
|
925
|
+
const translatedPlaceholders = new Set(extractPlaceholders(result.translatedText));
|
|
926
|
+
for (const placeholder of sourcePlaceholders) {
|
|
927
|
+
if (!translatedPlaceholders.has(placeholder)) {
|
|
928
|
+
errors.push({
|
|
929
|
+
type: "MISSING_PLACEHOLDER",
|
|
930
|
+
message: `Missing placeholder: ${placeholder}`,
|
|
931
|
+
field: placeholder
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
for (const placeholder of translatedPlaceholders) {
|
|
936
|
+
if (!sourcePlaceholders.has(placeholder)) {
|
|
937
|
+
errors.push({
|
|
938
|
+
type: "EXTRA_PLACEHOLDER",
|
|
939
|
+
message: `Unexpected placeholder: ${placeholder}`,
|
|
940
|
+
field: placeholder
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
return errors;
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* Validate semantic properties like length ratio
|
|
948
|
+
*/
|
|
949
|
+
validateSemantics(sourceUnit, result) {
|
|
950
|
+
const warnings = [];
|
|
951
|
+
const maxLengthRatio = this.config.maxLengthRatio || 3;
|
|
952
|
+
const sourceLength = sourceUnit.sourceText.length;
|
|
953
|
+
const translatedLength = result.translatedText.length;
|
|
954
|
+
const ratio = translatedLength / sourceLength;
|
|
955
|
+
if (ratio > maxLengthRatio) {
|
|
956
|
+
warnings.push({
|
|
957
|
+
type: "LENGTH_RATIO_EXCEEDED",
|
|
958
|
+
message: `Translation is ${ratio.toFixed(1)}x longer than source (max: ${maxLengthRatio}x)`
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
return warnings;
|
|
962
|
+
}
|
|
963
|
+
/**
|
|
964
|
+
* Detect if translation leaked source language
|
|
965
|
+
*/
|
|
966
|
+
detectSourceLeakage(sourceUnit, result) {
|
|
967
|
+
const normalizedSource = sourceUnit.sourceText.toLowerCase().trim();
|
|
968
|
+
const normalizedTranslation = result.translatedText.toLowerCase().trim();
|
|
969
|
+
if (normalizedSource === normalizedTranslation) {
|
|
970
|
+
return true;
|
|
971
|
+
}
|
|
972
|
+
const sourceWords = new Set(normalizedSource.split(/\s+/));
|
|
973
|
+
const translationWords = new Set(normalizedTranslation.split(/\s+/));
|
|
974
|
+
let matchCount = 0;
|
|
975
|
+
for (const word of sourceWords) {
|
|
976
|
+
if (translationWords.has(word)) {
|
|
977
|
+
matchCount++;
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
const similarity = matchCount / sourceWords.size;
|
|
981
|
+
return similarity > 0.8;
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* Validate that brand names are preserved
|
|
985
|
+
*/
|
|
986
|
+
validateBrandNames(sourceUnit, result) {
|
|
987
|
+
const warnings = [];
|
|
988
|
+
const brandNames = this.config.brandNames || [];
|
|
989
|
+
for (const brandName of brandNames) {
|
|
990
|
+
const sourceHasBrand = sourceUnit.sourceText.includes(brandName);
|
|
991
|
+
const translationHasBrand = result.translatedText.includes(brandName);
|
|
992
|
+
if (sourceHasBrand && !translationHasBrand) {
|
|
993
|
+
warnings.push({
|
|
994
|
+
type: "MISSING_BRAND_NAME",
|
|
995
|
+
message: `Brand name "${brandName}" not preserved in translation`,
|
|
996
|
+
field: brandName
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
return warnings;
|
|
1001
|
+
}
|
|
1002
|
+
};
|
|
1003
|
+
var AtomicFileWriter = class {
|
|
1004
|
+
/**
|
|
1005
|
+
* Get output file path based on configuration
|
|
1006
|
+
*/
|
|
1007
|
+
static getOutputPath(targetLanguage, outputConfig) {
|
|
1008
|
+
const pattern = outputConfig.fileNaming || "{shortCode}.json";
|
|
1009
|
+
const filename = pattern.replace(/\{shortCode\}/g, targetLanguage.shortCode).replace(/\{language\}/g, targetLanguage.language);
|
|
1010
|
+
return join(outputConfig.dir, filename);
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Write translations to a file atomically
|
|
1014
|
+
*/
|
|
1015
|
+
async writeTranslations(filePath, translations) {
|
|
1016
|
+
const dir = dirname(filePath);
|
|
1017
|
+
if (!existsSync(dir)) {
|
|
1018
|
+
mkdirSync(dir, { recursive: true });
|
|
1019
|
+
}
|
|
1020
|
+
let existing = {};
|
|
1021
|
+
if (existsSync(filePath)) {
|
|
1022
|
+
try {
|
|
1023
|
+
const content = readFileSync(filePath, "utf-8");
|
|
1024
|
+
existing = JSON.parse(content);
|
|
1025
|
+
} catch (error) {
|
|
1026
|
+
console.warn(`Failed to read existing file ${filePath}, starting fresh:`, error);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
const merged = this.deepMerge(existing, translations);
|
|
1030
|
+
const tempPath = `${filePath}.tmp`;
|
|
1031
|
+
try {
|
|
1032
|
+
writeFileSync(tempPath, JSON.stringify(merged, null, 2), "utf-8");
|
|
1033
|
+
renameSync(tempPath, filePath);
|
|
1034
|
+
} catch (error) {
|
|
1035
|
+
if (existsSync(tempPath)) {
|
|
1036
|
+
unlinkSync(tempPath);
|
|
1037
|
+
}
|
|
1038
|
+
throw error;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Deep merge two objects, preserving existing values not being updated
|
|
1043
|
+
*/
|
|
1044
|
+
deepMerge(target, source) {
|
|
1045
|
+
if (!source || typeof source !== "object") {
|
|
1046
|
+
return source;
|
|
1047
|
+
}
|
|
1048
|
+
if (Array.isArray(source)) {
|
|
1049
|
+
return source;
|
|
1050
|
+
}
|
|
1051
|
+
const result = { ...target };
|
|
1052
|
+
for (const [key, value] of Object.entries(source)) {
|
|
1053
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
1054
|
+
result[key] = this.deepMerge(result[key] || {}, value);
|
|
1055
|
+
} else {
|
|
1056
|
+
result[key] = value;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
return result;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Convert flat key-value pairs to nested object structure
|
|
1063
|
+
*/
|
|
1064
|
+
static flatToNested(flat) {
|
|
1065
|
+
const result = {};
|
|
1066
|
+
for (const [keyPath, value] of Object.entries(flat)) {
|
|
1067
|
+
const keys = keyPath.split(".");
|
|
1068
|
+
let current = result;
|
|
1069
|
+
for (let i = 0; i < keys.length - 1; i++) {
|
|
1070
|
+
const key = keys[i];
|
|
1071
|
+
if (!current[key]) {
|
|
1072
|
+
current[key] = {};
|
|
1073
|
+
}
|
|
1074
|
+
current = current[key];
|
|
1075
|
+
}
|
|
1076
|
+
current[keys[keys.length - 1]] = value;
|
|
1077
|
+
}
|
|
1078
|
+
return result;
|
|
1079
|
+
}
|
|
1080
|
+
};
|
|
1081
|
+
|
|
1082
|
+
// src/prompts/index.ts
|
|
1083
|
+
var PromptManager = class {
|
|
1084
|
+
config;
|
|
1085
|
+
constructor(config) {
|
|
1086
|
+
this.config = config || {};
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Get the system prompt for translation
|
|
1090
|
+
* Always returns the core SDK system prompt with optional customizations
|
|
1091
|
+
*/
|
|
1092
|
+
getSystemPrompt(context) {
|
|
1093
|
+
let prompt = this.getCoreSystemPrompt(context);
|
|
1094
|
+
if (this.config.formatting) {
|
|
1095
|
+
prompt += `
|
|
1096
|
+
|
|
1097
|
+
${this.getFormattingInstructions(this.config.formatting)}`;
|
|
1098
|
+
}
|
|
1099
|
+
if (this.config.glossary && Object.keys(this.config.glossary).length > 0) {
|
|
1100
|
+
prompt += `
|
|
1101
|
+
|
|
1102
|
+
${this.getGlossaryInstructions(this.config.glossary)}`;
|
|
1103
|
+
}
|
|
1104
|
+
if (this.config.brandVoice) {
|
|
1105
|
+
prompt += `
|
|
1106
|
+
|
|
1107
|
+
Brand voice: ${this.config.brandVoice}`;
|
|
1108
|
+
}
|
|
1109
|
+
if (this.config.customContext) {
|
|
1110
|
+
prompt += `
|
|
1111
|
+
|
|
1112
|
+
${this.config.customContext}`;
|
|
1113
|
+
}
|
|
1114
|
+
if (this.config.systemPrompt) {
|
|
1115
|
+
prompt += `
|
|
1116
|
+
|
|
1117
|
+
${this.config.systemPrompt}`;
|
|
1118
|
+
}
|
|
1119
|
+
return prompt;
|
|
1120
|
+
}
|
|
1121
|
+
/**
|
|
1122
|
+
* Get core SDK system prompt (always used as base)
|
|
1123
|
+
*/
|
|
1124
|
+
getCoreSystemPrompt(context) {
|
|
1125
|
+
return `You are a professional translator specializing in software localization.
|
|
1126
|
+
|
|
1127
|
+
Your task is to translate the provided strings accurately into ${context.targetLanguage} (${context.targetCode}).
|
|
1128
|
+
|
|
1129
|
+
CRITICAL RULES:
|
|
1130
|
+
1. **Preserve all placeholders exactly as they appear** - This includes {variable}, {{variable}}, $variable, %s, %d, and any other template syntax
|
|
1131
|
+
2. **Maintain the same structure** - Return translations in the exact same JSON array format and order
|
|
1132
|
+
3. **Context awareness** - Consider the context of UI strings, messages, and technical terms
|
|
1133
|
+
4. **Natural language** - Produce natural, idiomatic translations that native speakers would use
|
|
1134
|
+
5. **Consistency** - Maintain consistent terminology throughout all translations
|
|
1135
|
+
6. **No additions or omissions** - Translate only what is provided, nothing more, nothing less
|
|
1136
|
+
|
|
1137
|
+
OUTPUT FORMAT:
|
|
1138
|
+
Return ONLY a valid JSON array of translated strings in the same order as the input.
|
|
1139
|
+
Do not include any explanations, comments, or markdown formatting.`;
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Get formatting instructions based on style
|
|
1143
|
+
*/
|
|
1144
|
+
getFormattingInstructions(formatting) {
|
|
1145
|
+
const styles = {
|
|
1146
|
+
formal: "Use formal language and respectful address. Suitable for professional, business, or official contexts.",
|
|
1147
|
+
casual: "Use casual, friendly language. Suitable for consumer apps and informal communication.",
|
|
1148
|
+
technical: "Use precise technical terminology. Prioritize accuracy over naturalness. Keep technical terms in English when appropriate."
|
|
1149
|
+
};
|
|
1150
|
+
return `Tone and style: ${styles[formatting]}`;
|
|
1151
|
+
}
|
|
1152
|
+
/**
|
|
1153
|
+
* Get glossary instructions
|
|
1154
|
+
*/
|
|
1155
|
+
getGlossaryInstructions(glossary) {
|
|
1156
|
+
const terms = Object.entries(glossary).map(([source, target]) => `- "${source}" \u2192 "${target}"`).join("\n");
|
|
1157
|
+
return `GLOSSARY - Use these exact translations for the following terms:
|
|
1158
|
+
${terms}`;
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Get the current prompt version for tracking
|
|
1162
|
+
*/
|
|
1163
|
+
getPromptVersion() {
|
|
1164
|
+
return this.config.userPrompt || this.config.customContext ? 2 : 1;
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* Build user prompt from source units
|
|
1168
|
+
* Uses custom userPrompt if provided, otherwise defaults to JSON array of source texts
|
|
1169
|
+
*/
|
|
1170
|
+
getUserPrompt(sourceTexts) {
|
|
1171
|
+
if (this.config.userPrompt && this.config.userPrompt.length > 0) {
|
|
1172
|
+
const customPrompt = this.config.userPrompt.join("\n");
|
|
1173
|
+
return `${customPrompt}
|
|
1174
|
+
|
|
1175
|
+
${JSON.stringify(sourceTexts, null, 0)}`;
|
|
1176
|
+
}
|
|
1177
|
+
return JSON.stringify(sourceTexts, null, 0);
|
|
1178
|
+
}
|
|
1179
|
+
};
|
|
1180
|
+
var TranslationCompiler = class {
|
|
1181
|
+
constructor(config) {
|
|
1182
|
+
this.config = config;
|
|
1183
|
+
const ledgerPath = config.advanced?.ledgerPath || "./.translatronx/ledger.sqlite";
|
|
1184
|
+
this.ledger = new translatronxLedger(ledgerPath);
|
|
1185
|
+
this.planner = new IncrementalTranslationPlanner(this.ledger);
|
|
1186
|
+
this.writer = new AtomicFileWriter();
|
|
1187
|
+
this.promptManager = new PromptManager(this.config.prompts);
|
|
1188
|
+
}
|
|
1189
|
+
ledger;
|
|
1190
|
+
planner;
|
|
1191
|
+
writer;
|
|
1192
|
+
promptManager;
|
|
1193
|
+
/**
|
|
1194
|
+
* Synchronize translations (main compilation step)
|
|
1195
|
+
*/
|
|
1196
|
+
async sync(_options = {}) {
|
|
1197
|
+
const startTime = /* @__PURE__ */ new Date();
|
|
1198
|
+
const spinner = ora("Extracting source strings...").start();
|
|
1199
|
+
try {
|
|
1200
|
+
const extractor = new JsonExtractor();
|
|
1201
|
+
const extractorConfig = this.config.extractors[0];
|
|
1202
|
+
const sourceUnits = await extractor.extract([], extractorConfig);
|
|
1203
|
+
spinner.succeed(`Extracted ${sourceUnits.length} translatable strings`);
|
|
1204
|
+
spinner.start("Updating source hashes...");
|
|
1205
|
+
const configHash = computeHash(JSON.stringify(this.config));
|
|
1206
|
+
const runId = this.ledger.startRun(this.config.providers[0].model, configHash);
|
|
1207
|
+
for (const unit of sourceUnits) {
|
|
1208
|
+
this.ledger.updateSourceHash(unit.keyPath, unit.sourceHash, void 0, runId);
|
|
1209
|
+
}
|
|
1210
|
+
spinner.succeed("Source hashes updated");
|
|
1211
|
+
spinner.start("Creating translation plan...");
|
|
1212
|
+
const plan = await this.planner.createPlan(sourceUnits, this.config.targetLanguages);
|
|
1213
|
+
if (plan.batches.length === 0) {
|
|
1214
|
+
spinner.succeed(chalk4.green("\u2713 All translations are up to date!"));
|
|
1215
|
+
this.ledger.completeRun(runId, 0, 0, 0);
|
|
1216
|
+
return {
|
|
1217
|
+
runId,
|
|
1218
|
+
startedAt: startTime,
|
|
1219
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1220
|
+
totalUnits: sourceUnits.length,
|
|
1221
|
+
translatedUnits: 0,
|
|
1222
|
+
failedUnits: 0,
|
|
1223
|
+
skippedUnits: sourceUnits.length,
|
|
1224
|
+
tokensIn: 0,
|
|
1225
|
+
tokensOut: 0,
|
|
1226
|
+
costEstimateUsd: 0,
|
|
1227
|
+
model: this.config.providers[0].model
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
spinner.succeed(`Created plan: ${plan.batches.length} batches, ~$${plan.estimatedCost.toFixed(4)} estimated cost`);
|
|
1231
|
+
const provider = ProviderFactory.createProvider(this.config.providers[0]);
|
|
1232
|
+
const validator = new TranslationValidationPipeline(this.config.validation);
|
|
1233
|
+
let translatedUnits = 0;
|
|
1234
|
+
let failedUnits = 0;
|
|
1235
|
+
let totalTokensIn = 0;
|
|
1236
|
+
let totalTokensOut = 0;
|
|
1237
|
+
spinner.start(`Translating ${plan.totalUnits} strings...`);
|
|
1238
|
+
for (const batch of plan.batches) {
|
|
1239
|
+
try {
|
|
1240
|
+
const targetLangObj = this.config.targetLanguages.find((l) => l.shortCode === batch.targetLanguage);
|
|
1241
|
+
if (!targetLangObj) continue;
|
|
1242
|
+
const langName = targetLangObj.language;
|
|
1243
|
+
const systemPrompt = this.promptManager.getSystemPrompt({
|
|
1244
|
+
targetLanguage: langName,
|
|
1245
|
+
targetCode: batch.targetLanguage,
|
|
1246
|
+
sourceUnits: batch.sourceUnits
|
|
1247
|
+
});
|
|
1248
|
+
const sourceTexts = batch.sourceUnits.map((unit) => unit.sourceText);
|
|
1249
|
+
const userPrompt = this.promptManager.getUserPrompt(sourceTexts);
|
|
1250
|
+
const results = await provider.translate(batch, {
|
|
1251
|
+
system: systemPrompt,
|
|
1252
|
+
user: userPrompt,
|
|
1253
|
+
temperature: this.config.providers[0].temperature
|
|
1254
|
+
});
|
|
1255
|
+
for (let i = 0; i < results.length; i++) {
|
|
1256
|
+
const result = results[i];
|
|
1257
|
+
const sourceUnit = batch.sourceUnits[i];
|
|
1258
|
+
const validation = await validator.validate(result, sourceUnit);
|
|
1259
|
+
if (!validation.isValid) {
|
|
1260
|
+
console.error(chalk4.red(`\u2717 Validation failed for ${sourceUnit.keyPath}:`), validation.errors);
|
|
1261
|
+
failedUnits++;
|
|
1262
|
+
this.ledger.updateSyncStatus(sourceUnit.keyPath, batch.targetLanguage, "", "FAILED");
|
|
1263
|
+
continue;
|
|
1264
|
+
}
|
|
1265
|
+
const outputFile = AtomicFileWriter.getOutputPath(targetLangObj, this.config.output);
|
|
1266
|
+
const translations = AtomicFileWriter.flatToNested({ [sourceUnit.keyPath]: result.translatedText });
|
|
1267
|
+
await this.writer.writeTranslations(outputFile, translations);
|
|
1268
|
+
const targetHash = computeHash(result.translatedText);
|
|
1269
|
+
this.ledger.updateSyncStatus(
|
|
1270
|
+
sourceUnit.keyPath,
|
|
1271
|
+
batch.targetLanguage,
|
|
1272
|
+
targetHash,
|
|
1273
|
+
"CLEAN",
|
|
1274
|
+
provider.getModelFingerprint(),
|
|
1275
|
+
this.promptManager.getPromptVersion()
|
|
1276
|
+
);
|
|
1277
|
+
translatedUnits++;
|
|
1278
|
+
}
|
|
1279
|
+
totalTokensIn += batch.sourceUnits.length * 50;
|
|
1280
|
+
totalTokensOut += batch.sourceUnits.length * 50;
|
|
1281
|
+
} catch (error) {
|
|
1282
|
+
console.error(chalk4.red(`\u2717 Failed to translate batch ${batch.batchId}:`), error);
|
|
1283
|
+
failedUnits += batch.sourceUnits.length;
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
spinner.succeed(chalk4.green(`\u2713 Translated ${translatedUnits} strings, ${failedUnits} failed`));
|
|
1287
|
+
const actualCost = provider.estimateCost({ ...plan.batches[0], sourceUnits: sourceUnits.slice(0, translatedUnits) });
|
|
1288
|
+
this.ledger.completeRun(runId, totalTokensIn, totalTokensOut, actualCost);
|
|
1289
|
+
return {
|
|
1290
|
+
runId,
|
|
1291
|
+
startedAt: startTime,
|
|
1292
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1293
|
+
totalUnits: sourceUnits.length,
|
|
1294
|
+
translatedUnits,
|
|
1295
|
+
failedUnits,
|
|
1296
|
+
skippedUnits: sourceUnits.length - translatedUnits - failedUnits,
|
|
1297
|
+
tokensIn: totalTokensIn,
|
|
1298
|
+
tokensOut: totalTokensOut,
|
|
1299
|
+
costEstimateUsd: actualCost,
|
|
1300
|
+
model: this.config.providers[0].model
|
|
1301
|
+
};
|
|
1302
|
+
} catch (error) {
|
|
1303
|
+
spinner.fail("Translation failed");
|
|
1304
|
+
throw error;
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
/**
|
|
1308
|
+
* Close ledger connection
|
|
1309
|
+
*/
|
|
1310
|
+
close() {
|
|
1311
|
+
this.ledger.close();
|
|
1312
|
+
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Retry failed translation batches
|
|
1315
|
+
*/
|
|
1316
|
+
async retryFailed(options) {
|
|
1317
|
+
const spinner = ora("Identifying failed translations...").start();
|
|
1318
|
+
try {
|
|
1319
|
+
const failedItems = this.ledger.getFailedItems(options.lang, options.batch);
|
|
1320
|
+
if (failedItems.length === 0) {
|
|
1321
|
+
spinner.succeed(chalk4.green("\u2713 No failed translations found"));
|
|
1322
|
+
return {
|
|
1323
|
+
recoveredUnits: 0,
|
|
1324
|
+
remainingFailed: 0,
|
|
1325
|
+
tokensIn: 0,
|
|
1326
|
+
tokensOut: 0
|
|
1327
|
+
};
|
|
1328
|
+
}
|
|
1329
|
+
spinner.succeed(`Found ${failedItems.length} failed translations to retry`);
|
|
1330
|
+
if (options.dryRun) {
|
|
1331
|
+
console.log(chalk4.blue("\nDry-run mode - showing what would be retried:"));
|
|
1332
|
+
failedItems.forEach((item) => {
|
|
1333
|
+
console.log(chalk4.gray(` - ${item.key_path} (${item.lang_code})`));
|
|
1334
|
+
});
|
|
1335
|
+
return {
|
|
1336
|
+
recoveredUnits: 0,
|
|
1337
|
+
remainingFailed: failedItems.length,
|
|
1338
|
+
tokensIn: 0,
|
|
1339
|
+
tokensOut: 0
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
1342
|
+
const byLanguage = /* @__PURE__ */ new Map();
|
|
1343
|
+
failedItems.forEach((item) => {
|
|
1344
|
+
if (!byLanguage.has(item.lang_code)) {
|
|
1345
|
+
byLanguage.set(item.lang_code, []);
|
|
1346
|
+
}
|
|
1347
|
+
byLanguage.get(item.lang_code)?.push(item);
|
|
1348
|
+
});
|
|
1349
|
+
let recoveredUnits = 0;
|
|
1350
|
+
let remainingFailed = failedItems.length;
|
|
1351
|
+
let totalTokensIn = 0;
|
|
1352
|
+
let totalTokensOut = 0;
|
|
1353
|
+
for (const [langCode, items] of byLanguage) {
|
|
1354
|
+
const targetLangObj = this.config.targetLanguages.find((l) => l.shortCode === langCode);
|
|
1355
|
+
const langName = targetLangObj?.language || langCode;
|
|
1356
|
+
spinner.start(`Retrying ${items.length} strings for ${langName}...`);
|
|
1357
|
+
const batch = {
|
|
1358
|
+
batchId: `retry_${Date.now()}`,
|
|
1359
|
+
sourceUnits: items.map((item) => ({
|
|
1360
|
+
unitId: item.key_path,
|
|
1361
|
+
keyPath: item.key_path,
|
|
1362
|
+
sourceText: "",
|
|
1363
|
+
// Would need to load from source
|
|
1364
|
+
sourceHash: item.value_hash,
|
|
1365
|
+
placeholders: [],
|
|
1366
|
+
sourceFile: "",
|
|
1367
|
+
schemaVersion: 1
|
|
1368
|
+
})),
|
|
1369
|
+
targetLanguage: langCode,
|
|
1370
|
+
deduplicationKey: ""
|
|
1371
|
+
};
|
|
1372
|
+
try {
|
|
1373
|
+
const provider = ProviderFactory.createProvider(this.config.providers[0]);
|
|
1374
|
+
const validator = new TranslationValidationPipeline(this.config.validation);
|
|
1375
|
+
const targetLangObj2 = this.config.targetLanguages.find((l) => l.shortCode === langCode);
|
|
1376
|
+
if (!targetLangObj2) continue;
|
|
1377
|
+
const systemPrompt = this.promptManager.getSystemPrompt({
|
|
1378
|
+
targetLanguage: langName,
|
|
1379
|
+
targetCode: langCode,
|
|
1380
|
+
sourceUnits: batch.sourceUnits
|
|
1381
|
+
});
|
|
1382
|
+
const sourceTexts = batch.sourceUnits.map((unit) => unit.sourceText);
|
|
1383
|
+
const userPrompt = this.promptManager.getUserPrompt(sourceTexts);
|
|
1384
|
+
const results = await provider.translate(batch, {
|
|
1385
|
+
system: systemPrompt,
|
|
1386
|
+
user: userPrompt,
|
|
1387
|
+
temperature: this.config.providers[0].temperature
|
|
1388
|
+
});
|
|
1389
|
+
for (let i = 0; i < results.length; i++) {
|
|
1390
|
+
const translationResult = results[i];
|
|
1391
|
+
const sourceUnit = batch.sourceUnits[i];
|
|
1392
|
+
const validation = await validator.validate(translationResult, sourceUnit);
|
|
1393
|
+
if (validation.isValid) {
|
|
1394
|
+
const outputFile = AtomicFileWriter.getOutputPath(targetLangObj2, this.config.output);
|
|
1395
|
+
const translations = AtomicFileWriter.flatToNested({ [sourceUnit.keyPath]: translationResult.translatedText });
|
|
1396
|
+
await this.writer.writeTranslations(outputFile, translations);
|
|
1397
|
+
const targetHash = computeHash(translationResult.translatedText);
|
|
1398
|
+
this.ledger.updateSyncStatus(
|
|
1399
|
+
sourceUnit.keyPath,
|
|
1400
|
+
langCode,
|
|
1401
|
+
targetHash,
|
|
1402
|
+
"CLEAN",
|
|
1403
|
+
provider.getModelFingerprint(),
|
|
1404
|
+
this.promptManager.getPromptVersion()
|
|
1405
|
+
);
|
|
1406
|
+
recoveredUnits++;
|
|
1407
|
+
remainingFailed--;
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
totalTokensIn += batch.sourceUnits.length * 50;
|
|
1411
|
+
totalTokensOut += batch.sourceUnits.length * 50;
|
|
1412
|
+
spinner.succeed(chalk4.green(`\u2713 Retried ${recoveredUnits} strings for ${langName}`));
|
|
1413
|
+
} catch (error) {
|
|
1414
|
+
console.error(chalk4.red(`\u2717 Failed to retry batch for ${langName}:`), error);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
return {
|
|
1418
|
+
recoveredUnits,
|
|
1419
|
+
remainingFailed,
|
|
1420
|
+
tokensIn: totalTokensIn,
|
|
1421
|
+
tokensOut: totalTokensOut
|
|
1422
|
+
};
|
|
1423
|
+
} catch (error) {
|
|
1424
|
+
spinner.fail("Retry operation failed");
|
|
1425
|
+
throw error;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
};
|
|
1429
|
+
var ReportingSystem = class {
|
|
1430
|
+
constructor(ledger) {
|
|
1431
|
+
this.ledger = ledger;
|
|
1432
|
+
}
|
|
1433
|
+
/**
|
|
1434
|
+
* Get a summary of the latest run
|
|
1435
|
+
*/
|
|
1436
|
+
async getLatestRunSummary() {
|
|
1437
|
+
const run = this.ledger.getLatestRun();
|
|
1438
|
+
if (!run) return null;
|
|
1439
|
+
return {
|
|
1440
|
+
runId: run.run_id,
|
|
1441
|
+
startedAt: run.started_at,
|
|
1442
|
+
finishedAt: run.finished_at,
|
|
1443
|
+
modelUsed: run.model_used,
|
|
1444
|
+
tokensIn: run.tokens_in || 0,
|
|
1445
|
+
tokensOut: run.tokens_out || 0,
|
|
1446
|
+
costEstimateUsd: run.cost_estimate_usd || 0,
|
|
1447
|
+
configHash: run.config_hash
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
/**
|
|
1451
|
+
* Get overall statistics for the project
|
|
1452
|
+
*/
|
|
1453
|
+
async getProjectStats(targetLanguages) {
|
|
1454
|
+
const baseStats = this.ledger.getProjectStats();
|
|
1455
|
+
const coverage = {};
|
|
1456
|
+
let translatedCount = 0;
|
|
1457
|
+
let failedTotal = 0;
|
|
1458
|
+
for (const lang of targetLanguages) {
|
|
1459
|
+
const langStats = this.ledger.getLanguageStats(lang.shortCode);
|
|
1460
|
+
const label = `${lang.language} (${lang.shortCode})`;
|
|
1461
|
+
coverage[label] = langStats.translated;
|
|
1462
|
+
translatedCount += langStats.translated;
|
|
1463
|
+
failedTotal += langStats.failed;
|
|
1464
|
+
}
|
|
1465
|
+
return {
|
|
1466
|
+
totalStrings: baseStats.total_keys,
|
|
1467
|
+
translatedStrings: translatedCount,
|
|
1468
|
+
failedStrings: failedTotal,
|
|
1469
|
+
manualOverrides: baseStats.manual_count,
|
|
1470
|
+
languageCoverage: coverage
|
|
1471
|
+
};
|
|
1472
|
+
}
|
|
1473
|
+
/**
|
|
1474
|
+
* Format a report as a string for CLI output
|
|
1475
|
+
*/
|
|
1476
|
+
formatReport(stats) {
|
|
1477
|
+
let output = chalk4.bold("\n\u{1F4CA} Project Translation Status\n");
|
|
1478
|
+
output += chalk4.gray("-----------------------------------\n");
|
|
1479
|
+
output += `Total unique keys: ${stats.totalStrings}
|
|
1480
|
+
`;
|
|
1481
|
+
output += `Translated: ${chalk4.green(stats.translatedStrings)}
|
|
1482
|
+
`;
|
|
1483
|
+
output += `Failed/Dirty: ${chalk4.red(stats.failedStrings)}
|
|
1484
|
+
`;
|
|
1485
|
+
output += `Manual Overrides: ${chalk4.yellow(stats.manualOverrides)}
|
|
1486
|
+
|
|
1487
|
+
`;
|
|
1488
|
+
output += chalk4.bold("Language Coverage:\n");
|
|
1489
|
+
for (const [lang, count] of Object.entries(stats.languageCoverage)) {
|
|
1490
|
+
const percentage = stats.totalStrings > 0 ? (count / stats.totalStrings * 100).toFixed(1) : "0.0";
|
|
1491
|
+
const color = count === stats.totalStrings ? chalk4.green : chalk4.yellow;
|
|
1492
|
+
output += ` ${lang.padEnd(5)}: ${color(`${count}/${stats.totalStrings}`)} (${percentage}%)
|
|
1493
|
+
`;
|
|
1494
|
+
}
|
|
1495
|
+
return output;
|
|
1496
|
+
}
|
|
1497
|
+
};
|
|
1498
|
+
var program = new Command();
|
|
1499
|
+
program.name("translatronx").description("Deterministic, incremental, build-time translation compiler using LLMs");
|
|
1500
|
+
program.command("sync").description("Synchronize translations (incremental processing)").option("-f, --force", "Force regeneration of manual overrides").option("-v, --verbose", "Enable verbose output with streaming").action(async (options) => {
|
|
1501
|
+
try {
|
|
1502
|
+
console.log(chalk4.blue("\u{1F504} Syncing translations...\n"));
|
|
1503
|
+
const config = await loadConfig();
|
|
1504
|
+
const compiler = new TranslationCompiler(config);
|
|
1505
|
+
const stats = await compiler.sync(options);
|
|
1506
|
+
compiler.close();
|
|
1507
|
+
console.log(chalk4.green("\n\u2705 Translation sync complete!\n"));
|
|
1508
|
+
console.log(chalk4.white("Statistics:"));
|
|
1509
|
+
console.log(chalk4.gray(` Total strings: ${stats.totalUnits}`));
|
|
1510
|
+
console.log(chalk4.green(` Translated: ${stats.translatedUnits}`));
|
|
1511
|
+
console.log(chalk4.red(` Failed: ${stats.failedUnits}`));
|
|
1512
|
+
console.log(chalk4.yellow(` Skipped: ${stats.skippedUnits}`));
|
|
1513
|
+
console.log(chalk4.gray(` Tokens used: ${stats.tokensIn} (input) + ${stats.tokensOut} (output)`));
|
|
1514
|
+
console.log(chalk4.gray(` Duration: ${((stats.finishedAt.getTime() - stats.startedAt.getTime()) / 1e3).toFixed(2)}s
|
|
1515
|
+
`));
|
|
1516
|
+
process.exit(0);
|
|
1517
|
+
} catch (error) {
|
|
1518
|
+
console.error(chalk4.red("\u274C Sync failed:"), error.message);
|
|
1519
|
+
process.exit(1);
|
|
1520
|
+
}
|
|
1521
|
+
});
|
|
1522
|
+
program.command("init").description("Initialize translatronx configuration").action(async () => {
|
|
1523
|
+
try {
|
|
1524
|
+
console.log(chalk4.blue("\u{1F680} Initializing translatronx...\n"));
|
|
1525
|
+
const config = getDefaultConfig();
|
|
1526
|
+
const configContent = `import { defineConfig } from 'translatronx';
|
|
1527
|
+
|
|
1528
|
+
export default defineConfig(${JSON.stringify(config, null, 2)});
|
|
1529
|
+
`;
|
|
1530
|
+
writeFileSync("translatronx.config.ts", configContent, "utf-8");
|
|
1531
|
+
console.log(chalk4.green("\u2705 Created translatronx.config.ts"));
|
|
1532
|
+
console.log(chalk4.gray("\nNext steps:"));
|
|
1533
|
+
console.log(chalk4.gray(" 1. Edit translatronx.config.ts to configure your project"));
|
|
1534
|
+
console.log(chalk4.gray(" 2. Set API key: export OPENAI_API_KEY=your-key"));
|
|
1535
|
+
console.log(chalk4.gray(" 3. Run: translatronx sync\n"));
|
|
1536
|
+
process.exit(0);
|
|
1537
|
+
} catch (error) {
|
|
1538
|
+
console.error(chalk4.red("\u274C Init failed:"), error.message);
|
|
1539
|
+
process.exit(1);
|
|
1540
|
+
}
|
|
1541
|
+
});
|
|
1542
|
+
program.command("status").description("Display coverage statistics and system state").action(async () => {
|
|
1543
|
+
try {
|
|
1544
|
+
console.log(chalk4.blue("\u{1F4CA} Checking status...\n"));
|
|
1545
|
+
const config = await loadConfig();
|
|
1546
|
+
const ledgerPath = config.advanced?.ledgerPath || "./.translatronx/ledger.sqlite";
|
|
1547
|
+
const ledger = new translatronxLedger(ledgerPath);
|
|
1548
|
+
const reporting = new ReportingSystem(ledger);
|
|
1549
|
+
const stats = await reporting.getProjectStats(config.targetLanguages);
|
|
1550
|
+
const latestRun = await reporting.getLatestRunSummary();
|
|
1551
|
+
console.log(reporting.formatReport(stats));
|
|
1552
|
+
if (latestRun) {
|
|
1553
|
+
console.log(chalk4.bold("Latest Run:"));
|
|
1554
|
+
console.log(chalk4.gray(` Run ID: ${latestRun.runId}`));
|
|
1555
|
+
console.log(chalk4.gray(` Model: ${latestRun.modelUsed}`));
|
|
1556
|
+
console.log(chalk4.gray(` Cost: $${latestRun.costEstimateUsd.toFixed(4)}`));
|
|
1557
|
+
console.log(chalk4.gray(` Duration: ${latestRun.finishedAt ? "Completed" : "Interrupted"}
|
|
1558
|
+
`));
|
|
1559
|
+
}
|
|
1560
|
+
ledger.close();
|
|
1561
|
+
process.exit(0);
|
|
1562
|
+
} catch (error) {
|
|
1563
|
+
console.error(chalk4.red("\u274C Status check failed:"), error.message);
|
|
1564
|
+
process.exit(1);
|
|
1565
|
+
}
|
|
1566
|
+
});
|
|
1567
|
+
program.command("check").description("Validate target files without making changes").action(async () => {
|
|
1568
|
+
try {
|
|
1569
|
+
console.log(chalk4.blue("\u2713 Checking translations...\n"));
|
|
1570
|
+
console.log(chalk4.yellow("\u26A0\uFE0F Check command not fully implemented yet"));
|
|
1571
|
+
console.log(chalk4.gray(" This would validate all target files without making changes\n"));
|
|
1572
|
+
process.exit(0);
|
|
1573
|
+
} catch (error) {
|
|
1574
|
+
console.error(chalk4.red("\u274C Check failed:"), error.message);
|
|
1575
|
+
process.exit(1);
|
|
1576
|
+
}
|
|
1577
|
+
});
|
|
1578
|
+
program.command("retry").description("Retry failed translation batches").option("--batch <id>", "Specific batch ID to retry").option("--lang <code>", "Specific language to retry").option("--dry-run", "Show what would be retried without making changes").action(async (options) => {
|
|
1579
|
+
try {
|
|
1580
|
+
console.log(chalk4.blue("\u{1F504} Retrying failed translations...\n"));
|
|
1581
|
+
const config = await loadConfig();
|
|
1582
|
+
const compiler = new TranslationCompiler(config);
|
|
1583
|
+
const stats = await compiler.retryFailed(options);
|
|
1584
|
+
compiler.close();
|
|
1585
|
+
console.log(chalk4.green("\n\u2705 Retry operation complete!\n"));
|
|
1586
|
+
console.log(chalk4.white("Retry Statistics:"));
|
|
1587
|
+
console.log(chalk4.green(` Successfully retried: ${stats.recoveredUnits}`));
|
|
1588
|
+
console.log(chalk4.red(` Still failed: ${stats.remainingFailed}`));
|
|
1589
|
+
console.log(chalk4.gray(` Tokens used: ${stats.tokensIn} (input) + ${stats.tokensOut} (output)
|
|
1590
|
+
`));
|
|
1591
|
+
process.exit(0);
|
|
1592
|
+
} catch (error) {
|
|
1593
|
+
console.error(chalk4.red("\u274C Retry failed:"), error.message);
|
|
1594
|
+
process.exit(1);
|
|
1595
|
+
}
|
|
1596
|
+
});
|
|
1597
|
+
program.parse();
|
|
1598
|
+
//# sourceMappingURL=cli.js.map
|
|
1599
|
+
//# sourceMappingURL=cli.js.map
|