smartloc 1.2.1 → 2.0.0

package/src/adapters/adapter-base.ts

@@ -0,0 +1,58 @@
+import { IFormatAdapter } from '../cli/interfaces';
+import fs from 'fs-extra';
+import path from 'path';
+import type { Translation } from '../core/load';
+
+export abstract class AdapterBase implements IFormatAdapter {
+
+    constructor(private target: string | null) {
+    }
+
+    async loadFile(filePath: string): Promise<Translation> {
+        if (!await fs.pathExists(filePath)) {
+            throw new Error(`File ${filePath} does not exist`);
+        }
+        const content = await fs.readFile(filePath, 'utf8');
+        return await this.doLoadFile(content);
+    }
+
+    async loadLocale(locale: string): Promise<Translation> {
+        if (!this.target || !await fs.pathExists(this.target)) {
+            return {
+                resources: {},
+                targetLanguage: locale,
+            };
+        }
+        return await this.loadFile(path.join(this.target, `${locale}.json`));
+    }
+
+    protected abstract doLoadFile(content: string): Promise<Translation>;
+
+    async loadLocales(locales: string[]): Promise<Translation[]> {
+        if (!this.target || !await fs.pathExists(this.target)) {
+            return locales.map<Translation>(x => ({
+                resources: {},
+                targetLanguage: x,
+            }));
+        }
+        const ret: Translation[] = [];
+        for (const f of locales) {
+            const t = await this.loadFile(path.join(this.target, `${f}.json`));
+            t.targetLanguage = f;
+            ret.push(t);
+        }
+        return ret;
+    }
+
+    async write(translation: Translation) {
+        if (!this.target) {
+            throw new Error('Target directory not set');
+        }
+        await fs.ensureDir(this.target);
+        const content = await this.contentFor(translation);
+        const toFile = path.join(this.target, translation.targetLanguage + '.json');
+        await fs.writeFile(toFile, content, 'utf8');
+    }
+
+    protected abstract contentFor(translation: Translation): Promise<string>;
+}

package/src/adapters/index.ts

@@ -0,0 +1,31 @@
+import type { IFormatAdapter } from '../cli/interfaces';
+import { XliffAdapter } from './xliff';
+import { JsonAdapter } from './json';
+import type { Translation } from '../core/load';
+
+export function getAdapter(format: string, dir: string): IFormatAdapter {
+    switch (format) {
+        case 'xliff':
+            return new XliffAdapter(dir);
+        case 'json':
+            return new JsonAdapter(dir);
+        default:
+            throw new Error('Unknown translation format: ' + format);
+    }
+}
+
+export async function loadLocaleFrom(path: string): Promise<Translation | null> {
+    const [ext] = /\.[a-z]+$/.exec(path.toLowerCase()) || [null];
+    let adapter: IFormatAdapter;
+    switch (ext) {
+        case '.json':
+            adapter = new JsonAdapter(null);
+            break;
+        case '.xlf':
+            adapter = new XliffAdapter(null);
+            break;
+        default:
+            return null;
+    }
+    return await adapter.loadFile(path);
+}

package/src/adapters/json.ts

@@ -0,0 +1,18 @@
+import { type Translation, type GroupedTranslation, ungroupTranslation } from '../core/load';
+import stableJson from 'json-stable-stringify';
+import { AdapterBase } from './adapter-base';
+import { toGroupedTranslation } from '../cli/utils';
+
+export class JsonAdapter extends AdapterBase {
+
+    protected async doLoadFile(content: string): Promise<Translation> {
+        const parsed = JSON.parse(content) as GroupedTranslation;
+        return ungroupTranslation(parsed);
+    }
+
+    protected async contentFor(translation: Translation) {
+        const data = toGroupedTranslation(translation);
+        const content = stableJson(data, { space: '    ' });
+        return content!;
+    }
+}

package/src/adapters/xliff.ts

@@ -0,0 +1,30 @@
+// @ts-ignore
+import js2xliff from 'xliff/js2xliff';
+// @ts-ignore
+import xliff2js from 'xliff/xliff2js';
+import type { Translation, GroupedTranslation } from '../core/load';
+import { toGroupedTranslation } from '../cli/utils';
+import { ungroupTranslation } from '../core/load';
+import { AdapterBase } from './adapter-base';
+
+export class XliffAdapter extends AdapterBase {
+
+    protected async doLoadFile(content: string): Promise<Translation> {
+        const parsed = await new Promise<GroupedTranslation>((res, rej) => xliff2js(content, (err: any, val: any) => err ? rej(err) : res(val)));
+        return ungroupTranslation(parsed);
+    }
+
+    protected async contentFor(translation: Translation) {
+        const data = toGroupedTranslation(translation);
+        return await new Promise<string>((res, rej) => js2xliff(data, async (err: any, result: any) => {
+            if (err) {
+                return rej(err);
+            }
+            try {
+                res(result);
+            } catch (e) {
+                rej(e);
+            }
+        }));
+    }
+}

package/src/cli/cli.ts

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+
+import yargs, { Argv } from 'yargs';
+import path from 'path';
+import { collect } from './collect';
+import { reconciliate } from './reconciliate';
+import { getAdapter } from '../adapters';
+
+// === arguments parsing ===
+
+function isValidLocale(x: string) {
+    return typeof x === 'string' && !!x && /^[a-zA-Z]+(\-[a-zA-Z]+)?$/.test(x);
+}
+
+function defaultArgs<T>(b: Argv<T>): Argv<T & { format: string; outDir: string; }> {
+    return b.positional('format', {
+        describe: 'The output format',
+        choices: ['json', 'xliff'],
+        default: 'xliff',
+    })
+        .positional('outDir', {
+            describe: 'The translation output directory',
+            default: 'i18n',
+            normalize: true,
+        })
+}
+
+yargs
+    .scriptName('smartloc')
+    .command('add', 'Add a new locale', b => defaultArgs(b).demandCommand()
+        , args => {
+            console.log(args);
+        })
+    .command('collect', 'Collects all strings to translate', b => defaultArgs(b)
+        .positional('source', {
+            describe: 'The source directory where to look for translations (defaults to current working directory)',
+            type: 'string',
+            normalize: true,
+        })
+        .positional('defaultLocale', {
+            description: 'Sets the default locale (the locale used in code for your strings)',
+            type: 'string',
+            coerce: x => {
+                if (!isValidLocale(x)) {
+                    console.error('Invalid locale ' + x);
+                    process.exit(1);
+                }
+                return x;
+            },
+        })
+        .positional('generateDefault', {
+            description: 'Also generates a translation file for the default locale',
+            type: 'boolean',
+        })
+        .positional('generateDefault', {
+            description: 'Also generates the default locale translation file (which does not need to be translated)',
+            type: 'boolean',
+        })
+        .positional('locales', {
+            description: 'Comma separated list of locales to generate (others than the default one)',
+            type: 'string',
+            coerce: x => {
+                if (typeof x !== 'string' || !x || !/^[a-zA-Z]+(\-[a-zA-Z]+)?$/.test(x)) {
+                    console.error('Invalid locale ' + x);
+                    process.exit(1);
+                }
+                return x;
+            },
+        })
+        , async args => {
+            try {
+
+                const locales = args.locales?.split(',')
+                    .map(x => x.trim())
+                    .filter(x => x !== args.defaultLocale);
+                if (!locales?.length) {
+                    console.error('You must provide at least one locale thats different from your default locale')
+                    process.exit(1);
+                }
+                if (locales.some(x => !isValidLocale(x))) {
+                    console.error('--locales argument is invalid')
+                    process.exit(1);
+                }
+
+                // create adapter
+                const outdir = path.join(process.cwd(), args.outDir);
+                const adapter = getAdapter(args.format, outdir);
+
+                // collect from sources
+                console.log('Collecting from sources...');
+                const { collected, files } = await collect(args.source || '.', adapter, args.defaultLocale);
+                console.log(`...found ${Object.keys(collected.resources).length} translatable items over ${files} files with default language ${collected.targetLanguage}`);
+
+
+                // load other locales
+                const others = (await adapter.loadLocales(locales))
+                    .filter(x => x.targetLanguage !== collected.targetLanguage);
+
+                console.log(`Updating ${others.length} other translations...`);
+
+                // reconciliate existing
+                for (const l of others) {
+                    const collectedCopy = JSON.parse(JSON.stringify(collected));
+                    const { dirty, missing, same } = reconciliate(l, collectedCopy);
+                    const feats: string[] = [];
+                    if (dirty) {
+                        feats.push(`${dirty} dirty translations (source changed)`);
+                    }
+                    if (missing) {
+                        feats.push(`${missing} missing translations`);
+                    }
+                    if (same.length) {
+                        feats.push(`${same.length} translations matching original (ex: ${same[0]})`);
+                    }
+                    if (!feats.length) {
+                        console.log(`${l.targetLanguage} is OK ♥`);
+                    } else {
+                        console.log(`${l.targetLanguage} has ${feats.join(', ')}`);
+                    }
+                }
+
+                // write others
+                console.log(`Writing results...`);
+                for (const l of others) {
+                    await adapter.write(l);
+                }
+
+                if (args.defaultLocale) {
+                    await adapter.write(collected);
+                }
+                console.log(`I'm done ♥`);
+            } catch (e) {
+                console.error('Command failed', e);
+                process.exit(1);
+            }
+        })
+    .argv;

package/src/cli/collect.ts

@@ -0,0 +1,196 @@
+import fs from 'fs-extra';
+import path from 'path';
+import giparser from 'gitignore-parser';
+import type { CollectResult, Translation } from '../core/load';
+import type { IFormatAdapter } from './interfaces';
+import { autoGenerateId } from '../core/utils';
+import util from 'util';
+export interface Loc {
+    id: string;
+    file: string | undefined;
+    line: number;
+    source: string;
+    plural?: boolean;
+}
+
+export async function collect(source: string, adapter: IFormatAdapter, forceLocale?: string): Promise<{ collected: Translation; files: number }> {
+    source = source
+        ? path.join(process.cwd(), source)
+        : process.cwd();
+    // parse gitignore
+    // todo: parse .gitignore files in nested directories.
+    let ignore: { accepts(path: string): boolean; denies(path: string): boolean; };
+    const gitIgnorePath = path.join(source, '.gitignore');
+    if (await fs.pathExists(gitIgnorePath)) {
+        const content = await fs.readFile(gitIgnorePath, 'utf8');
+        ignore = giparser.compile(content);
+    } else {
+        ignore = giparser.compile('');
+    }
+
+
+    async function* walk(dirRelative: string): AsyncIterableIterator<string> {
+        try {
+            const files = await fs.readdir(path.join(source, dirRelative));
+            for (const file of files) {
+                const filepath = path.join(source, dirRelative, file);
+                const fileRelative = path.join(dirRelative, file);
+                if (ignore.denies(fileRelative)) {
+                    continue;
+                }
+                const stats = await fs.stat(filepath);
+                if (stats.isDirectory()) {
+                    yield* walk(fileRelative);
+                } else {
+                    if (ignore.accepts(fileRelative)) {
+                        yield fileRelative;
+                    }
+                }
+            }
+        } catch (e) {
+            console.error('Failed to read directory ' + dirRelative);
+        }
+    }
+
+    const ids = new Map<string, Loc>();
+    const all: Loc[] = [];
+    let defaultLocale: { val: string; file: string; } | undefined = undefined;
+    let fcount = 0;
+    for await (const f of walk('')) {
+        const ext = f.toLowerCase();
+        if (!ext.endsWith('.js') && !ext.endsWith('.ts') && !ext.endsWith('.jsx') && !ext.endsWith('.tsx')) {
+            continue;
+        }
+        fcount++;
+        try {
+            const content = await fs.readFile(path.join(source, f), 'utf8');
+
+            // === try to find setDefaultLocale()
+            if (!forceLocale) {
+                const defLoc = /\bsetDefaultLocale\s*\(\s*('|")([a-zA-Z\-]+)('|")\s*\)/.exec(content);
+                const newDefLoc = defLoc?.[2];
+                if (newDefLoc) {
+                    if (defaultLocale && defaultLocale.val !== newDefLoc) {
+                        console.error(`Ambiguous default locale. Found '${newDefLoc}' in ${f}, but was defined already defined as '${defaultLocale.val}' in ${defaultLocale.file}. Please use --locale:${defaultLocale.val}`)
+                    }
+                    defaultLocale = {
+                        file: f,
+                        val: newDefLoc,
+                    };
+                }
+            }
+
+            collectFromSource(content, ids, all, f);
+
+        } catch (e) {
+            console.error('Failed to read file ' + f + ': ' + util.inspect(e));
+        }
+    }
+
+    // === load default locale ===
+    const targetLanguage = defaultLocale?.val || forceLocale;
+    const origFile = targetLanguage ? await adapter.loadLocale(targetLanguage) : null;
+
+    // === WRITE RESULT ===
+    const result: CollectResult = {};
+    for (const v of all) {
+        if (v.plural) {
+
+            // Pluralizable strings must also be "transalted" in source language,
+            //  since the code will only contain the pluralized form
+            // => they must be marked as dirty if need be.
+            const orig = origFile?.resources?.[v.id];
+            let singular: string | null = null;
+            let dirty: boolean | undefined = undefined;
+            if (orig?.target && typeof orig.target !== 'string') {
+                // if the plural has changed, or if there is no singlar in orig file,
+                //  then it must be fixed
+                if (!orig.target.singular || orig.target.plural !== v.source) {
+                    dirty = true;
+                }
+                singular = orig.target?.singular ?? null;
+            }
+
+            result[v.id] = {
+                target: {
+                    singular,
+                    plural: v.source,
+                },
+                dirty,
+            };
+        } else {
+            // simple translations are never dirty: code is law
+            result[v.id] = {
+                target: v.source,
+            };
+        }
+    }
+
+    if (!targetLanguage) {
+        console.error('Cannot infer the default locale. Either call setDefaultLocale() in analyzed sources, or set the --defaultLocale argument');
+        process.exit(1);
+    }
+    const collected = {
+        resources: result,
+        targetLanguage,
+    };
+    return { collected, files: fcount };
+}
+
+
+export function collectFromSource(content: string, ids: Map<string, Loc>, all: Loc[], f?: string) {
+
+
+    // === parse loc() calls
+    const re = /\bloc(\.plural\([^\)]+\))?\s*(\(\s*('|")([^'"\n]+)('|")\s*(?:,\s*)?\))?\s*`([^`]*)`/g;
+    let m: RegExpExecArray | null;
+    while (m = re.exec(content)) {
+        const plural = m[1];
+        let id = m[4];
+        const val = m[6];
+
+        // parse format
+        const reFormat = /\$\{([^\}]+)\}/g;
+        let fm: RegExpExecArray | null;
+        let index = 0;
+        let cnt = 0;
+        const parts: string[] = [];
+        const idParts: string[] | null = id ? null : [];
+        while (fm = reFormat.exec(val!)) {
+            const literal = val!.substr(index, fm.index - index);
+            parts.push(literal);
+            idParts?.push(literal);
+            index = fm.index + fm[0].length;
+            parts.push('{', cnt.toString(), '}');
+            cnt++;
+        }
+        // add trailing
+        idParts?.push(val!.substr(index));
+        parts.push(val!.substr(index));
+
+
+        const raw = parts.join('');
+        id = id || autoGenerateId(idParts!);
+        const exists = ids.get(id);
+        // check no dupplicate
+        if (exists && exists.source !== raw) {
+            if (f === exists.file) {
+                console.error(`Duplicate translation id '${id}' found in ${f} but was already defined in ${exists.file}`);
+            } else {
+                console.error(`Duplicate translation id '${id}' found in ${f}`);
+            }
+            continue;
+        }
+
+
+        const found: Loc = {
+            file: f,
+            id,
+            line: 0, // todo
+            source: raw,
+            plural: plural ? true : undefined,
+        };
+        all.push(found);
+        ids.set(id, found);
+    }
+}

package/src/cli/interfaces.ts

@@ -0,0 +1,7 @@
+import type { Translation } from '../core/load';
+export interface IFormatAdapter {
+    write(data: Translation): Promise<void>;
+    loadLocales(locales: string[]): Promise<Translation[]>;
+    loadFile(path: string): Promise<Translation>;
+    loadLocale(locale: string): Promise<Translation>;
+}

package/src/cli/reconciliate.ts

@@ -0,0 +1,119 @@
+import type { Translation, CollectItem } from '../core/load';
+import levenshtein from 'js-levenshtein';
+import diacritics from 'diacritics';
+import { deepEqual } from './utils';
+import type { TranslationTarget } from '../core/interfaces';
+
+const cache: Record<string, string> = {};
+function asKey(t: TranslationTarget) {
+    const str = typeof t === 'string' ? t : t.plural;
+    if (cache[str]) {
+        return cache[str];
+    }
+    return cache[str] = diacritics.remove(str.toLowerCase().trim());
+}
+
+export function reconciliate(translations: Translation, source: Translation) {
+    // source is the new one
+    translations.sourceLanguage = source.targetLanguage;
+
+    // stores which translations do not exist in code anymore
+    const translatedLost: { [original: string]: CollectItem; } = {};
+
+    function updateWithOrig(trans: CollectItem, origId: string, src: CollectItem) {
+        if (trans.source && !deepEqual(trans.source, src.target)) {
+            trans.dirty = true;
+        }
+        trans.source = src.target ?? undefined;
+        translations.resources[origId] = trans;
+        // remove it from 'def'
+        delete source.resources[origId];
+    }
+
+    // update source lines
+    for (const [id, trans] of Object.entries(translations.resources)) {
+        const d = source.resources[id];
+        if (d) {
+            // already exists => update
+            updateWithOrig(trans, id, d);
+        } else {
+            // does not exist anymore => remove, but keep in mind.
+            delete translations.resources[id];
+            if (trans.source) {
+                translatedLost[asKey(trans.source)] = trans;
+            }
+        }
+    }
+
+    // for each NEW translations missing
+    // try to reconciliate with those which have changed id, but still same translation.
+    for (const [id, value] of Object.entries(source.resources)) {
+        if (!value.target) {
+            continue;
+        }
+        const cleanTarget = asKey(value.target);
+        if (translatedLost[cleanTarget]) {
+            updateWithOrig(translatedLost[cleanTarget], id, value);
+        }
+    }
+
+    // try to reconciliate with strings that look alike
+    // nb: avoid doing that when too much deleted translations: Levenshtein distance computation is expensive
+    const lostKeys = Object.keys(translatedLost);
+    if (Object.keys(source.resources).length * lostKeys.length < 5000 && lostKeys.length > 0) {
+        const reconc: { origId: string; orig: CollectItem; translatedLostSource: string; distance: number }[] = [];
+        for (const lk of lostKeys) {
+            reconc.push(...Object.entries(source.resources)
+                .map(([id, orig]) => {
+                    // distance between the lost source & the new source
+                    const target = orig.target ? asKey(orig.target) : null;
+                    const distance = target ? levenshtein(lk, target) : Infinity;
+                    return {
+                        origId: id,
+                        orig,
+                        translatedLostSource: lk,
+                        distance: distance,
+                    }
+                })
+                // forbids to much change
+                .filter(x => x.distance < lk.length / 1.5)
+            );
+        }
+
+        reconc.sort((a, b) => a.distance - b.distance);
+        const done = new Set();
+        for (const d of reconc) {
+            if (done.has(d.origId)) {
+                continue;
+            }
+            done.add(d.origId);
+            updateWithOrig(translatedLost[d.translatedLostSource], d.origId, d.orig);
+        }
+    }
+
+    // finally, add new translations...
+    for (const [id, value] of Object.entries(source.resources)) {
+        translations.resources[id] = {
+            source: value.target ?? undefined,
+        };
+    }
+
+    // try to reconciliate
+
+    // count things
+    let missing = 0;
+    const same: string[] = [];
+    let dirty = 0;
+    for (const [k, c] of Object.entries(translations.resources)) {
+        if (!c.target) {
+            missing++;
+            c.target = null; // so it is serialized as json
+        } else if (c.target === c.source) {
+            same.push(k);
+        }
+        if (c.dirty) {
+            dirty++;
+        }
+    }
+    return { missing, same, dirty };
+}

package/src/cli/utils.ts

@@ -0,0 +1,36 @@
+import fastDeepEqual from 'fast-deep-equal';
+import type { CollectResult, GroupedTranslation, Translation } from '../core/load';
+
+export function groupByNamespace(trans: CollectResult): {[namespace: string]: CollectResult} {
+    const result: {[namespace: string]: CollectResult} = {};
+    for (let [id, v] of Object.entries(trans)) {
+        let namespace = '$default';
+        const i = id.indexOf('.');
+        if (i > 0 && i < id.length - 2) {
+            namespace = id.substr(0, i);
+            id = id.substr(i + 1);
+        }
+        if (!result[namespace]) {
+            result[namespace] = {};
+        }
+        result[namespace][id] = v;
+    }
+    return result;
+}
+
+export function toGroupedTranslation(translation: Translation): GroupedTranslation {
+    return {
+        resources: groupByNamespace(translation.resources),
+        sourceLanguage: translation.sourceLanguage,
+        targetLanguage: translation.targetLanguage,
+    };
+}
+
+
+export function deepEqual<T>(obj1: T, obj2: T): boolean {
+    // if (typeof Bun !== 'undefined') {
+    //     const inst = Bun;
+    //     return inst.deepEquals(obj1, obj2);
+    // }
+    return fastDeepEqual(obj1, obj2);
+}