gettext-translator 0.1.1__py3-none-any.whl

gettext_translator/__init__.py

@@ -0,0 +1,17 @@
+"""
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""

gettext_translator/cache.py

@@ -0,0 +1,65 @@
+"""
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+# -----------------------------------------------------------------------------
+
+import json
+import sqlite3
+
+# -----------------------------------------------------------------------------
+
+
+class TranslationCache:
+    """
+    Persistent cache of validated translations, stored in SQLite.
+
+    scope identifies what produced the translation (backend, model, languages),
+    so changing any of them doesn't reuse old results.
+    A key is (ctx, msgid, msgid_plural, nplurals), a value is a str or a list of plural forms.
+    """
+
+    def __init__(self, path: str, scope: tuple) -> None:
+        self.scope = json.dumps(scope)
+        self.db = sqlite3.connect(path)
+        self.db.execute(
+            "CREATE TABLE IF NOT EXISTS translations ("
+            " scope TEXT, ctx TEXT, msgid TEXT, plural TEXT, nplurals INTEGER, value TEXT,"
+            " PRIMARY KEY (scope, ctx, msgid, plural, nplurals))"
+        )
+    # __init__
+
+    # -------------------------------------------------------------------------
+
+    def get(self, key: tuple):
+        row = self.db.execute(
+            "SELECT value FROM translations WHERE scope=? AND ctx=? AND msgid=? AND plural=? AND nplurals=?",
+            (self.scope, *key),
+        ).fetchone()
+        return json.loads(row[0]) if row else None
+    # get
+
+    # -------------------------------------------------------------------------
+
+    def put(self, key: tuple, value) -> None:
+        with self.db:
+            self.db.execute(
+                "INSERT OR REPLACE INTO translations VALUES (?, ?, ?, ?, ?, ?)",
+                (self.scope, *key, json.dumps(value, ensure_ascii=False)),
+            )
+    # put
+# TranslationCache

gettext_translator/gettext_translator.py

@@ -0,0 +1,308 @@
+"""
+Gettext Translator
+
+This Python script provides a tool for translating gettext .po files
+using cloud services (OpenAI, Gemini, Azure Translator) or local AI models.
+
+---
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+# -------------------------------------------------------------------------
+
+import logging
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+import polib
+from langcodes import Language, LanguageTagError
+from pydantic import ValidationError
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from .settings import Settings
+from .translator_factory import TranslatorFactory
+
+# -------------------------------------------------------------------------
+
+AI_COMMENT = "AI-translated"
+
+# printf-style (%s, %1$s, %(name)s, %.2f, %%), brace-style ({name}) and HTML tags (<b, </b)
+PLACEHOLDER_RE = re.compile(r"%(?:\d+\$|\([^)]*\))?[-+#0]*\d*(?:\.\d+)?[sdifouxXeEgGcr%]|\{[^{}]*\}|</?[a-zA-Z][\w-]*")
+
+# -------------------------------------------------------------------------
+
+
+def placeholders(text: str) -> Counter:
+    return Counter(PLACEHOLDER_RE.findall(text))
+# placeholders
+
+# -------------------------------------------------------------------------
+
+
+def nplurals(po_file) -> int:
+    match = re.search(r"nplurals\s*=\s*(\d+)", po_file.metadata.get("Plural-Forms", ""))
+    return int(match.group(1)) if match else 2
+# nplurals
+
+# -------------------------------------------------------------------------
+
+
+def validate(key, result):
+    """Returns the translation in result if it can be used for key, None otherwise."""
+    _, msgid, plural, n = key
+
+    if plural:
+        forms = result.get("msgstr_plural")
+        if not (isinstance(forms, list) and len(forms) == n and all(isinstance(f, str) and f for f in forms)):
+            return None
+        # Some forms may drop placeholders ("One file"), but the last and most general one must keep them
+        allowed = placeholders(msgid) | placeholders(plural)
+        if all(placeholders(f) <= allowed for f in forms) and placeholders(forms[-1]) == placeholders(plural):
+            return forms
+        return None
+
+    msgstr = result.get("msgstr")
+    if isinstance(msgstr, str) and msgstr and placeholders(msgstr) == placeholders(msgid):
+        return msgstr
+    return None
+# validate
+
+# -------------------------------------------------------------------------
+
+
+class GettextTranslator:
+    def __init__(self, service, cache=None) -> None:
+        self.service = service
+        self.config = service.config
+        self.cache = cache
+    # __init__
+
+    # -------------------------------------------------------------------------
+
+    def po_files(self):
+        """Returns (file, root) pairs. root is the directory the --output path is relative to."""
+        files = []
+        for path in map(Path, self.config.po):
+            if path.is_dir():
+                files += [(f, path) for f in sorted(path.rglob("*.po"))]
+            else:
+                files.append((path, path.parent))
+        return files
+    # po_files
+
+    # -------------------------------------------------------------------------
+
+    def output_path(self, path, root):
+        if not self.config.output:
+            return path
+        output = Path(self.config.output)
+        return output if output.suffix == ".po" else output / path.relative_to(root)
+    # output_path
+
+    # -------------------------------------------------------------------------
+
+    def load(self, path):
+        """Loads a .po file, or returns None if its language is not the destination language."""
+        po_file = polib.pofile(str(path))
+        file_lang = po_file.metadata.get("Language", "")
+
+        try:
+            language = Language.get(file_lang).language
+        except LanguageTagError:
+            language = None
+
+        if language != self.config.dst.language:
+            logging.warning("[💣] Skipping %s: its language %r is not %s", path, file_lang, self.config.dst.language)
+            return None
+
+        if self.config.fuzzy:
+            for entry in po_file:
+                entry.fuzzy = False
+            logging.info("[✔️] Fuzzy translations disabled in file: %s", path)
+
+        return po_file
+    # load
+
+    # -------------------------------------------------------------------------
+
+    @staticmethod
+    def item(n, key):
+        ctx, msgid, plural, count = key
+        item = {"n": n, "id": msgid}
+        if ctx:
+            item["ctx"] = ctx
+        if plural:
+            item["plural"] = plural
+            item["nplurals"] = count
+        return item
+    # item
+
+    # -------------------------------------------------------------------------
+
+    def apply(self, entry, value):
+        if isinstance(value, list):
+            entry.msgstr_plural = dict(enumerate(value))
+        else:
+            entry.msgstr = value
+
+        if self.config.ascribe and AI_COMMENT not in entry.comment:
+            entry.comment = f"{entry.comment}\n{AI_COMMENT}".strip()
+    # apply
+
+    # -------------------------------------------------------------------------
+
+    def request(self, pending):
+        """Sends the pending keys to the backend and returns the valid translations."""
+        self.service.configure()
+        results = self.service.translate([self.item(n, key) for n, key in enumerate(pending)])
+
+        translations = {}
+        for result in results:
+            n = result.get("n")
+            if not isinstance(n, int) or not 0 <= n < len(pending):
+                logging.warning("[❌] Discarding result with unknown item number: %r", result)
+                continue
+            value = validate(pending[n], result)
+            if value is None:
+                logging.warning("[❌] Discarding invalid translation for %r: %r", pending[n][1], result)
+                continue
+            translations[pending[n]] = value
+        # for
+
+        return translations
+    # request
+
+    # -------------------------------------------------------------------------
+
+    def show(self, translations):
+        table = Table("msgctxt", "msgid", "msgstr", show_lines=True)
+        for (ctx, msgid, plural, _), value in translations.items():
+            source = f"{msgid}\n{plural}" if plural else msgid
+            target = "\n".join(value) if isinstance(value, list) else value
+            table.add_row(Text(ctx), Text(source), Text(target))
+        Console().print(table)
+    # show
+
+    # -------------------------------------------------------------------------
+
+    def translate(self) -> bool:
+        """
+        Translates the .po files. Returns False if there was something to translate but nothing was translated.
+        """
+        files = self.po_files()
+        if self.config.output.endswith(".po") and len(files) != 1:
+            raise ValueError("--output can be a .po file only when translating a single .po file")
+
+        # Group the untranslated entries of all the files by what has to be translated,
+        # so each unique string is translated only once
+        groups = {}
+        loaded = []
+        for path, root in files:
+            po_file = self.load(path)
+            if po_file is None:
+                continue
+
+            count = nplurals(po_file)
+            for entry in po_file.untranslated_entries():
+                key = (entry.msgctxt or self.config.context, entry.msgid, entry.msgid_plural,
+                       count if entry.msgid_plural else 0)
+                groups.setdefault(key, []).append(entry)
+            loaded.append((path, root, po_file))
+        # for
+
+        translations = {}
+        if self.cache:
+            for key in groups:
+                value = self.cache.get(key)
+                if value is not None:
+                    translations[key] = value
+        cached = len(translations)
+
+        pending = [key for key in groups if key not in translations]
+        logging.info("[⚙️] %d strings to translate, %d from cache", len(pending), cached)
+
+        new = self.request(pending) if pending else {}
+        if self.cache:
+            for key, value in new.items():
+                self.cache.put(key, value)
+        translations.update(new)
+
+        for key, entries in groups.items():
+            if key in translations:
+                for entry in entries:
+                    self.apply(entry, translations[key])
+
+        missing = len(pending) - len(new)
+        if missing:
+            logging.warning("[❌] %d of %d strings were not translated", missing, len(pending))
+
+        if self.config.dry_run:
+            self.show(translations)
+            print(f"{len(new)} translated, {cached} from cache, {missing} not translated. No file was saved.")
+        else:
+            for path, root, po_file in loaded:
+                output = self.output_path(path, root)
+                output.parent.mkdir(parents=True, exist_ok=True)
+                po_file.save(str(output))
+                logging.info("[✅] Saved %s", output)
+
+        return not pending or bool(new)
+    # translate
+# GettextTranslator
+
+# -----------------------------------------------------------------------------
+
+
+def main(argv=None) -> int:
+    try:
+        settings = Settings(**vars(Settings.parse_cli_args(argv)))
+    except ValidationError as e:
+        print(e)
+        return 1
+
+    logging.basicConfig(level=settings.verbose.upper())
+
+    try:
+        backend = TranslatorFactory().create_translator(settings)
+
+        # Shows the info for the given --backend and exit
+        if settings.info:
+            backend.show_info()
+            return 0
+
+        cache = None
+        if settings.cache:
+            # Imported here because some Python builds lack sqlite3, which is only needed for --cache
+            from .cache import TranslationCache
+            scope = (settings.backend, str(getattr(settings, "model", "")), str(settings.src), str(settings.dst))
+            cache = TranslationCache(settings.cache, scope)
+
+        ok = GettextTranslator(backend, cache).translate()
+    except Exception as e:  # pylint: disable=W0718
+        logging.error("[☠️] %s", e, exc_info=settings.verbose == "debug")
+        return 1
+
+    return 0 if ok else 1
+# main
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gettext_translator/settings.py

@@ -0,0 +1,178 @@
+"""
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+# -----------------------------------------------------------------------------
+
+
+import argparse
+import os
+import yaml
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings.sources import (
+    PydanticBaseSettingsSource
+)
+from pydantic import Field, field_validator, model_validator
+from typing import Any, Dict, Self, Tuple
+from langcodes import Language
+from .version import __version__
+
+# -----------------------------------------------------------------------------
+
+
+class YamlConfigSettingsSource(PydanticBaseSettingsSource):
+    def __init__(self, settings_cls, yaml_file: str = None):
+        super().__init__(settings_cls)
+        self.yaml_file = yaml_file
+        self._yaml_data = {}
+
+        if yaml_file:
+            try:
+                with open(yaml_file, 'r', encoding='utf-8') as f:
+                    self._yaml_data = yaml.safe_load(f) or {}
+            except FileNotFoundError:
+                self._yaml_data = {}
+
+        # Values like "env:MY_VAR" are read from the environment variable MY_VAR
+        for key, value in self._yaml_data.items():
+            if isinstance(value, str) and value.startswith("env:"):
+                self._yaml_data[key] = os.getenv(value[len("env:"):])
+
+    def get_field_value(self, field_info, field_name: str) -> Tuple[Any, str, bool]:
+        value = self._yaml_data.get(field_name)
+        if value is None:
+            return None, field_name, False
+        return value, field_name, True
+
+    def prepare_field_value(self, field_name: str, value: Any, value_origin: Any) -> Any:
+        return value
+
+    def __call__(self) -> Dict[str, Any]:
+        return self._yaml_data
+# YamlConfigSettingsSource
+
+# -----------------------------------------------------------------------------
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        extra="allow",
+    )
+
+    # -------------------------------------------------------------------------
+
+    info: bool = Field(default=False)
+    backend: str = Field(default="")
+    po: list[str] = Field(default_factory=list)
+    src: str = Field(default=Language.make(language='en').language)
+    dst: str = Field(default=Language.make(language='en').language)
+    fuzzy: bool = Field(default=False)
+    context: str = Field(default="")
+    verbose: str = Field(default="warning")
+    ascribe: bool = Field(default=False)
+    config: str = Field(default="config.yaml")
+    dry_run: bool = Field(default=False)
+    output: str = Field(default="")
+    cache: str = Field(default="")
+
+    # -------------------------------------------------------------------------
+
+    @field_validator('po')
+    @classmethod
+    def validate_po(cls, v, info):
+        if info.data.get('info'):
+            return v  # Skip validation if info is present
+        if not v:
+            raise ValueError('po must be provided unless --info is used.')
+        return v
+    # validate_po
+
+    # -------------------------------------------------------------------------
+
+    @field_validator("src", "dst")
+    @classmethod
+    def validate_iso639(cls, v, info):
+        if Language.get(v).is_valid():
+            return Language.get(v).maximize()
+        raise ValueError("invalid language code")
+    # validate_iso639
+
+    # -------------------------------------------------------------------------
+
+    @model_validator(mode='after')
+    def validate_info_backend(self) -> Self:
+        if self.info and not self.backend:
+            raise ValueError("please tell me which backend do you want to see information about")
+        return self
+    # validate_info_backend
+
+    # -------------------------------------------------------------------------
+
+    @classmethod
+    def parse_cli_args(cls, argv=None):
+        # Only the arguments given in the command line are returned, so the
+        # YAML file and the field defaults can fill in the rest
+        parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS)
+        parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+        parser.add_argument("--info", action='store_true',
+                            help="Shows information about the backend")
+        parser.add_argument("--backend", type=str,
+                            help="Which backend to use")
+        parser.add_argument("--po", type=str, nargs="+",
+                            help="The .po file(s) or directories containing .po files")
+        parser.add_argument("--src", type=str,
+                            help="The source language. Default: en")
+        parser.add_argument("--dst", type=str,
+                            help="The language to translate to")
+        parser.add_argument("--fuzzy", action='store_true',
+                            help="Remove the fuzzy flag from all entries before translating")
+        parser.add_argument("--context", type=str,
+                            help="Optional default translation context for msgid without msgctxt")
+        parser.add_argument("--verbose", type=str.lower,
+                            choices=["debug", "info", "warning", "error", "critical"],
+                            help="Logging level. Default: warning")
+        parser.add_argument("--ascribe", action='store_true',
+                            help="Include a comment in each entry indicating that it was translated with AI")
+        parser.add_argument("--config", type=str,
+                            help="Path to the .yaml configuration file for the backend. Default: config.yaml")
+        parser.add_argument("--dry-run", action='store_true',
+                            help="Show the translations without saving any file")
+        parser.add_argument("--output", type=str,
+                            help="Save to this .po file (single input) or directory, instead of overwriting the input")
+        parser.add_argument("--cache", type=str,
+                            help="Path to a SQLite file used to cache translations between runs")
+        return parser.parse_args(argv)
+    # parse_cli_args
+
+    # -------------------------------------------------------------------------
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls,
+        init_settings: PydanticBaseSettingsSource,
+        **kwargs
+    ):
+        config_file = init_settings.init_kwargs.get('config', cls.model_fields['config'].default)
+        yaml_source = YamlConfigSettingsSource(settings_cls, config_file)
+
+        return (
+            init_settings,   # CLI args
+            yaml_source,     # YAML
+        )
+    # settings_customise_sources
+# Settings

gettext_translator/translator_factory.py

@@ -0,0 +1,36 @@
+"""
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from pathlib import Path
+
+from gettext_translator_service import load_plugins
+
+
+class TranslatorFactory:
+    @staticmethod
+    def create_translator(settings):
+        plugins = load_plugins()
+        if settings.backend in plugins:
+            impl_cls = plugins[settings.backend]
+            if impl_cls.REQUIRES_CONFIG and not settings.info and not Path(settings.config).is_file():
+                raise ValueError(f"❌ Configuration file {settings.config} not found, use --config")
+            return impl_cls(settings)
+        else:
+            installed = ', '.join(plugins) or 'none'
+            raise ValueError(f"❌ Unknown translation backend {settings.backend!r}. Installed: {installed}")
+# TranslatorFactory

gettext_translator/version.py

@@ -0,0 +1,19 @@
+"""
+    AI-based Gettext automatic translator.
+    Copyright (C) 2026 Rodolfo González González.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+__version__ = "0.1.1"