lokit-python 0.1.0__cp313-cp313-win_amd64.whl

lokit/parsers/json_i18n/extraction.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, AsyncIterator, Iterator, Optional
+
+from lokit.data.structure import Data, Meta, TranslationStatus
+
+ExtractItem = tuple[str, Data]
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class _AsyncJsonI18nExtraction:
+    def __init__(self, extractor: JsonI18nExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> _AsyncJsonI18nExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class JsonI18nExtractor:
+    def __init__(
+        self,
+        filepath: str,
+        source_locale: str = "",
+        target_locale: str | None = None,
+        target_filepath: str | None = None,
+    ) -> None:
+        self.filepath = filepath
+        self.source_locale = source_locale
+        self.target_locale = target_locale
+        self.target_filepath = target_filepath
+        self.source_language: str | None = None
+        self.target_language: str | None = None
+        self.export_origin = ""
+        self.extensions: dict[str, str] = {"input_format": "json_i18n"}
+
+    def extract(self) -> Iterator[ExtractItem]:
+        source_data = self._load_json(self.filepath)
+        source_flat = self._flatten(source_data)
+
+        target_flat: dict[str, str] = {}
+        if self.target_filepath is not None:
+            target_data = self._load_json(self.target_filepath)
+            target_flat = self._flatten(target_data)
+
+        self._infer_locale()
+
+        for key, source_value in source_flat.items():
+            target_value = target_flat.get(key)
+            status = (
+                TranslationStatus.TRANSLATED
+                if target_value
+                else TranslationStatus.NEW
+            )
+            data = Data(
+                source=source_value,
+                target=target_value,
+                meta=Meta(),
+                status=status,
+                extensions={"input_format": "json_i18n"},
+            )
+            yield key, data
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return _AsyncJsonI18nExtraction(self)
+
+    def _load_json(self, filepath: str) -> dict[str, Any]:
+        with Path(filepath).open("r", encoding="utf-8") as f:
+            result: dict[str, Any] = json.load(f)
+        return result
+
+    def _flatten(
+        self, obj: dict[str, Any], prefix: str = ""
+    ) -> dict[str, str]:
+        flat: dict[str, str] = {}
+        for key, value in obj.items():
+            full_key = f"{prefix}.{key}" if prefix else key
+            if isinstance(value, dict):
+                flat.update(self._flatten(value, full_key))
+            elif isinstance(value, str):
+                flat[full_key] = value
+        return flat
+
+    def _infer_locale(self) -> None:
+        if not self.source_locale:
+            inferred = self._locale_from_filename(self.filepath)
+            if inferred:
+                self.source_locale = inferred
+        if self.source_locale:
+            self.source_language = self._base_language(self.source_locale)
+        if not self.target_locale and self.target_filepath:
+            inferred = self._locale_from_filename(self.target_filepath)
+            if inferred:
+                self.target_locale = inferred
+        if self.target_locale:
+            self.target_language = self._base_language(self.target_locale)
+
+    def _locale_from_filename(self, filepath: str) -> str | None:
+        path = Path(filepath)
+        if path.suffix.lower() != ".json":
+            return None
+        stem = path.stem
+        if not stem or not all(c.isalnum() or c in "_-" for c in stem):
+            return None
+        if len(stem) < 2 or not stem[:2].isalpha():
+            return None
+        return stem
+
+    def _base_language(self, locale: str) -> str:
+        return locale.replace("_", "-").split("-")[0].lower()

lokit/parsers/po/__init__.py

@@ -0,0 +1,3 @@
+from lokit.parsers.po.extraction import PoExtractor
+
+__all__ = ["PoExtractor"]

lokit/parsers/po/extraction.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Any, AsyncIterator, Iterator, Optional
+
+import polib
+
+from lokit.data.structure import (
+    Comment,
+    Data,
+    Meta,
+    Plural,
+    PluralCategory,
+    TranslationStatus,
+)
+
+ExtractItem = tuple[str, Data]
+
+_PLURAL_CATEGORIES: tuple[PluralCategory, ...] = (
+    PluralCategory.ONE,
+    PluralCategory.TWO,
+    PluralCategory.FEW,
+    PluralCategory.MANY,
+    PluralCategory.OTHER,
+)
+
+
+def _category_from_index(index: int) -> PluralCategory:
+    if index < len(_PLURAL_CATEGORIES):
+        return _PLURAL_CATEGORIES[index]
+    return PluralCategory.OTHER
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class _AsyncPoExtraction:
+    def __init__(self, extractor: PoExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> _AsyncPoExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class PoExtractor:
+    def __init__(
+        self,
+        filepath: str,
+        source_locale: str = "",
+        target_locale: str | None = None,
+    ) -> None:
+        self.filepath = filepath
+        self.source_locale = source_locale
+        self.target_locale = target_locale
+        self.source_language: str | None = None
+        self.target_language: str | None = None
+        self.export_origin = ""
+        self.extensions: dict[str, str] = {"input_format": "po"}
+
+    def extract(self) -> Iterator[ExtractItem]:
+        po: Any = polib.pofile(self.filepath)
+        self._read_metadata(po)
+
+        for entry in po:
+            if entry.obsolete:
+                continue
+
+            if entry.msgid_plural:
+                yield from self._extract_plural(entry)
+            else:
+                yield self._extract_singular(entry)
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return _AsyncPoExtraction(self)
+
+    def _read_metadata(self, po: Any) -> None:
+        metadata: dict[str, str] = po.metadata or {}
+        lang = metadata.get("Language", "")
+        if lang and not self.target_locale:
+            self.target_locale = lang
+        if self.target_locale:
+            self.target_language = self._base_language(self.target_locale)
+        if self.source_locale:
+            self.source_language = self._base_language(self.source_locale)
+        self.export_origin = metadata.get("X-Generator", "")
+
+    def _extract_singular(self, entry: Any) -> ExtractItem:
+        unit_id = self._unit_id(entry)
+        target = entry.msgstr if entry.msgstr else None
+        status = self._status(entry)
+        comments = self._comments(entry)
+        extensions = self._extensions(entry)
+        data = Data(
+            source=entry.msgid,
+            target=target,
+            meta=Meta(),
+            status=status,
+            comments=comments,
+            extensions=extensions,
+        )
+        return unit_id, data
+
+    def _extract_plural(self, entry: Any) -> Iterator[ExtractItem]:
+        unit_id = self._unit_id(entry)
+        plural_dict: dict[int, str] = entry.msgstr_plural or {}
+        base_target = plural_dict.get(0) or None
+        status = self._status(entry)
+        comments = self._comments(entry)
+        extensions = self._extensions(entry)
+        data = Data(
+            source=entry.msgid,
+            target=base_target,
+            plural=Plural(variant=entry.msgid_plural),
+            meta=Meta(),
+            status=status,
+            comments=comments,
+            extensions=extensions,
+        )
+        yield unit_id, data
+
+        for n in sorted(plural_dict):
+            if n == 0:
+                continue
+            plural_target = plural_dict[n] if plural_dict[n] else None
+            plural_data = Data(
+                source=entry.msgid,
+                target=plural_target,
+                plural=Plural(
+                    variant=entry.msgid_plural,
+                    category=_category_from_index(n),
+                ),
+                meta=Meta(),
+                status=self._plural_form_status(plural_target, entry),
+                comments=[],
+                extensions=extensions.copy(),
+            )
+            yield f"{unit_id}[{n}]", plural_data
+
+    def _unit_id(self, entry: Any) -> str:
+        if entry.msgctxt:
+            return f"{entry.msgctxt}\x04{entry.msgid}"
+        return str(entry.msgid)
+
+    def _status(self, entry: Any) -> TranslationStatus:
+        if "fuzzy" in entry.flags:
+            return TranslationStatus.DRAFT
+        target = entry.msgstr if not entry.msgid_plural else (entry.msgstr_plural or {}).get(0, "")
+        if target:
+            return TranslationStatus.TRANSLATED
+        return TranslationStatus.NEW
+
+    def _plural_form_status(
+        self, target: str | None, entry: Any
+    ) -> TranslationStatus:
+        if "fuzzy" in entry.flags:
+            return TranslationStatus.DRAFT
+        if target:
+            return TranslationStatus.TRANSLATED
+        return TranslationStatus.NEW
+
+    def _comments(self, entry: Any) -> list[Comment]:
+        comments: list[Comment] = []
+        if entry.comment:
+            comments.append(
+                Comment(
+                    context=entry.comment,
+                    context_key=entry.msgctxt or None,
+                )
+            )
+        if entry.tcomment:
+            comments.append(Comment(context=entry.tcomment))
+        return comments
+
+    def _extensions(self, entry: Any) -> dict[str, str]:
+        extensions: dict[str, str] = {}
+        if entry.occurrences:
+            refs = ", ".join(
+                f"{path}:{line}" for path, line in entry.occurrences
+            )
+            extensions["references"] = refs
+        non_fuzzy = [f for f in entry.flags if f != "fuzzy"]
+        if non_fuzzy:
+            extensions["flags"] = ", ".join(non_fuzzy)
+        return extensions
+
+    def _base_language(self, locale: str) -> str:
+        return locale.replace("_", "-").split("-")[0].lower()

lokit/parsers/tmx/base.py

@@ -0,0 +1,145 @@
+from typing import Optional
+
+from lxml import etree
+
+from lokit.core.logger import logger
+from lokit.parsers.tmx.header import TmxHeaderParser
+from lokit.parsers.tmx.models import HeaderData
+from lokit.parsers.tmx.xml_utils import clear_element, element_children, iterparse_safe, local_name
+
+
+class TmxParser:
+    def __init__(
+        self,
+        tmx_file_path: str,
+        source_language: Optional[str] = None,
+        target_language: Optional[str] = None,
+        domain: Optional[str] = None,
+    ) -> None:
+        self.filepath: str = tmx_file_path
+        self.domain: str = domain or ""
+
+        self.native_source: str = source_language or ""
+        self.native_target: str = target_language or ""
+
+        self.source_language: Optional[str] = None
+        self.source_locale: Optional[str] = None
+        self.target_language: Optional[str] = None
+        self.target_locale: Optional[str] = None
+
+        self.export_origin: str = ""
+        self.export_timestamp: str = ""
+        self.extensions: dict[str, str] = {}
+
+        self.header_parser: TmxHeaderParser = TmxHeaderParser()
+
+        self._initialize_from_file()
+        self._validate_and_set_languages()
+
+    def _initialize_from_file(self) -> None:
+        context = iterparse_safe(self.filepath, events=("end",))
+
+        for _, elem in context:
+            elem_name = local_name(elem.tag)
+            if elem_name == "header":
+                header_data: HeaderData = self.header_parser.parse(elem)
+                self.export_origin = header_data.origin
+                self.export_timestamp = header_data.timestamp
+                self.extensions.update(header_data.extensions)
+
+                if self.native_source:
+                    if header_data.srclang and not self._compare_base_lang(
+                        self.native_source, header_data.srclang
+                    ):
+                        logger.warning(
+                            f"Provided source '{self.native_source}' mismatches header '{header_data.srclang}'"
+                        )
+                else:
+                    self.native_source = header_data.srclang
+
+                if self.native_target:
+                    if header_data.tgtlang and not self._compare_base_lang(
+                        self.native_target, header_data.tgtlang
+                    ):
+                        logger.warning(
+                            f"Provided target '{self.native_target}' mismatches header '{header_data.tgtlang}'"
+                        )
+                else:
+                    self.native_target = header_data.tgtlang
+
+                clear_element(elem)
+                if self.native_source and self.native_target:
+                    break
+
+            elif elem_name == "tu" and not (self.native_source and self.native_target):
+                self._initialize_missing_languages_from_tu(elem)
+                clear_element(elem)
+                if self.native_source and self.native_target:
+                    break
+
+    def _compare_base_lang(self, lang1: str, lang2: str) -> bool:
+        if not lang1 or not lang2:
+            return False
+        l1 = lang1.replace("_", "-").split("-")[0].lower()
+        l2 = lang2.replace("_", "-").split("-")[0].lower()
+        return l1 == l2
+
+    def _initialize_missing_languages_from_tu(self, element: etree._Element) -> None:
+        langs: list[str] = []
+        for tuv in element_children(element, "tuv"):
+            lang = self._get_xml_lang(tuv)
+            if lang:
+                langs.append(lang)
+
+        if not self.native_source and langs:
+            self.native_source = langs[0]
+
+        if not self.native_target:
+            self.native_target = next(
+                (
+                    lang
+                    for lang in langs
+                    if not self._compare_base_lang(lang, self.native_source)
+                ),
+                "",
+            )
+
+    def _validate_and_set_languages(self) -> None:
+        if self.native_source:
+            self.source_language, self.source_locale = self._parse_locale_string(
+                self.native_source
+            )
+        if self.native_target:
+            self.target_language, self.target_locale = self._parse_locale_string(
+                self.native_target
+            )
+
+    def _parse_locale_string(self, lang_string: str) -> tuple[str, str]:
+        if not lang_string:
+            raise ValueError("Cannot parse empty language string")
+
+        normalized: str = lang_string.replace("_", "-")
+        parts: list[str] = normalized.split("-")
+
+        lang_code: str = parts[0].lower()
+        canonical_parts = [lang_code]
+        if len(parts) > 1:
+            canonical_parts.extend(
+                self._canonicalize_subtag(part) for part in parts[1:]
+            )
+
+        return lang_code, "-".join(canonical_parts)
+
+    def _canonicalize_subtag(self, subtag: str) -> str:
+        if len(subtag) == 2 and subtag.isalpha():
+            return subtag.upper()
+        if len(subtag) == 4 and subtag.isalpha():
+            return subtag.title()
+        return subtag
+
+    def _get_xml_lang(self, element: etree._Element) -> str:
+        return (
+            element.get("{http://www.w3.org/XML/1998/namespace}lang")
+            or element.get("lang")
+            or ""
+        )