lokit-python 0.1.1__tar.gz → 0.1.2__tar.gz

{lokit_python-0.1.1 → lokit_python-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lokit-python
-Version: 0.1.1
+Version: 0.1.2
 Summary: A type-safe localization toolkit for parsing, converting, and matching TMX, XLIFF, PO, JSON, HTML, CSV, XLSX, and IDML files.
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{lokit_python-0.1.1 → lokit_python-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lokit-python"
-version = "0.1.1"
+version = "0.1.2"
 description = "A type-safe localization toolkit for parsing, converting, and matching TMX, XLIFF, PO, JSON, HTML, CSV, XLSX, and IDML files."
 readme = "README.md"
 requires-python = ">=3.12"

{lokit_python-0.1.1 → lokit_python-0.1.2}/src/lokit/__init__.py

@@ -51,7 +51,11 @@ from lokit.importers import (
     import_po_async,
     import_tmx,
     import_tmx_async,
+    import_tmx_batches_async,
+    import_tmx_parallel,
+    process_tmx_async,
     stream_tmx,
+    stream_tmx_parallel,
     convert_tmx_to_csv,
     convert_tmx_to_tmx,
     convert_tmx_to_xliff,
@@ -69,6 +73,8 @@ from lokit.parsers.po.extraction import PoExtractor
 from lokit.parsers.json_i18n.extraction import JsonI18nExtractor
 from lokit.parsers.idml.extraction import IdmlExtractor
 from lokit.parsers.tmx.extraction import TmxExtractor
+from lokit.parsers.tmx.models import TmxParseMode
+from lokit.parsers.tmx.parallel import TmxParallelOptions
 from lokit.parsers.xliff.extraction import XliffExtractor
 
 __all__ = [
@@ -91,6 +97,8 @@ __all__ = [
     "TieData",
     "TieType",
     "TmxExtractor",
+    "TmxParseMode",
+    "TmxParallelOptions",
     "TranslationStatus",
     "XliffExtractor",
     "CsvExtractor",
@@ -131,7 +139,11 @@ __all__ = [
     "import_po_async",
     "import_tmx",
     "import_tmx_async",
+    "import_tmx_batches_async",
+    "import_tmx_parallel",
+    "process_tmx_async",
     "stream_tmx",
+    "stream_tmx_parallel",
     "convert_tmx_to_csv",
     "convert_tmx_to_tmx",
     "convert_tmx_to_xliff",

{lokit_python-0.1.1 → lokit_python-0.1.2}/src/lokit/importers.py

@@ -1,9 +1,8 @@
 from __future__ import annotations
 
-from collections.abc import AsyncIterator, Callable, Iterable, Iterator
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Iterator
 from pathlib import Path
 from time import perf_counter
-from typing import Any
 
 from lokit.data.structure import BaseStructure, Data, StreamingStructure, ConversionStats
 from lokit.format_detection import LokitInputFormat, detect_format
@@ -15,15 +14,21 @@ from lokit.parsers.html.extraction import HtmlExtractor
 from lokit.parsers.po.extraction import PoExtractor
 from lokit.parsers.json_i18n.extraction import JsonI18nExtractor
 from lokit.parsers.idml.extraction import IdmlExtractor
+from lokit.parsers.async_bridge import AsyncExtractionBridge
 from lokit.parsers.tmx.extraction import TmxExtractor
+from lokit.parsers.tmx.models import TmxParseMode
+from lokit.parsers.tmx.parallel import TmxParallelOptions, extract_tmx_parallel
 from lokit.parsers.xliff.extraction import XliffExtractor
 
+TmxBatch = list[tuple[str, Data]]
+
 
 def import_tmx(
     filepath: str,
     source_language: str | None = None,
     target_language: str | None = None,
     domain: str | None = None,
+    mode: TmxParseMode = TmxParseMode.FULL,
 ) -> BaseStructure:
     _validate_xml_root(filepath, "tmx")
     extractor = TmxExtractor(
@@ -32,6 +37,7 @@ def import_tmx(
         target_language=target_language,
         domain=domain,
         parse_header=not (source_language and target_language),
+        mode=mode,
     )
     parsed_data: dict[str, Data] = {
         unit_id: data for unit_id, data in extractor.extract()
@@ -39,11 +45,77 @@ def import_tmx(
     return _build_tmx_structure(extractor, parsed_data)
 
 
+def import_tmx_parallel(
+    filepath: str,
+    source_language: str | None = None,
+    target_language: str | None = None,
+    domain: str | None = None,
+    mode: TmxParseMode = TmxParseMode.FULL,
+    options: TmxParallelOptions | None = None,
+) -> BaseStructure:
+    _validate_xml_root(filepath, "tmx")
+    extractor = TmxExtractor(
+        filepath=filepath,
+        source_language=source_language,
+        target_language=target_language,
+        domain=domain,
+        parse_header=not (source_language and target_language),
+        mode=mode,
+    )
+    parsed_data: dict[str, Data] = {
+        unit_id: data
+        for unit_id, data in extract_tmx_parallel(
+            filepath=filepath,
+            source_language=extractor.native_source,
+            target_language=extractor.native_target,
+            domain=domain,
+            mode=mode,
+            options=options,
+        )
+    }
+    return _build_tmx_structure(extractor, parsed_data)
+
+
+def stream_tmx_parallel(
+    filepath: str,
+    source_language: str | None = None,
+    target_language: str | None = None,
+    domain: str | None = None,
+    mode: TmxParseMode = TmxParseMode.FULL,
+    options: TmxParallelOptions | None = None,
+) -> StreamingStructure:
+    _validate_xml_root(filepath, "tmx")
+    extractor = TmxExtractor(
+        filepath=filepath,
+        source_language=source_language,
+        target_language=target_language,
+        domain=domain,
+        parse_header=not (source_language and target_language),
+        mode=mode,
+    )
+    return StreamingStructure(
+        source_locale=extractor.source_locale or extractor.native_source,
+        target_locale=extractor.target_locale or extractor.native_target or None,
+        items=extract_tmx_parallel(
+            filepath=filepath,
+            source_language=extractor.native_source,
+            target_language=extractor.native_target,
+            domain=domain,
+            mode=mode,
+            options=options,
+        ),
+        source_language=extractor.source_language,
+        target_language=extractor.target_language,
+        extensions=extractor.extensions,
+    )
+
+
 async def import_tmx_async(
     filepath: str,
     source_language: str | None = None,
     target_language: str | None = None,
     domain: str | None = None,
+    mode: TmxParseMode = TmxParseMode.FULL,
 ) -> AsyncIterator[tuple[str, Data]]:
     _validate_xml_root(filepath, "tmx")
     extractor = TmxExtractor(
@@ -52,11 +124,74 @@ async def import_tmx_async(
         target_language=target_language,
         domain=domain,
         parse_header=not (source_language and target_language),
+        mode=mode,
     )
     async for unit_id, data in extractor.extract_async():
         yield unit_id, data
 
 
+async def import_tmx_batches_async(
+    filepath: str,
+    source_language: str | None = None,
+    target_language: str | None = None,
+    domain: str | None = None,
+    *,
+    batch_size: int = 1000,
+    mode: TmxParseMode = TmxParseMode.FULL,
+) -> AsyncIterator[TmxBatch]:
+    _validate_xml_root(filepath, "tmx")
+    extractor = TmxExtractor(
+        filepath=filepath,
+        source_language=source_language,
+        target_language=target_language,
+        domain=domain,
+        parse_header=not (source_language and target_language),
+        mode=mode,
+    )
+    async for batch in AsyncExtractionBridge(
+        lambda: _iter_batches(extractor.extract(), batch_size),
+        batch_size=1,
+    ):
+        yield batch
+
+
+def _iter_batches(
+    items: Iterator[tuple[str, Data]],
+    batch_size: int,
+) -> Iterator[TmxBatch]:
+    if batch_size < 1:
+        raise ValueError("batch_size must be at least 1")
+    batch: TmxBatch = []
+    for item in items:
+        batch.append(item)
+        if len(batch) >= batch_size:
+            yield batch
+            batch = []
+    if batch:
+        yield batch
+
+
+async def process_tmx_async(
+    filepath: str,
+    callback: Callable[[TmxBatch], Awaitable[None]],
+    source_language: str | None = None,
+    target_language: str | None = None,
+    domain: str | None = None,
+    *,
+    batch_size: int = 1000,
+    mode: TmxParseMode = TmxParseMode.FULL,
+) -> None:
+    async for batch in import_tmx_batches_async(
+        filepath,
+        source_language=source_language,
+        target_language=target_language,
+        domain=domain,
+        batch_size=batch_size,
+        mode=mode,
+    ):
+        await callback(batch)
+
+
 def import_xliff(filepath: str) -> BaseStructure:
     _validate_xml_root(filepath, "xliff")
     extractor = XliffExtractor(filepath)
@@ -138,6 +273,7 @@ def stream_tmx(
     filepath: str,
     source_language: str | None = None,
     target_language: str | None = None,
+    mode: TmxParseMode = TmxParseMode.FULL,
 ) -> StreamingStructure:
     _validate_xml_root(filepath, "tmx")
     extractor = TmxExtractor(
@@ -145,6 +281,7 @@ def stream_tmx(
         source_language=source_language,
         target_language=target_language,
         parse_header=not (source_language and target_language),
+        mode=mode,
     )
     return StreamingStructure(
         source_locale=extractor.source_locale or extractor.native_source,
@@ -460,7 +597,7 @@ def _validate_xml_root(filepath: str, expected: str) -> None:
 def _convert_tmx(
     source_path: str,
     target_path: str,
-    exporter: Callable[[Any, str], None],
+    exporter: Callable[[StreamingStructure, str], None],
     source_language: str | None,
     target_language: str | None,
 ) -> ConversionStats:

lokit_python-0.1.2/src/lokit/io/atomic.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import contextlib
+import os
+import tempfile
+from collections.abc import Iterator
+from contextlib import AbstractContextManager, contextmanager
+from pathlib import Path
+from typing import BinaryIO, Literal, TextIO, cast, overload
+
+
+@overload
+def atomic_output_path(
+    path: Path,
+    mode: Literal[
+        "w",
+        "wt",
+        "w+",
+        "wt+",
+        "a",
+        "at",
+        "a+",
+        "at+",
+        "x",
+        "xt",
+        "x+",
+        "xt+",
+    ],
+) -> AbstractContextManager[TextIO]: ...
+
+
+@overload
+def atomic_output_path(
+    path: Path,
+    mode: Literal[
+        "wb",
+        "w+b",
+        "wb+",
+        "ab",
+        "a+b",
+        "ab+",
+        "xb",
+        "x+b",
+        "xb+",
+    ] = "wb",
+) -> AbstractContextManager[BinaryIO]: ...
+
+
+@overload
+def atomic_output_path(
+    path: Path,
+    mode: str,
+) -> AbstractContextManager[BinaryIO | TextIO]: ...
+
+
+def atomic_output_path(
+    path: Path,
+    mode: str = "wb",
+) -> AbstractContextManager[BinaryIO | TextIO]:
+    return _atomic_output_path(path, mode)
+
+
+@contextmanager
+def _atomic_output_path(path: Path, mode: str) -> Iterator[BinaryIO | TextIO]:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = tempfile.NamedTemporaryFile(
+        mode=mode,
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        delete=False,
+    )
+    tmp_path = Path(tmp.name)
+    try:
+        with tmp:
+            yield cast(BinaryIO | TextIO, tmp)
+            tmp.flush()
+            os.fsync(tmp.fileno())
+        os.replace(tmp_path, path)
+        directory_flag = getattr(os, "O_DIRECTORY", None)
+        if directory_flag is not None:
+            dir_fd = os.open(path.parent, directory_flag)
+            try:
+                os.fsync(dir_fd)
+            finally:
+                os.close(dir_fd)
+    except BaseException:
+        with contextlib.suppress(FileNotFoundError):
+            tmp_path.unlink()
+        raise

lokit_python-0.1.2/src/lokit/parsers/async_bridge.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from collections.abc import Callable, Iterator
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class AsyncExtractionBatch(Generic[T]):
+    __slots__ = ("done", "error", "items")
+
+    def __init__(
+        self,
+        items: list[T] | None = None,
+        error: BaseException | None = None,
+        done: bool = False,
+    ) -> None:
+        self.items = items
+        self.error = error
+        self.done = done
+
+
+class AsyncExtractionBridge(Generic[T]):
+    def __init__(
+        self,
+        iterator_factory: Callable[[], Iterator[T]],
+        maxsize: int = 4,
+        batch_size: int = 1000,
+    ) -> None:
+        if maxsize < 1:
+            raise ValueError("maxsize must be at least 1")
+        if batch_size < 1:
+            raise ValueError("batch_size must be at least 1")
+        self._iterator_factory = iterator_factory
+        self._queue: asyncio.Queue[AsyncExtractionBatch[T]] = asyncio.Queue(
+            maxsize=maxsize
+        )
+        self._batch_size = batch_size
+        self._current_batch: list[T] = []
+        self._batch_index = 0
+        self._stop = threading.Event()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> AsyncExtractionBridge[T]:
+        return self
+
+    async def __anext__(self) -> T:
+        if self._producer is None:
+            self._start()
+
+        while self._batch_index >= len(self._current_batch):
+            result = await self._queue.get()
+            if result.done:
+                await self.aclose()
+                raise StopAsyncIteration
+            if result.error is not None:
+                await self.aclose()
+                raise result.error
+            if result.items is None:
+                await self.aclose()
+                raise StopAsyncIteration
+            self._current_batch = result.items
+            self._batch_index = 0
+
+        item = self._current_batch[self._batch_index]
+        self._batch_index += 1
+        return item
+
+    async def aclose(self) -> None:
+        self._stop.set()
+        if self._producer is not None:
+            await self._producer
+            self._producer = None
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                batch: list[T] = []
+                for item in self._iterator_factory():
+                    if self._stop.is_set():
+                        break
+                    batch.append(item)
+                    if len(batch) >= self._batch_size:
+                        self._put(loop, AsyncExtractionBatch(items=batch))
+                        batch = []
+                if batch:
+                    self._put(loop, AsyncExtractionBatch(items=batch))
+            except BaseException as exc:
+                self._put(loop, AsyncExtractionBatch(error=exc))
+            finally:
+                self._put(loop, AsyncExtractionBatch(done=True))
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    def _put(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        result: AsyncExtractionBatch[T],
+    ) -> None:
+        if self._stop.is_set() and not result.done:
+            return
+        future = asyncio.run_coroutine_threadsafe(self._queue.put(result), loop)
+        future.result()

{lokit_python-0.1.1 → lokit_python-0.1.2}/src/lokit/parsers/tmx/base.py

@@ -5,7 +5,12 @@ from lxml import etree
 from lokit.core.logger import logger
 from lokit.parsers.tmx.header import TmxHeaderParser
 from lokit.parsers.tmx.models import HeaderData
-from lokit.parsers.tmx.xml_utils import clear_element, element_children, iterparse_safe, local_name
+from lokit.parsers.tmx.xml_utils import (
+    clear_element,
+    element_children,
+    iterparse_safe,
+    local_name,
+)
 
 
 class TmxParser:
@@ -37,6 +42,9 @@ class TmxParser:
         if parse_header:
             self._initialize_from_file()
         self._validate_and_set_languages()
+        self.native_source_base: str = self._base_lang(self.native_source)
+        self.native_target_base: str = self._base_lang(self.native_target)
+        self._lang_base_cache: dict[str, str] = {}
 
     def _initialize_from_file(self) -> None:
         context = iterparse_safe(self.filepath, events=("end",))
@@ -82,9 +90,21 @@ class TmxParser:
     def _compare_base_lang(self, lang1: str, lang2: str) -> bool:
         if not lang1 or not lang2:
             return False
-        l1 = lang1.replace("_", "-").split("-")[0].lower()
-        l2 = lang2.replace("_", "-").split("-")[0].lower()
-        return l1 == l2
+        return self._base_lang(lang1) == self._base_lang(lang2)
+
+    def _base_lang(self, lang: str) -> str:
+        if not lang:
+            return ""
+        normalized = lang.replace("_", "-")
+        return normalized.split("-", 1)[0].lower()
+
+    def _cached_base_lang(self, lang: str) -> str:
+        cached = self._lang_base_cache.get(lang)
+        if cached is not None:
+            return cached
+        base_lang = self._base_lang(lang)
+        self._lang_base_cache[lang] = base_lang
+        return base_lang
 
     def _initialize_missing_languages_from_tu(self, element: etree._Element) -> None:
         langs: list[str] = []

lokit_python-0.1.2/src/lokit/parsers/tmx/extraction.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from typing import AsyncIterator, Iterator, Optional
+from uuid import uuid4
+
+from lxml.etree import _Element
+
+from lokit.data.structure import Data, Meta, SegmentPart, Tags, TranslationStatus
+from lokit.data.tag_types import TieData
+from lokit.parsers.async_bridge import AsyncExtractionBridge
+from lokit.parsers.tmx.base import TmxParser
+from lokit.parsers.tmx.models import TmxParseMode
+from lokit.parsers.tmx.props import ParsedTmxProps, TmxProps
+from lokit.parsers.tmx.tags import TmxTagParser
+from lokit.parsers.tmx.xml_utils import (
+    clear_element,
+    is_tag,
+    iterparse_safe,
+    local_name,
+)
+
+ExtractItem = tuple[str, Data]
+
+
+class TmxExtractor(TmxParser):
+    def __init__(
+        self,
+        filepath: str,
+        source_language: Optional[str] = None,
+        target_language: Optional[str] = None,
+        domain: Optional[str] = None,
+        parse_header: bool = True,
+        mode: TmxParseMode = TmxParseMode.FULL,
+    ) -> None:
+        super().__init__(
+            tmx_file_path=filepath,
+            source_language=source_language,
+            target_language=target_language,
+            domain=domain,
+            parse_header=parse_header,
+        )
+        self.tag_parser: TmxTagParser = TmxTagParser()
+        self.prop_parser: TmxProps = TmxProps()
+        self.namespace: str = "{http://www.w3.org/XML/1998/namespace}"
+        self.mode = mode
+
+    def extract(self) -> Iterator[tuple[str, Data]]:
+        with open(self.filepath, "rb") as stream:
+            context = iterparse_safe(stream, events=("end",))
+
+            for _, elem in context:
+                if local_name(elem.tag) != "tu":
+                    continue
+
+                yield self.extract_element(elem)
+
+                clear_element(elem)
+
+    def extract_element(self, elem: _Element) -> tuple[str, Data]:
+        unit_id: str = elem.attrib.get("tuid") or str(uuid4())
+
+        props: ParsedTmxProps | None = None
+        status = TranslationStatus.UNKNOWN
+        if self.mode is TmxParseMode.FULL:
+            props = self.prop_parser.parse_all(elem)
+            status = props.status
+        elif self.mode is TmxParseMode.TEXT_WITH_STATUS:
+            status = self.prop_parser.parse_status(elem)
+
+        source_text: str = ""
+        target_text: str = ""
+        source_tags: dict[str, TieData] | None = None
+        target_tags: dict[str, TieData] | None = None
+        source_parts: list[SegmentPart] | None = None
+        target_parts: list[SegmentPart] | None = None
+
+        for tuv in elem:
+            if not is_tag(tuv, "tuv"):
+                continue
+            lang: str = tuv.get(f"{self.namespace}lang") or tuv.get("lang") or ""
+            seg: _Element | None = None
+            for tuv_child in tuv:
+                if is_tag(tuv_child, "seg"):
+                    seg = tuv_child
+                    break
+
+            if seg is not None:
+                text, tags, parts = self.tag_parser.parse_fast(seg)
+
+                if self._cached_base_lang(lang) == self.native_source_base:
+                    source_text = text
+                    source_tags = tags
+                    source_parts = parts
+                else:
+                    target_text = text
+                    target_tags = tags
+                    target_parts = parts
+
+        tags_obj: Tags | None = None
+        if source_tags is not None or target_tags is not None:
+            tags_obj = Tags(
+                source_tag_map=source_tags or {},
+                target_tag_map=target_tags or {},
+                source_parts=source_parts or [],
+                target_parts=target_parts or [],
+            )
+
+        data_obj = Data(
+            source=source_text,
+            target=target_text if target_text else None,
+            plural=None,
+            tags=tags_obj,
+            meta=props.meta if props is not None else Meta(),
+            status=status,
+            comments=props.comments if props is not None else [],
+            previous_context=(props.previous_context if props is not None else None),
+            next_context=props.next_context if props is not None else None,
+            extensions=props.extensions if props is not None else {},
+        )
+
+        return unit_id, data_obj
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return AsyncExtractionBridge(self.extract)

{lokit_python-0.1.1 → lokit_python-0.1.2}/src/lokit/parsers/tmx/models.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from enum import StrEnum
 
 
 @dataclass
@@ -8,3 +9,9 @@ class HeaderData:
     srclang: str
     tgtlang: str
     extensions: dict[str, str]
+
+
+class TmxParseMode(StrEnum):
+    FULL = "full"
+    TEXT = "text"
+    TEXT_WITH_STATUS = "text_status"