lokit-python 0.1.0__cp313-cp313-win_amd64.whl

lokit/parsers/tmx/extraction.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import AsyncIterator, Iterator, Optional
+from uuid import uuid4
+
+from lxml.etree import _Element
+
+from lokit.data.structure import Data, SegmentPart, Tags
+from lokit.data.tag_types import TieData
+from lokit.parsers.tmx.base import TmxParser
+from lokit.parsers.tmx.props import TmxProps
+from lokit.parsers.tmx.tags import TmxTagParser
+from lokit.parsers.tmx.xml_utils import (
+    clear_element,
+    element_children,
+    find_child,
+    iterparse_safe,
+    local_name,
+)
+
+ExtractItem = tuple[str, Data]
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class AsyncTmxExtraction:
+    def __init__(self, extractor: TmxExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> AsyncTmxExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class TmxExtractor(TmxParser):
+    def __init__(
+        self,
+        filepath: str,
+        source_language: Optional[str] = None,
+        target_language: Optional[str] = None,
+        domain: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            tmx_file_path=filepath,
+            source_language=source_language,
+            target_language=target_language,
+            domain=domain,
+        )
+        self.tag_parser: TmxTagParser = TmxTagParser()
+        self.prop_parser: TmxProps = TmxProps()
+        self.namespace: str = "{http://www.w3.org/XML/1998/namespace}"
+
+    def extract(self) -> Iterator[tuple[str, Data]]:
+        with open(self.filepath, "rb") as stream:
+            context = iterparse_safe(stream, events=("end",))
+
+            for _, elem in context:
+                if local_name(elem.tag) != "tu":
+                    continue
+
+                unit_id: str = elem.attrib.get("tuid") or str(uuid4())
+
+                meta = self.prop_parser.parse_meta(elem)
+                comments = self.prop_parser.parse_comments(elem)
+                prev_ctx, next_ctx = self.prop_parser.parse_adjacent_context(elem)
+                status = self.prop_parser.parse_status(elem)
+                extensions = self.prop_parser.parse_extensions(elem)
+
+                source_text: str = ""
+                target_text: str = ""
+                source_tags: dict[str, TieData] = {}
+                target_tags: dict[str, TieData] = {}
+                source_parts: list[SegmentPart] = []
+                target_parts: list[SegmentPart] = []
+
+                for tuv in element_children(elem, "tuv"):
+                    lang: str = (
+                        tuv.get(f"{self.namespace}lang") or tuv.get("lang") or ""
+                    )
+                    seg: _Element | None = find_child(tuv, "seg")
+
+                    if seg is not None:
+                        text, tags, parts = self.tag_parser.parse(seg)
+
+                        if self._compare_base_lang(lang, self.native_source):
+                            source_text = text
+                            source_tags = tags
+                            source_parts = parts
+                        else:
+                            target_text = text
+                            target_tags = tags
+                            target_parts = parts
+
+                tags_obj = Tags(
+                    source_tag_map=source_tags,
+                    target_tag_map=target_tags,
+                    source_parts=source_parts,
+                    target_parts=target_parts,
+                )
+
+                data_obj = Data(
+                    source=source_text,
+                    target=target_text if target_text else None,
+                    plural=None,
+                    tags=tags_obj if (source_tags or target_tags) else None,
+                    meta=meta,
+                    status=status,
+                    comments=comments,
+                    previous_context=prev_ctx,
+                    next_context=next_ctx,
+                    extensions=extensions,
+                )
+
+                yield unit_id, data_obj
+
+                clear_element(elem)
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return AsyncTmxExtraction(self)

lokit/parsers/tmx/header.py

@@ -0,0 +1,55 @@
+from lxml.etree import _Element
+
+from lokit.parsers.tmx.models import HeaderData
+from lokit.parsers.tmx.xml_utils import element_children, local_name
+
+
+class TmxHeaderParser:
+    def __init__(self) -> None:
+        pass
+
+    def parse(self, element: _Element) -> HeaderData:
+        creation_tool: str = element.attrib.get("creationtool") or "unknown_origin"
+        tool_version: str = element.attrib.get("creationtoolversion") or ""
+        origin: str = f"{creation_tool} {tool_version}".strip()
+        timestamp: str = element.attrib.get("creationdate") or ""
+        extensions: dict[str, str] = {
+            "input_format": "tmx",
+        }
+        self._add_if_present(extensions, "admin_locale", element.attrib.get("adminlang"))
+        self._add_if_present(extensions, "data_type", element.attrib.get("datatype"))
+        self._add_if_present(extensions, "segmentation", element.attrib.get("segtype"))
+        self._add_if_present(extensions, "translation_memory_format", element.attrib.get("o-tmf"))
+        self._add_if_present(extensions, "tool_name", creation_tool)
+        self._add_if_present(extensions, "tool_version", tool_version)
+
+        srclang: str = element.attrib.get("srclang") or ""
+        if srclang == "*all*":
+            srclang = ""
+
+        tgtlang: str = element.attrib.get("tgtlang") or ""
+
+        for child in element_children(element):
+            child_name = local_name(child.tag)
+            if child_name == "prop":
+                prop_type = child.attrib.get("type") or "unknown"
+                extensions[f"property.{self._normalize_key(prop_type)}"] = child.text or ""
+            elif child.text:
+                extensions[f"property.{self._normalize_key(child_name)}"] = child.text
+
+        return HeaderData(
+            origin=origin,
+            timestamp=timestamp,
+            srclang=srclang,
+            tgtlang=tgtlang,
+            extensions=extensions,
+        )
+
+    def _add_if_present(
+        self, extensions: dict[str, str], key: str, value: str | None
+    ) -> None:
+        if value:
+            extensions[key] = value
+
+    def _normalize_key(self, value: str) -> str:
+        return value.lower().replace(" ", "_").replace("-", "_")

lokit/parsers/tmx/helpers.py

@@ -0,0 +1,9 @@
+from lokit.data.tag_types import TieType as tt
+
+TMX_TAG_MAP: dict[str, tt] = {
+    "bpt": tt.CUSTOM_OPEN,
+    "ept": tt.CUSTOM_CLOSE,
+    "ph": tt.CUSTOM_STANDALONE,
+    "it": tt.CUSTOM_STANDALONE,
+    "ut": tt.CUSTOM_STANDALONE,
+}

lokit/parsers/tmx/models.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class HeaderData:
+    origin: str
+    timestamp: str
+    srclang: str
+    tgtlang: str
+    extensions: dict[str, str]

lokit/parsers/tmx/props.py

@@ -0,0 +1,201 @@
+from typing import Optional
+
+from lxml.etree import _Element
+
+from lokit.data.structure import AdjacentContext, Comment, Meta, Origin, TranslationStatus
+from lokit.parsers.tmx.xml_utils import element_children, local_name
+
+
+class TmxProps:
+    def __init__(self) -> None:
+        self._known_props: frozenset[str] = frozenset(
+            {
+                "status",
+                "x-status",
+                "x-xtm-status",
+                "x-project",
+                "x-system",
+                "x-domain",
+                "x-context",
+                "x-key",
+                "note",
+                "x-note",
+                "comment",
+                "x-comment",
+                "x-previous-id",
+                "x-previous-source",
+                "x-previous-source-text",
+                "x-previous-target",
+                "x-previous-target-text",
+                "x-next-id",
+                "x-next-source",
+                "x-next-source-text",
+                "x-next-target",
+                "x-next-target-text",
+            }
+        )
+
+    def parse_meta(self, element: _Element) -> Meta:
+        creation_date: str = element.attrib.get("creationdate") or ""
+        _usage_count: str = element.attrib.get("usagecount") or ""
+        usage_count: Optional[int] = (
+            int(_usage_count) if _usage_count.isdigit() else None
+        )
+        return Meta(
+            usage_count=usage_count,
+            last_used=element.attrib.get("lastusagedate"),
+            first_used=None,
+            created=creation_date if creation_date else None,
+            updated=element.attrib.get("changedate"),
+            max_length=None,
+            min_length=None,
+            extensions=self.parse_meta_extensions(element),
+        )
+
+    def parse_comments(self, element: _Element) -> list[Comment]:
+        project: str = ""
+        system: str = ""
+        creator: str = element.attrib.get("creationid") or ""
+        context_key: str = ""
+        comments: list[Comment] = []
+
+        for child in element_children(element):
+            tag_name: str = local_name(child.tag)
+            if tag_name == "prop":
+                prop_type: str = child.attrib.get("type", "").lower()
+                text_val: str = child.text or ""
+                if prop_type == "x-project":
+                    project = text_val
+                elif prop_type in ("x-system", "x-domain"):
+                    system = text_val
+                elif prop_type in ("x-context", "x-key"):
+                    context_key = text_val
+                elif prop_type in ("note", "x-note", "comment", "x-comment"):
+                    comments.append(
+                        Comment(
+                            context=text_val.strip(),
+                            timestamp=element.attrib.get("changedate"),
+                        )
+                    )
+
+            elif tag_name == "note" and child.text:
+                comments.append(
+                    Comment(
+                        context=child.text.strip(),
+                        timestamp=element.attrib.get("changedate"),
+                    )
+                )
+
+        if not any([project, system, creator, context_key]) and not comments:
+            return []
+
+        origin: Origin = Origin(
+            system=system if system else None,
+            project=project if project else None,
+            creator_id=creator if creator else None,
+        )
+
+        if not comments:
+            comments.append(Comment(context=""))
+
+        for comment in comments:
+            comment.origin = origin if any([system, project, creator]) else None
+            comment.context_key = context_key if context_key else None
+            comment.timestamp = comment.timestamp or element.attrib.get("changedate")
+
+        return comments
+
+    def parse_status(self, element: _Element) -> TranslationStatus:
+        status_values: list[str] = []
+
+        for child in element_children(element, "prop"):
+            prop_type: str = child.attrib.get("type", "").lower()
+            if prop_type in ("status", "x-status", "x-xtm-status"):
+                status_values.append((child.text or "").strip().lower())
+
+        for value in reversed(status_values):
+            if value in ("approved", "signed-off", "final"):
+                return TranslationStatus.APPROVED
+            if value in ("reviewed", "review"):
+                return TranslationStatus.REVIEWED
+            if value in ("translated", "complete"):
+                return TranslationStatus.TRANSLATED
+            if value in ("new",):
+                return TranslationStatus.NEW
+            if value in ("draft", "notapproved", "not-approved", "unapproved"):
+                return TranslationStatus.DRAFT
+            if value in ("rejected",):
+                return TranslationStatus.REJECTED
+
+        return TranslationStatus.UNKNOWN
+
+    def parse_adjacent_context(
+        self, element: _Element
+    ) -> tuple[Optional[AdjacentContext], Optional[AdjacentContext]]:
+        prev_id: Optional[str] = None
+        prev_src: Optional[str] = None
+        prev_tgt: Optional[str] = None
+        next_id: Optional[str] = None
+        next_src: Optional[str] = None
+        next_tgt: Optional[str] = None
+
+        for child in element_children(element, "prop"):
+            prop_type: str = child.attrib.get("type", "").lower()
+            text_val: str = child.text or ""
+            if prop_type == "x-previous-id":
+                prev_id = text_val
+            elif prop_type in ("x-previous-source", "x-previous-source-text"):
+                prev_src = text_val
+            elif prop_type in ("x-previous-target", "x-previous-target-text"):
+                prev_tgt = text_val
+            elif prop_type == "x-next-id":
+                next_id = text_val
+            elif prop_type in ("x-next-source", "x-next-source-text"):
+                next_src = text_val
+            elif prop_type in ("x-next-target", "x-next-target-text"):
+                next_tgt = text_val
+
+        prev_ctx: Optional[AdjacentContext] = (
+            AdjacentContext(unit_id=prev_id, source=prev_src, target=prev_tgt)
+            if any([prev_id, prev_src, prev_tgt])
+            else None
+        )
+
+        next_ctx: Optional[AdjacentContext] = (
+            AdjacentContext(
+                unit_id=next_id,
+                source=next_src,
+                target=next_tgt,
+            )
+            if any([next_id, next_src, next_tgt])
+            else None
+        )
+
+        return prev_ctx, next_ctx
+
+    def parse_meta_extensions(self, element: _Element) -> dict[str, str]:
+        extensions: dict[str, str] = {}
+
+        change_id = element.attrib.get("changeid")
+        if change_id:
+            extensions["change_id"] = change_id
+
+        usage_count = element.attrib.get("usagecount")
+        if usage_count:
+            extensions["usage_count_raw"] = usage_count
+
+        return extensions
+
+    def parse_extensions(self, element: _Element) -> dict[str, str]:
+        extensions: dict[str, str] = {}
+
+        for child in element_children(element, "prop"):
+            prop_type = (child.attrib.get("type") or "unknown").lower()
+            if prop_type in self._known_props:
+                continue
+            extensions[f"property.{self._normalize_key(prop_type)}"] = child.text or ""
+
+        return extensions
+
+    def _normalize_key(self, value: str) -> str:
+        return value.lower().replace(" ", "_").replace("-", "_")

lokit/parsers/tmx/tags.py

@@ -0,0 +1,59 @@
+from lxml.etree import _Element
+
+from lokit.data.structure import CodePart, SegmentPart, TextPart
+from lokit.data.tag_types import TieData, TieType
+from lokit.parsers.tmx.helpers import TMX_TAG_MAP
+from lokit.parsers.tmx.xml_utils import element_children, local_name
+
+
+class TmxTagParser:
+    def __init__(self) -> None:
+        pass
+
+    def parse(
+        self, element: _Element
+    ) -> tuple[str, dict[str, TieData], list[SegmentPart]]:
+        raw_txt: str = ""
+        tag_map: dict[str, TieData] = {}
+        parts: list[SegmentPart] = []
+        order: int = 0
+        pair_ids: dict[str, str] = {}
+
+        if element.text:
+            raw_txt += element.text
+            parts.append(TextPart(element.text))
+
+        for child in element_children(element):
+            tag_name: str = local_name(child.tag)
+            tie_type: TieType = TMX_TAG_MAP.get(tag_name, TieType.CUSTOM_STANDALONE)
+            source_pair_id: str | None = child.attrib.get("i") or child.attrib.get("id")
+            pair_id = self._normalize_pair_id(source_pair_id, pair_ids)
+            tie_id = f"c{order}"
+
+            tag_map[tie_id] = TieData(
+                id=tie_id,
+                type=tie_type,
+                position=len(raw_txt),
+                order=order,
+                pair_id=pair_id,
+            )
+            parts.append(CodePart(tie_id))
+            order += 1
+
+            if child.tail:
+                raw_txt += child.tail
+                parts.append(TextPart(child.tail))
+
+        return raw_txt, tag_map, parts
+
+    def _normalize_pair_id(
+        self, source_pair_id: str | None, pair_ids: dict[str, str]
+    ) -> str | None:
+        if source_pair_id is None:
+            return None
+        existing = pair_ids.get(source_pair_id)
+        if existing is not None:
+            return existing
+        normalized = f"p{len(pair_ids)}"
+        pair_ids[source_pair_id] = normalized
+        return normalized

lokit/parsers/tmx/xml_utils.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import BinaryIO
+
+from lxml import etree
+from lxml.etree import _Element
+
+
+def local_name(tag: object) -> str:
+    if not isinstance(tag, str):
+        return ""
+    return tag.rsplit("}", 1)[-1]
+
+
+def iterparse_safe(
+    source: str | BinaryIO,
+    events: tuple[str, ...],
+) -> etree.iterparse[etree._Element]:
+    return etree.iterparse(
+        source,
+        events=events,
+        no_network=True,
+        resolve_entities=False,
+    )
+
+
+def element_children(element: _Element, name: str | None = None) -> Iterator[_Element]:
+    for child in element:
+        if name is None or local_name(child.tag) == name:
+            yield child
+
+
+def find_child(element: _Element, name: str) -> _Element | None:
+    for child in element_children(element, name):
+        return child
+    return None
+
+
+def clear_element(element: _Element) -> None:
+    element.clear()
+    while element.getprevious() is not None:
+        parent = element.getparent()
+        if parent is None:
+            break
+        del parent[0]

lokit/parsers/xliff/__init__.py

@@ -0,0 +1,3 @@
+from lokit.parsers.xliff.extraction import XliffExtractor
+
+__all__ = ["XliffExtractor"]