lokit-python 0.1.0__cp313-cp313-win_amd64.whl

lokit/parsers/xliff/extraction.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import AsyncIterator, Iterator, Optional
+
+from lxml.etree import _Element
+
+from lokit.data.structure import Comment, Data, Meta, SegmentPart, Tags, TranslationStatus
+from lokit.data.tag_types import TieData
+from lokit.parsers.tmx.xml_utils import (
+    clear_element,
+    element_children,
+    find_child,
+    iterparse_safe,
+    local_name,
+)
+from lokit.parsers.xliff.tags import XliffTagParser
+
+ExtractItem = tuple[str, Data]
+
+
+@dataclass(slots=True)
+class XliffFileContext:
+    index: int
+    original: str
+    source_locale: str
+    target_locale: str | None
+    data_type: str
+    tool_name: str | None = None
+    tool_version: str | None = None
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class AsyncXliffExtraction:
+    def __init__(self, extractor: XliffExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> AsyncXliffExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class XliffExtractor:
+    def __init__(self, filepath: str) -> None:
+        self.filepath = filepath
+        self.version = "1.2"
+        self.source_locale: str | None = None
+        self.target_locale: str | None = None
+        self.source_language: str | None = None
+        self.target_language: str | None = None
+        self.export_origin = ""
+        self.export_timestamp = ""
+        self.extensions: dict[str, str] = {"input_format": "xliff"}
+        self.tag_parser = XliffTagParser()
+
+    def extract(self) -> Iterator[ExtractItem]:
+        context = iterparse_safe(self.filepath, events=("start", "end"))
+        file_stack: list[XliffFileContext] = []
+        file_index = 0
+
+        for event, elem in context:
+            name = local_name(elem.tag)
+            if event == "start" and name == "xliff":
+                self.version = elem.attrib.get("version", "1.2")
+                self.extensions["xliff_version"] = self.version
+            elif event == "start" and name == "file":
+                current = self._file_context(elem, file_index)
+                file_index += 1
+                file_stack.append(current)
+                self._set_document_languages(current)
+            elif event == "end" and name == "file":
+                if file_stack:
+                    file_stack.pop()
+                clear_element(elem)
+            elif event == "end" and name == "trans-unit" and file_stack:
+                current_file = file_stack[-1]
+                yield self._parse_unit(elem, current_file)
+                clear_element(elem)
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return AsyncXliffExtraction(self)
+
+    def _file_context(self, element: _Element, index: int) -> XliffFileContext:
+        original = element.attrib.get("original", "")
+        source_locale = element.attrib.get("source-language", "")
+        target_locale = element.attrib.get("target-language")
+        data_type = element.attrib.get("datatype", "")
+        return XliffFileContext(
+            index=index,
+            original=original,
+            source_locale=source_locale,
+            target_locale=target_locale,
+            data_type=data_type,
+        )
+
+    def _set_document_languages(self, context: XliffFileContext) -> None:
+        if self.source_locale is None and context.source_locale:
+            self.source_locale = context.source_locale
+            self.source_language = self._base_language(context.source_locale)
+        if self.target_locale is None and context.target_locale:
+            self.target_locale = context.target_locale
+            self.target_language = self._base_language(context.target_locale)
+
+    def _parse_unit(
+        self,
+        element: _Element,
+        file_context: XliffFileContext,
+    ) -> ExtractItem:
+        source = find_child(element, "source")
+        target = find_child(element, "target")
+        source_text, source_tags, source_parts = self._parse_segment(source)
+        target_text, target_tags, target_parts = self._parse_segment(target)
+        unit_id = element.attrib.get("id", "")
+        stable_id = f"{file_context.index}:{unit_id}" if unit_id else f"{file_context.index}"
+        tags = Tags(
+            source_tag_map=source_tags,
+            target_tag_map=target_tags,
+            source_parts=source_parts,
+            target_parts=target_parts,
+        )
+        data = Data(
+            source=source_text,
+            target=target_text if target is not None else None,
+            tags=tags if source_tags or target_tags else None,
+            meta=Meta(),
+            status=self._status(target),
+            comments=self._comments(element),
+            extensions=self._extensions(element, file_context, unit_id),
+        )
+        return stable_id, data
+
+    def _parse_segment(
+        self, element: _Element | None
+    ) -> tuple[str, dict[str, TieData], list[SegmentPart]]:
+        if element is None:
+            return "", {}, []
+        return self.tag_parser.parse(element)
+
+    def _status(self, target: _Element | None) -> TranslationStatus:
+        if target is None:
+            return TranslationStatus.NEW
+        state = (target.attrib.get("state") or "").lower()
+        if state in ("final", "signed-off"):
+            return TranslationStatus.APPROVED
+        if state in ("translated", "needs-review-translation"):
+            return TranslationStatus.TRANSLATED
+        if state in ("needs-review-adaptation", "needs-review-l10n"):
+            return TranslationStatus.REVIEWED
+        if state in ("new", "needs-translation"):
+            return TranslationStatus.NEW
+        return TranslationStatus.UNKNOWN
+
+    def _comments(self, element: _Element) -> list[Comment]:
+        comments: list[Comment] = []
+        for child in element_children(element, "note"):
+            if child.text:
+                comments.append(Comment(context=child.text.strip()))
+        return comments
+
+    def _extensions(
+        self,
+        element: _Element,
+        file_context: XliffFileContext,
+        unit_id: str,
+    ) -> dict[str, str]:
+        extensions = {
+            "resource": file_context.original,
+            "resource_index": str(file_context.index),
+            "unit_id": unit_id,
+        }
+        if file_context.data_type:
+            extensions["data_type"] = file_context.data_type
+        xml_space = element.attrib.get("{http://www.w3.org/XML/1998/namespace}space")
+        if xml_space:
+            extensions["space"] = xml_space
+        return extensions
+
+    def _base_language(self, locale: str) -> str:
+        return locale.replace("_", "-").split("-")[0].lower()

lokit/parsers/xliff/tags.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from lxml.etree import _Element
+
+from lokit.data.structure import CodePart, SegmentPart, TextPart
+from lokit.data.tag_types import TieData, TieType
+from lokit.parsers.tmx.xml_utils import element_children, local_name
+
+
+class XliffTagParser:
+    def parse(
+        self, element: _Element
+    ) -> tuple[str, dict[str, TieData], list[SegmentPart]]:
+        raw_text = ""
+        tag_map: dict[str, TieData] = {}
+        parts: list[SegmentPart] = []
+        pair_ids: dict[str, str] = {}
+        order = 0
+
+        raw_text, order = self._append_content(
+            element,
+            raw_text,
+            parts,
+            tag_map,
+            pair_ids,
+            order,
+            include_element_code=False,
+        )
+        return raw_text, tag_map, parts
+
+    def _append_content(
+        self,
+        element: _Element,
+        raw_text: str,
+        parts: list[SegmentPart],
+        tag_map: dict[str, TieData],
+        pair_ids: dict[str, str],
+        order: int,
+        include_element_code: bool,
+    ) -> tuple[str, int]:
+        if include_element_code:
+            open_id = f"c{order}"
+            pair_id = self._pair_id(element, pair_ids)
+            tag_map[open_id] = TieData(
+                id=open_id,
+                type=self._open_type(element),
+                position=len(raw_text),
+                order=order,
+                pair_id=pair_id,
+            )
+            parts.append(CodePart(open_id))
+            order += 1
+
+        if element.text:
+            raw_text += element.text
+            parts.append(TextPart(element.text))
+
+        for child in element_children(element):
+            child_name = local_name(child.tag)
+            if child_name in ("g", "mrk", "sub"):
+                raw_text, order = self._append_content(
+                    child,
+                    raw_text,
+                    parts,
+                    tag_map,
+                    pair_ids,
+                    order,
+                    include_element_code=True,
+                )
+            else:
+                code_id = f"c{order}"
+                tag_map[code_id] = TieData(
+                    id=code_id,
+                    type=self._inline_type(child),
+                    position=len(raw_text),
+                    order=order,
+                    pair_id=self._pair_id(child, pair_ids),
+                )
+                parts.append(CodePart(code_id))
+                order += 1
+
+            if child.tail:
+                raw_text += child.tail
+                parts.append(TextPart(child.tail))
+
+        if include_element_code:
+            close_id = f"c{order}"
+            tag_map[close_id] = TieData(
+                id=close_id,
+                type=self._close_type(element),
+                position=len(raw_text),
+                order=order,
+                pair_id=self._pair_id(element, pair_ids),
+            )
+            parts.append(CodePart(close_id))
+            order += 1
+
+        return raw_text, order
+
+    def _pair_id(self, element: _Element, pair_ids: dict[str, str]) -> str | None:
+        source_id = (
+            element.attrib.get("rid")
+            or element.attrib.get("id")
+            or element.attrib.get("xid")
+            or element.attrib.get("ctype")
+        )
+        if source_id is None:
+            return None
+        existing = pair_ids.get(source_id)
+        if existing is not None:
+            return existing
+        normalized = f"p{len(pair_ids)}"
+        pair_ids[source_id] = normalized
+        return normalized
+
+    def _inline_type(self, element: _Element) -> TieType:
+        name = local_name(element.tag)
+        if name in ("bpt", "bx"):
+            return TieType.CUSTOM_OPEN
+        if name in ("ept", "ex"):
+            return TieType.CUSTOM_CLOSE
+        return TieType.CUSTOM_STANDALONE
+
+    def _open_type(self, element: _Element) -> TieType:
+        return TieType.CUSTOM_OPEN
+
+    def _close_type(self, element: _Element) -> TieType:
+        return TieType.CUSTOM_CLOSE

lokit/parsers/xlsx/__init__.py

@@ -0,0 +1 @@
+

lokit/parsers/xlsx/extraction.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncIterator, Iterator, Optional
+
+from openpyxl import load_workbook
+from openpyxl.cell.cell import Cell, MergedCell
+
+from lokit.data.structure import Comment, Data, TranslationStatus
+
+ExtractItem = tuple[str, Data]
+
+_KNOWN_COLUMNS = frozenset({"id", "source", "target", "status", "comment"})
+
+
+def _parse_base_lang(locale: str) -> str:
+    return locale.replace("_", "-").split("-")[0].lower()
+
+
+def _parse_status(value: str) -> TranslationStatus:
+    normalized = value.strip().lower()
+    try:
+        return TranslationStatus(normalized)
+    except ValueError:
+        return TranslationStatus.UNKNOWN
+
+
+def _cell_str(cell: Cell | MergedCell) -> str:
+    if cell.value is None:
+        return ""
+    return str(cell.value)
+
+
+def _infer_locales_from_filename(filepath: str) -> tuple[str, str | None]:
+    stem = Path(filepath).stem
+    if "-" in stem:
+        parts = stem.split("-")
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        if len(parts) == 4:
+            return f"{parts[0]}-{parts[1]}", f"{parts[2]}-{parts[3]}"
+    if "_" in stem:
+        parts = stem.split("_")
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        if len(parts) == 4:
+            return f"{parts[0]}_{parts[1]}", f"{parts[2]}_{parts[3]}"
+    return "", None
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class AsyncXlsxExtraction:
+    def __init__(self, extractor: XlsxExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> AsyncXlsxExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class XlsxExtractor:
+    def __init__(
+        self,
+        filepath: str,
+        source_locale: str = "",
+        target_locale: str | None = None,
+    ) -> None:
+        self.filepath: str = filepath
+
+        if source_locale:
+            self.source_locale: str = source_locale
+            self.target_locale: str | None = target_locale
+        else:
+            inferred_source, inferred_target = _infer_locales_from_filename(filepath)
+            self.source_locale = inferred_source
+            self.target_locale = target_locale or inferred_target
+
+        self.source_language: str | None = (
+            _parse_base_lang(self.source_locale) if self.source_locale else None
+        )
+        self.target_language: str | None = (
+            _parse_base_lang(self.target_locale) if self.target_locale else None
+        )
+
+        self.export_origin: str = ""
+        self.export_timestamp: str = ""
+        self.extensions: dict[str, str] = {"input_format": "xlsx"}
+
+    def extract(self) -> Iterator[ExtractItem]:
+        wb = load_workbook(self.filepath, read_only=True, data_only=True)
+        try:
+            ws = wb.active
+            if ws is None:
+                return
+
+            rows = ws.iter_rows()
+            header_row = next(rows, None)
+            if header_row is None:
+                return
+
+            headers: list[str] = [_cell_str(c).strip().lower() for c in header_row]
+            col_map: dict[str, int] = {name: i for i, name in enumerate(headers) if name}
+            has_id = "id" in col_map
+            extra_columns = [h for h in headers if h and h not in _KNOWN_COLUMNS]
+
+            for index, row in enumerate(rows):
+                cells = list(row)
+
+                def get(col: str) -> str:
+                    idx = col_map.get(col)
+                    if idx is None or idx >= len(cells):
+                        return ""
+                    return _cell_str(cells[idx])
+
+                unit_id = get("id") if has_id else ""
+                if not unit_id:
+                    unit_id = f"xlsx:{index}"
+
+                source = get("source")
+                raw_target = get("target")
+                target = raw_target if raw_target else None
+                status = _parse_status(get("status")) if get("status") else TranslationStatus.UNKNOWN
+
+                comments: list[Comment] = []
+                comment_text = get("comment").strip()
+                if comment_text:
+                    comments.append(Comment(context=comment_text))
+
+                extensions: dict[str, str] = {}
+                for col in extra_columns:
+                    val = get(col)
+                    if val:
+                        extensions[col] = val
+
+                yield unit_id, Data(
+                    source=source,
+                    target=target,
+                    status=status,
+                    comments=comments,
+                    extensions=extensions,
+                )
+        finally:
+            wb.close()
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return AsyncXlsxExtraction(self)

lokit/py.typed

@@ -0,0 +1 @@
+