lokit-python 0.1.0__cp313-cp313-win_amd64.whl

lokit/exporters/json_i18n.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+from lokit.data.structure import BaseStructure
+
+
+def export_json_i18n(
+    document: BaseStructure,
+    filepath: str | Path,
+    nested: bool = True,
+) -> None:
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    output: dict[str, Any] = {}
+    for key, unit in document.data.items():
+        value = unit.target if unit.target is not None else unit.source
+        if nested:
+            _set_nested(output, key, value)
+        else:
+            output[key] = value
+
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(output, f, ensure_ascii=False, indent=2)
+        f.write("\n")
+
+
+async def export_json_i18n_async(
+    document: BaseStructure,
+    filepath: str | Path,
+    nested: bool = True,
+) -> None:
+    await asyncio.to_thread(export_json_i18n, document, filepath, nested)
+
+
+def _set_nested(obj: dict[str, Any], dot_key: str, value: str) -> None:
+    parts = dot_key.split(".")
+    current = obj
+    for part in parts[:-1]:
+        if part not in current or not isinstance(current[part], dict):
+            current[part] = {}
+        current = current[part]
+    current[parts[-1]] = value

lokit/exporters/po.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+import polib
+
+from lokit.data.structure import BaseStructure, Data, TranslationStatus
+
+_PLURAL_SUFFIX_PATTERN = "["
+
+
+def export_po(document: BaseStructure, filepath: str | Path) -> None:
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    po: Any = polib.POFile()
+    po.metadata = _build_metadata(document)
+
+    plural_groups: dict[str, list[tuple[str, Data]]] = defaultdict(list)
+    singular_units: list[tuple[str, Data]] = []
+
+    for unit_id, unit in document.data.items():
+        if _PLURAL_SUFFIX_PATTERN in unit_id and unit.plural is not None:
+            base_id = unit_id[: unit_id.index(_PLURAL_SUFFIX_PATTERN)]
+            plural_groups[base_id].append((unit_id, unit))
+        elif unit.plural is not None:
+            plural_groups[unit_id].append((unit_id, unit))
+        else:
+            singular_units.append((unit_id, unit))
+
+    for unit_id, unit in singular_units:
+        po.append(_build_entry(unit_id, unit))
+
+    for base_id, forms in plural_groups.items():
+        po.append(_build_plural_entry(base_id, forms))
+
+    po.save(str(path))
+
+
+async def export_po_async(document: BaseStructure, filepath: str | Path) -> None:
+    await asyncio.to_thread(export_po, document, filepath)
+
+
+def _build_metadata(document: BaseStructure) -> dict[str, str]:
+    meta: dict[str, str] = {
+        "Content-Type": "text/plain; charset=UTF-8",
+        "Content-Transfer-Encoding": "8bit",
+    }
+    if document.target_locale:
+        meta["Language"] = document.target_locale
+    if document.export_origin:
+        meta["X-Generator"] = document.export_origin
+    return meta
+
+
+def _parse_unit_id(unit_id: str) -> tuple[str | None, str]:
+    if "\x04" in unit_id:
+        ctx, msgid = unit_id.split("\x04", 1)
+        return ctx, msgid
+    return None, unit_id
+
+
+def _build_entry(unit_id: str, unit: Data) -> Any:
+    msgctxt, msgid = _parse_unit_id(unit_id)
+    context_key = _find_context_key(unit)
+    if context_key is not None:
+        msgctxt = context_key
+
+    entry: Any = polib.POEntry(
+        msgid=msgid,
+        msgstr=unit.target or "",
+        msgctxt=msgctxt,
+    )
+
+    _apply_comments(entry, unit)
+    _apply_flags(entry, unit)
+    _apply_occurrences(entry, unit)
+    return entry
+
+
+def _build_plural_entry(
+    base_id: str, forms: list[tuple[str, Data]]
+) -> Any:
+    msgctxt, msgid = _parse_unit_id(base_id)
+    base_unit = forms[0][1]
+    context_key = _find_context_key(base_unit)
+    if context_key is not None:
+        msgctxt = context_key
+
+    variant = base_unit.plural.variant if base_unit.plural else msgid
+
+    msgstr_plural: dict[int, str] = {}
+    msgstr_plural[0] = base_unit.target or ""
+
+    for uid, unit in forms:
+        if _PLURAL_SUFFIX_PATTERN in uid:
+            idx_str = uid[uid.index(_PLURAL_SUFFIX_PATTERN) + 1 : uid.rindex("]")]
+            idx = int(idx_str)
+            msgstr_plural[idx] = unit.target or ""
+
+    entry: Any = polib.POEntry(
+        msgid=msgid,
+        msgid_plural=variant,
+        msgstr_plural=msgstr_plural,
+        msgctxt=msgctxt,
+    )
+
+    _apply_comments(entry, base_unit)
+    _apply_flags(entry, base_unit)
+    _apply_occurrences(entry, base_unit)
+    return entry
+
+
+def _find_context_key(unit: Data) -> str | None:
+    for comment in unit.comments:
+        if comment.context_key is not None:
+            return comment.context_key
+    return None
+
+
+def _apply_comments(entry: Any, unit: Data) -> None:
+    translator_comments: list[str] = []
+    extracted_comments: list[str] = []
+    for i, comment in enumerate(unit.comments):
+        if i == 0 and comment.context_key is not None:
+            translator_comments.append(comment.context)
+        elif i == 0:
+            translator_comments.append(comment.context)
+        else:
+            extracted_comments.append(comment.context)
+    if translator_comments:
+        entry.comment = "\n".join(translator_comments)
+    if extracted_comments:
+        entry.tcomment = "\n".join(extracted_comments)
+
+
+def _apply_flags(entry: Any, unit: Data) -> None:
+    flags: list[str] = []
+    if unit.status == TranslationStatus.DRAFT:
+        flags.append("fuzzy")
+    extra = unit.extensions.get("flags")
+    if extra:
+        flags.extend(f.strip() for f in extra.split(","))
+    entry.flags = flags
+
+
+def _apply_occurrences(entry: Any, unit: Data) -> None:
+    refs = unit.extensions.get("references")
+    if not refs:
+        return
+    occurrences: list[tuple[str, str]] = []
+    for ref in refs.split(","):
+        ref = ref.strip()
+        if ":" in ref:
+            path, line = ref.rsplit(":", 1)
+            occurrences.append((path, line))
+        else:
+            occurrences.append((ref, ""))
+    entry.occurrences = occurrences

lokit/exporters/tmx.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from lxml import etree
+from lxml.etree import _Element
+
+from lokit.data.structure import (
+    BaseStructure,
+    CodePart,
+    Data,
+    SegmentPart,
+    TextPart,
+    TranslationStatus,
+)
+from lokit.data.tag_types import TieData, TieType
+from lokit.io.json import load_lokit_json
+
+
+def export_tmx(document: BaseStructure, filepath: str | Path) -> None:
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("wb") as stream:
+        with etree.xmlfile(stream, encoding="UTF-8") as xf:
+            xf.write_declaration()
+            with xf.element("tmx", version="1.4"):
+                xf.write(_build_header(document))
+                with xf.element("body"):
+                    for unit_id, unit in document.data.items():
+                        xf.write(_build_tu(unit_id, unit, document))
+
+
+def export_tmx_from_json(source_json: str | Path, target_tmx: str | Path) -> None:
+    export_tmx(load_lokit_json(source_json), target_tmx)
+
+
+def _build_header(document: BaseStructure) -> _Element:
+    header = etree.Element(
+        "header",
+        {
+            "creationtool": document.extensions.get("tool_name", "lokit"),
+            "creationtoolversion": document.extensions.get("tool_version", "0.1"),
+            "segtype": document.extensions.get("segmentation", "sentence"),
+            "o-tmf": document.extensions.get("translation_memory_format", "lokit"),
+            "adminlang": document.extensions.get("admin_locale", document.source_locale),
+            "srclang": document.source_locale,
+            "datatype": document.extensions.get("data_type", "text"),
+        },
+    )
+    if document.export_timestamp:
+        header.attrib["creationdate"] = document.export_timestamp
+    for key, value in document.extensions.items():
+        if key.startswith("property."):
+            prop = etree.SubElement(header, "prop", type=_property_type(key))
+            prop.text = value
+    return header
+
+
+def _build_tu(unit_id: str, unit: Data, document: BaseStructure) -> _Element:
+    attrs: dict[str, str] = {"tuid": unit_id}
+    if unit.meta.created:
+        attrs["creationdate"] = unit.meta.created
+    if unit.meta.updated:
+        attrs["changedate"] = unit.meta.updated
+    creator_id = _first_creator_id(unit)
+    if creator_id:
+        attrs["creationid"] = creator_id
+    change_id = unit.meta.extensions.get("change_id")
+    if change_id:
+        attrs["changeid"] = change_id
+    if unit.meta.usage_count is not None:
+        attrs["usagecount"] = str(unit.meta.usage_count)
+
+    tu = etree.Element("tu", attrs)
+    _append_unit_properties(tu, unit)
+    _append_comments(tu, unit)
+    tu.append(
+        _build_tuv(
+            document.source_locale,
+            unit.source,
+            unit.tags.source_parts if unit.tags else [],
+            unit.tags.source_tag_map if unit.tags else {},
+        )
+    )
+    if document.target_locale is not None and unit.target is not None:
+        tu.append(
+            _build_tuv(
+                document.target_locale,
+                unit.target,
+                unit.tags.target_parts if unit.tags else [],
+                unit.tags.target_tag_map if unit.tags else {},
+            )
+        )
+    return tu
+
+
+def _append_unit_properties(tu: _Element, unit: Data) -> None:
+    if unit.status != TranslationStatus.UNKNOWN:
+        prop = etree.SubElement(tu, "prop", type="x-status")
+        prop.text = unit.status.value
+
+    if unit.previous_context is not None:
+        _append_prop_if_present(tu, "x-previous-id", unit.previous_context.unit_id)
+        _append_prop_if_present(tu, "x-previous-source-text", unit.previous_context.source)
+        _append_prop_if_present(tu, "x-previous-target-text", unit.previous_context.target)
+
+    if unit.next_context is not None:
+        _append_prop_if_present(tu, "x-next-id", unit.next_context.unit_id)
+        _append_prop_if_present(tu, "x-next-source-text", unit.next_context.source)
+        _append_prop_if_present(tu, "x-next-target-text", unit.next_context.target)
+
+    project = _first_project(unit)
+    if project:
+        _append_prop_if_present(tu, "x-project", project)
+
+    system = _first_system(unit)
+    if system:
+        _append_prop_if_present(tu, "x-system", system)
+
+    for key, value in unit.extensions.items():
+        if key.startswith("property."):
+            _append_prop_if_present(tu, _property_type(key), value)
+
+
+def _append_comments(tu: _Element, unit: Data) -> None:
+    for comment in unit.comments:
+        if not comment.context:
+            continue
+        note = etree.SubElement(tu, "note")
+        note.text = comment.context
+
+
+def _build_tuv(
+    locale: str,
+    text: str,
+    parts: list[SegmentPart],
+    tag_map: dict[str, TieData],
+) -> _Element:
+    tuv = etree.Element("tuv", {"{http://www.w3.org/XML/1998/namespace}lang": locale})
+    tuv.append(_build_seg(text, parts, tag_map))
+    return tuv
+
+
+def _build_seg(
+    text: str,
+    parts: list[SegmentPart],
+    tag_map: dict[str, TieData],
+) -> _Element:
+    seg = etree.Element("seg")
+    effective_parts = parts if parts else [TextPart(text)]
+    pair_numbers = _pair_numbers(tag_map)
+    last_child: _Element | None = None
+
+    for part in effective_parts:
+        if isinstance(part, TextPart):
+            last_child = _append_text(seg, last_child, part.value)
+        elif isinstance(part, CodePart):
+            code = tag_map.get(part.ref)
+            if code is None:
+                last_child = _append_text(seg, last_child, "")
+            else:
+                child = _build_code_element(code, pair_numbers)
+                seg.append(child)
+                last_child = child
+
+    return seg
+
+
+def _build_code_element(code: TieData, pair_numbers: dict[str, str]) -> _Element:
+    if _is_open(code.type):
+        element = etree.Element("bpt", i=_pair_number(code, pair_numbers), type=code.type.value)
+        element.text = f"<lokit id=\"{code.pair_id or code.id}\">"
+        return element
+    if _is_close(code.type):
+        element = etree.Element("ept", i=_pair_number(code, pair_numbers))
+        element.text = "</lokit>"
+        return element
+    element = etree.Element("ph", x=str(code.order), type=code.type.value)
+    element.text = f"<lokit id=\"{code.id}\"/>"
+    return element
+
+
+def _append_text(seg: _Element, last_child: _Element | None, value: str) -> _Element | None:
+    if last_child is None:
+        seg.text = (seg.text or "") + value
+    else:
+        last_child.tail = (last_child.tail or "") + value
+    return last_child
+
+
+def _pair_numbers(tag_map: dict[str, TieData]) -> dict[str, str]:
+    pair_ids: dict[str, str] = {}
+    index = 0
+    for code in sorted(tag_map.values(), key=lambda item: item.order):
+        if code.pair_id is not None and code.pair_id not in pair_ids:
+            pair_ids[code.pair_id] = str(index)
+            index += 1
+    return pair_ids
+
+
+def _pair_number(code: TieData, pair_numbers: dict[str, str]) -> str:
+    if code.pair_id is None:
+        return str(code.order)
+    return pair_numbers.get(code.pair_id, str(code.order))
+
+
+def _is_open(tie_type: TieType) -> bool:
+    return tie_type.value.endswith(".open")
+
+
+def _is_close(tie_type: TieType) -> bool:
+    return tie_type.value.endswith(".close")
+
+
+def _append_prop_if_present(tu: _Element, prop_type: str, value: str | None) -> None:
+    if value is None or value == "":
+        return
+    prop = etree.SubElement(tu, "prop", type=prop_type)
+    prop.text = value
+
+
+def _first_creator_id(unit: Data) -> str | None:
+    for comment in unit.comments:
+        if comment.origin is not None and comment.origin.creator_id:
+            return comment.origin.creator_id
+    return None
+
+
+def _first_project(unit: Data) -> str | None:
+    for comment in unit.comments:
+        if comment.origin is not None and comment.origin.project:
+            return comment.origin.project
+    return None
+
+
+def _first_system(unit: Data) -> str | None:
+    for comment in unit.comments:
+        if comment.origin is not None and comment.origin.system:
+            return comment.origin.system
+    return None
+
+
+def _property_type(key: str) -> str:
+    prefix = "property."
+    if key.startswith(prefix):
+        return key[len(prefix) :]
+    return key

lokit/exporters/xliff.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import asyncio
+from collections import OrderedDict
+from pathlib import Path
+from typing import cast
+
+from lxml import etree
+from lxml.etree import _Element
+
+from lokit.data.structure import BaseStructure, CodePart, Data, SegmentPart, TextPart
+from lokit.data.tag_types import TieData, TieType
+from lokit.io.json import load_lokit_json
+
+XLIFF_NS = "urn:oasis:names:tc:xliff:document:1.2"
+NSMAP = cast(dict[str, str], {None: XLIFF_NS})
+
+
+def export_xliff(document: BaseStructure, filepath: str | Path) -> None:
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("wb") as stream:
+        with etree.xmlfile(stream, encoding="UTF-8") as xf:
+            xf.write_declaration()
+            with xf.element(f"{{{XLIFF_NS}}}xliff", nsmap=NSMAP, version="1.2"):
+                for resource_key, units in _group_by_resource(document).items():
+                    attrs = {
+                        "original": resource_key or "lokit",
+                        "datatype": _first_extension(units, "data_type", "plaintext"),
+                        "source-language": document.source_locale,
+                    }
+                    if document.target_locale is not None:
+                        attrs["target-language"] = document.target_locale
+                    with xf.element(f"{{{XLIFF_NS}}}file", attrs):
+                        xf.write(etree.Element(f"{{{XLIFF_NS}}}header"))
+                        with xf.element(f"{{{XLIFF_NS}}}body"):
+                            for unit_id, unit in units:
+                                xf.write(_build_trans_unit(unit_id, unit))
+
+
+def export_xliff_from_json(source_json: str | Path, target_xliff: str | Path) -> None:
+    export_xliff(load_lokit_json(source_json), target_xliff)
+
+
+async def export_xliff_async(document: BaseStructure, filepath: str | Path) -> None:
+    await asyncio.to_thread(export_xliff, document, filepath)
+
+
+async def export_xliff_from_json_async(
+    source_json: str | Path, target_xliff: str | Path
+) -> None:
+    await asyncio.to_thread(export_xliff_from_json, source_json, target_xliff)
+
+
+def _group_by_resource(
+    document: BaseStructure,
+) -> OrderedDict[str, list[tuple[str, Data]]]:
+    groups: OrderedDict[str, list[tuple[str, Data]]] = OrderedDict()
+    for unit_id, unit in document.data.items():
+        resource = unit.extensions.get("resource", "lokit")
+        groups.setdefault(resource, []).append((unit_id, unit))
+    return groups
+
+
+def _build_trans_unit(unit_id: str, unit: Data) -> _Element:
+    attrs = {"id": unit.extensions.get("unit_id", unit_id)}
+    space = unit.extensions.get("space")
+    if space:
+        attrs["{http://www.w3.org/XML/1998/namespace}space"] = space
+    trans_unit = etree.Element(f"{{{XLIFF_NS}}}trans-unit", attrs)
+    trans_unit.append(
+        _build_segment(
+            "source",
+            unit.source,
+            unit.tags.source_parts if unit.tags else [],
+            unit.tags.source_tag_map if unit.tags else {},
+        )
+    )
+    if unit.target is not None:
+        target = _build_segment(
+            "target",
+            unit.target,
+            unit.tags.target_parts if unit.tags else [],
+            unit.tags.target_tag_map if unit.tags else {},
+        )
+        trans_unit.append(target)
+    for comment in unit.comments:
+        if comment.context:
+            note = etree.SubElement(trans_unit, f"{{{XLIFF_NS}}}note")
+            note.text = comment.context
+    return trans_unit
+
+
+def _build_segment(
+    name: str,
+    text: str,
+    parts: list[SegmentPart],
+    tag_map: dict[str, TieData],
+) -> _Element:
+    element = etree.Element(f"{{{XLIFF_NS}}}{name}")
+    effective_parts = parts if parts else [TextPart(text)]
+    last_child: _Element | None = None
+    for part in effective_parts:
+        if isinstance(part, TextPart):
+            last_child = _append_text(element, last_child, part.value)
+        elif isinstance(part, CodePart):
+            code = tag_map.get(part.ref)
+            if code is not None:
+                child = _build_code(code)
+                element.append(child)
+                last_child = child
+    return element
+
+
+def _build_code(code: TieData) -> _Element:
+    if _is_open(code.type):
+        element = etree.Element(f"{{{XLIFF_NS}}}bx", id=code.id)
+    elif _is_close(code.type):
+        element = etree.Element(f"{{{XLIFF_NS}}}ex", id=code.id)
+    else:
+        element = etree.Element(f"{{{XLIFF_NS}}}x", id=code.id)
+    if code.pair_id is not None:
+        element.attrib["rid"] = code.pair_id
+    return element
+
+
+def _append_text(
+    parent: _Element, last_child: _Element | None, value: str
+) -> _Element | None:
+    if last_child is None:
+        parent.text = (parent.text or "") + value
+    else:
+        last_child.tail = (last_child.tail or "") + value
+    return last_child
+
+
+def _is_open(tie_type: TieType) -> bool:
+    return tie_type.value.endswith(".open")
+
+
+def _is_close(tie_type: TieType) -> bool:
+    return tie_type.value.endswith(".close")
+
+
+def _first_extension(
+    units: list[tuple[str, Data]], key: str, fallback: str
+) -> str:
+    for _, unit in units:
+        value = unit.extensions.get(key)
+        if value:
+            return value
+    return fallback

lokit/exporters/xlsx.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from openpyxl import Workbook
+
+from lokit.data.structure import BaseStructure, TranslationStatus
+
+_HEADERS = ["id", "source", "target", "status", "comment"]
+
+
+def export_xlsx(document: BaseStructure, filepath: str | Path) -> None:
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    wb = Workbook(write_only=True)
+    ws = wb.create_sheet()
+
+    ws.append(_HEADERS)
+
+    for unit_id, unit in document.data.items():
+        comment = "; ".join(c.context for c in unit.comments if c.context)
+        status = unit.status.value if unit.status != TranslationStatus.UNKNOWN else ""
+
+        ws.append([
+            unit_id,
+            unit.source,
+            unit.target or "",
+            status,
+            comment,
+        ])
+
+    wb.save(str(path))
+    wb.close()
+
+
+async def export_xlsx_async(document: BaseStructure, filepath: str | Path) -> None:
+    await asyncio.to_thread(export_xlsx, document, filepath)