lokit-python 0.1.0__cp313-cp313-win_amd64.whl

lokit/logic.py

@@ -0,0 +1,324 @@
+from __future__ import annotations
+
+import json
+import tempfile
+from collections import defaultdict
+from collections.abc import Callable, Iterator
+from dataclasses import asdict, dataclass, is_dataclass
+from difflib import SequenceMatcher
+from pathlib import Path
+from typing import Self, cast
+
+from lokit.data.structure import BaseStructure, Data
+from lokit.exporters import (
+    export_csv,
+    export_idml,
+    export_html,
+    export_json_i18n,
+    export_po,
+    export_tmx,
+    export_xliff,
+    export_xlsx,
+)
+from lokit.format_detection import LokitInputFormat, detect_format, detect_format_from_bytes
+from lokit.importers import (
+    import_csv,
+    import_idml,
+    import_html,
+    import_json_i18n,
+    import_po,
+    import_tmx,
+    import_xliff,
+    import_xlsx,
+)
+from lokit.io import load_lokit_json, load_lokit_json_bytes
+
+
+@dataclass(slots=True)
+class MatchResult:
+    unit_id: str
+    score: float
+    kind: str
+    source_equal: bool
+    tags_equal: bool
+    previous_equal: bool
+    next_equal: bool
+
+
+class Lokit:
+    def __init__(self, document: BaseStructure) -> None:
+        self.document = document
+        self._ids: list[str] = list(document.data)
+        self._positions: dict[str, int] = {
+            unit_id: index for index, unit_id in enumerate(self._ids)
+        }
+        self._source_index: dict[str, list[str]] = defaultdict(list)
+        for unit_id, unit in document.data.items():
+            self._source_index[_normalize_text(unit.source)].append(unit_id)
+
+    @classmethod
+    def parse(cls, filepath: str | Path) -> Self:
+        path = Path(filepath)
+        input_format = detect_format(path)
+        if input_format == LokitInputFormat.TMX:
+            return cls(import_tmx(str(path)))
+        if input_format == LokitInputFormat.XLIFF:
+            return cls(import_xliff(str(path)))
+        if input_format == LokitInputFormat.CSV:
+            return cls(import_csv(str(path)))
+        if input_format == LokitInputFormat.XLSX:
+            return cls(import_xlsx(str(path)))
+        if input_format == LokitInputFormat.HTML:
+            return cls(import_html(str(path)))
+        if input_format == LokitInputFormat.PO:
+            return cls(import_po(str(path)))
+        if input_format == LokitInputFormat.JSON_I18N:
+            return cls(import_json_i18n(str(path)))
+        if input_format == LokitInputFormat.IDML:
+            return cls(import_idml(str(path)))
+        return cls(load_lokit_json(path))
+
+    @classmethod
+    def parse_bytes(cls, data: bytes) -> Self:
+        input_format = detect_format_from_bytes(data)
+        if input_format == LokitInputFormat.LOKIT_JSON:
+            return cls(load_lokit_json_bytes(data))
+        suffix_map = {
+            LokitInputFormat.TMX: ".tmx",
+            LokitInputFormat.XLIFF: ".xliff",
+            LokitInputFormat.CSV: ".csv",
+            LokitInputFormat.XLSX: ".xlsx",
+            LokitInputFormat.HTML: ".html",
+            LokitInputFormat.PO: ".po",
+            LokitInputFormat.JSON_I18N: ".json",
+            LokitInputFormat.IDML: ".idml",
+        }
+        suffix = suffix_map.get(input_format, ".json")
+        with tempfile.NamedTemporaryFile(suffix=suffix) as temp:
+            temp.write(data)
+            temp.flush()
+            return cls.parse(temp.name)
+
+    @classmethod
+    def from_document(cls, document: BaseStructure) -> Self:
+        return cls(document)
+
+    def output(self, filepath: str | Path) -> None:
+        path = Path(filepath)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        suffix = path.suffix.lower()
+        if suffix == ".tmx":
+            export_tmx(self.document, path)
+        elif suffix in (".xlf", ".xliff"):
+            export_xliff(self.document, path)
+        elif suffix == ".csv":
+            export_csv(self.document, path)
+        elif suffix == ".xlsx":
+            export_xlsx(self.document, path)
+        elif suffix in (".html", ".htm"):
+            source_html = self.document.extensions.get("source_file") or self.document.extensions.get("source_html")
+            export_html(self.document, path, source_html)
+        elif suffix == ".po":
+            export_po(self.document, path)
+        elif suffix == ".json":
+            if self.document.extensions.get("input_format") == "json_i18n":
+                export_json_i18n(self.document, path)
+            else:
+                path.write_text(
+                    json.dumps(asdict(self.document), ensure_ascii=False, indent=2, default=str),
+                    encoding="utf-8",
+                )
+        elif suffix == ".idml":
+            source_idml = self.document.extensions.get("source_file") or self.document.extensions.get("source_idml")
+            if not source_idml:
+                raise ValueError("Original IDML file path not found in document extensions. Cannot export IDML without source IDML.")
+            export_idml(self.document, path, source_idml)
+        else:
+            path.write_text(
+                json.dumps(asdict(self.document), ensure_ascii=False, indent=2, default=str),
+                encoding="utf-8",
+            )
+
+    def unit(self, unit_id: str) -> Data:
+        return self.document.data[unit_id]
+
+    def all(self) -> Iterator[tuple[str, Data]]:
+        yield from self.document.data.items()
+
+    def ids(self) -> list[str]:
+        return list(self._ids)
+
+    def previous(self, unit_id: str) -> tuple[str, Data] | None:
+        index = self._positions.get(unit_id)
+        if index is None or index == 0:
+            return None
+        prev_id = self._ids[index - 1]
+        return prev_id, self.document.data[prev_id]
+
+    def next(self, unit_id: str) -> tuple[str, Data] | None:
+        index = self._positions.get(unit_id)
+        if index is None or index + 1 >= len(self._ids):
+            return None
+        next_id = self._ids[index + 1]
+        return next_id, self.document.data[next_id]
+
+    def plurals(self) -> Iterator[tuple[str, Data]]:
+        for unit_id, unit in self.document.data.items():
+            if unit.plural is not None:
+                yield unit_id, unit
+
+    def filter(
+        self,
+        predicate: Callable[[str, Data], bool],
+    ) -> list[str]:
+        return [
+            unit_id
+            for unit_id, unit in self.document.data.items()
+            if predicate(unit_id, unit)
+        ]
+
+    def where(self, key_path: str, value: object) -> list[str]:
+        expected = str(value)
+        return [
+            unit_id
+            for unit_id, unit in self.document.data.items()
+            if expected in _values_at_path(unit, key_path.split("."))
+        ]
+
+    def fuzzy_find(
+        self,
+        source: str,
+        limit: int = 10,
+        threshold: float = 0.0,
+    ) -> list[MatchResult]:
+        normalized = _normalize_text(source)
+        exact_ids = self._source_index.get(normalized, [])
+        exact_results = [
+            self._match_against_unit(source, unit_id, require_context=False, require_tags=False)
+            for unit_id in exact_ids
+        ]
+        if len(exact_results) >= limit:
+            return exact_results[:limit]
+
+        candidates: list[MatchResult] = exact_results
+        exact_set = set(exact_ids)
+        for unit_id, unit in self.document.data.items():
+            if unit_id in exact_set:
+                continue
+            score = SequenceMatcher(None, normalized, _normalize_text(unit.source)).ratio()
+            if score >= threshold:
+                candidates.append(
+                    MatchResult(
+                        unit_id=unit_id,
+                        score=score,
+                        kind="fuzzy",
+                        source_equal=False,
+                        tags_equal=False,
+                        previous_equal=False,
+                        next_equal=False,
+                    )
+                )
+
+        candidates.sort(key=lambda item: item.score, reverse=True)
+        return candidates[:limit]
+
+    def match(
+        self,
+        source: str,
+        target_unit_id: str,
+        previous_source: str | None = None,
+        next_source: str | None = None,
+        tag_signature: tuple[tuple[str, str | None], ...] | None = None,
+        require_context: bool = False,
+        require_tags: bool = False,
+    ) -> MatchResult:
+        return self._match_against_unit(
+            source,
+            target_unit_id,
+            previous_source=previous_source,
+            next_source=next_source,
+            tag_signature=tag_signature,
+            require_context=require_context,
+            require_tags=require_tags,
+        )
+
+    def _match_against_unit(
+        self,
+        source: str,
+        unit_id: str,
+        previous_source: str | None = None,
+        next_source: str | None = None,
+        tag_signature: tuple[tuple[str, str | None], ...] | None = None,
+        require_context: bool = False,
+        require_tags: bool = False,
+    ) -> MatchResult:
+        unit = self.document.data[unit_id]
+        source_equal = _normalize_text(source) == _normalize_text(unit.source)
+        score = SequenceMatcher(None, _normalize_text(source), _normalize_text(unit.source)).ratio()
+        tags_equal = (not require_tags) or (
+            tag_signature is not None and tag_signature == _tags_signature(unit)
+        )
+        previous_equal = (not require_context) or (
+            previous_source is not None
+            and _normalize_text(previous_source)
+            == _normalize_text(_context_text(unit.previous_context) or "")
+        )
+        next_equal = (not require_context) or (
+            next_source is not None
+            and _normalize_text(next_source)
+            == _normalize_text(_context_text(unit.next_context) or "")
+        )
+        checked_ice_context = require_context or require_tags
+        is_ice = checked_ice_context and source_equal and tags_equal and previous_equal and next_equal
+        return MatchResult(
+            unit_id=unit_id,
+            score=1.0 if is_ice else score,
+            kind="ice" if is_ice else ("exact" if source_equal else "fuzzy"),
+            source_equal=source_equal,
+            tags_equal=tags_equal,
+            previous_equal=previous_equal,
+            next_equal=next_equal,
+        )
+
+
+def _normalize_text(value: str) -> str:
+    return " ".join(value.casefold().split())
+
+
+def _values_at_path(root: object, path: list[str]) -> list[str]:
+    if not path:
+        return [str(root)] if root is not None else []
+    head = path[0]
+    tail = path[1:]
+
+    if isinstance(root, list):
+        values: list[str] = []
+        for item in root:
+            values.extend(_values_at_path(item, path))
+        return values
+
+    if isinstance(root, dict):
+        if head not in root:
+            return []
+        return _values_at_path(root[head], tail)
+
+    if is_dataclass(root):
+        if not hasattr(root, head):
+            return []
+        return _values_at_path(getattr(root, head), tail)
+
+    return []
+
+
+def _tags_signature(unit: Data) -> tuple[tuple[str, str | None], ...]:
+    if unit.tags is None:
+        return ()
+    ordered = sorted(unit.tags.source_tag_map.values(), key=lambda item: item.order)
+    return tuple((tag.type.value, tag.pair_id) for tag in ordered)
+
+
+def _context_text(context: object) -> str | None:
+    if context is None:
+        return None
+    source = getattr(context, "source", None)
+    return cast(str | None, source)

lokit/parsers/__init__.py

@@ -0,0 +1 @@
+

lokit/parsers/csv/__init__.py

@@ -0,0 +1 @@
+

lokit/parsers/csv/extraction.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import asyncio
+import csv
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncIterator, Iterator, Optional
+
+from lokit.data.structure import Comment, Data, TranslationStatus
+
+ExtractItem = tuple[str, Data]
+
+_KNOWN_COLUMNS = frozenset({"id", "source", "target", "status", "comment"})
+
+
+def _parse_base_lang(locale: str) -> str:
+    return locale.replace("_", "-").split("-")[0].lower()
+
+
+def _parse_status(value: str) -> TranslationStatus:
+    normalized = value.strip().lower()
+    try:
+        return TranslationStatus(normalized)
+    except ValueError:
+        return TranslationStatus.UNKNOWN
+
+
+def _infer_locales_from_filename(filepath: str) -> tuple[str, str | None]:
+    stem = Path(filepath).stem
+    if "-" in stem:
+        parts = stem.split("-")
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        if len(parts) == 4:
+            return f"{parts[0]}-{parts[1]}", f"{parts[2]}-{parts[3]}"
+    if "_" in stem:
+        parts = stem.split("_")
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        if len(parts) == 4:
+            return f"{parts[0]}_{parts[1]}", f"{parts[2]}_{parts[3]}"
+    return "", None
+
+
+@dataclass(slots=True)
+class _AsyncExtractionResult:
+    item: Optional[ExtractItem] = None
+    error: Optional[BaseException] = None
+    done: bool = False
+
+
+class AsyncCsvExtraction:
+    def __init__(self, extractor: CsvExtractor) -> None:
+        self._extractor = extractor
+        self._queue: asyncio.Queue[_AsyncExtractionResult] = asyncio.Queue()
+        self._producer: asyncio.Task[None] | None = None
+
+    def __aiter__(self) -> AsyncCsvExtraction:
+        return self
+
+    async def __anext__(self) -> ExtractItem:
+        if self._producer is None:
+            self._start()
+
+        result = await self._queue.get()
+        if result.done:
+            await self._finish()
+            raise StopAsyncIteration
+        if result.error is not None:
+            await self._finish()
+            raise result.error
+        if result.item is None:
+            await self._finish()
+            raise StopAsyncIteration
+        return result.item
+
+    def _start(self) -> None:
+        loop = asyncio.get_running_loop()
+
+        def produce() -> None:
+            try:
+                for item in self._extractor.extract():
+                    loop.call_soon_threadsafe(
+                        self._queue.put_nowait,
+                        _AsyncExtractionResult(item=item),
+                    )
+            except BaseException as exc:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(error=exc),
+                )
+            finally:
+                loop.call_soon_threadsafe(
+                    self._queue.put_nowait,
+                    _AsyncExtractionResult(done=True),
+                )
+
+        self._producer = asyncio.create_task(asyncio.to_thread(produce))
+
+    async def _finish(self) -> None:
+        if self._producer is not None:
+            await self._producer
+
+
+class CsvExtractor:
+    def __init__(
+        self,
+        filepath: str,
+        source_locale: str = "",
+        target_locale: str | None = None,
+    ) -> None:
+        self.filepath: str = filepath
+
+        if source_locale:
+            self.source_locale: str = source_locale
+            self.target_locale: str | None = target_locale
+        else:
+            inferred_source, inferred_target = _infer_locales_from_filename(filepath)
+            self.source_locale = inferred_source
+            self.target_locale = target_locale or inferred_target
+
+        self.source_language: str | None = (
+            _parse_base_lang(self.source_locale) if self.source_locale else None
+        )
+        self.target_language: str | None = (
+            _parse_base_lang(self.target_locale) if self.target_locale else None
+        )
+
+        self.export_origin: str = ""
+        self.export_timestamp: str = ""
+        self.extensions: dict[str, str] = {"input_format": "csv"}
+
+    def extract(self) -> Iterator[ExtractItem]:
+        with open(self.filepath, newline="", encoding="utf-8") as fh:
+            reader = csv.DictReader(fh)
+            fieldnames: list[str] = list(reader.fieldnames or [])
+            has_id = "id" in fieldnames
+            extra_columns = [c for c in fieldnames if c not in _KNOWN_COLUMNS]
+
+            for index, row in enumerate(reader):
+                unit_id = row["id"] if has_id and row.get("id") else f"csv:{index}"
+                source = row.get("source", "")
+                target = row.get("target") or None
+                status = _parse_status(row["status"]) if row.get("status") else TranslationStatus.UNKNOWN
+
+                comments: list[Comment] = []
+                comment_text = row.get("comment", "").strip()
+                if comment_text:
+                    comments.append(Comment(context=comment_text))
+
+                extensions: dict[str, str] = {
+                    col: row[col] for col in extra_columns if row.get(col)
+                }
+
+                yield unit_id, Data(
+                    source=source,
+                    target=target,
+                    status=status,
+                    comments=comments,
+                    extensions=extensions,
+                )
+
+    def extract_async(self) -> AsyncIterator[ExtractItem]:
+        return AsyncCsvExtraction(self)

lokit/parsers/html/__init__.py

@@ -0,0 +1,3 @@
+from lokit.parsers.html.extraction import HtmlExtractor
+
+__all__ = ["HtmlExtractor"]