contextweave 0.3.0__tar.gz → 0.3.1__tar.gz

{contextweave-0.3.0 → contextweave-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: contextweave
-Version: 0.3.0
+Version: 0.3.1
 Summary: ContextWeave: context-aware document translation with glossary management
 Project-URL: Homepage, https://github.com/bot-32142/ContextWeave
 Project-URL: Documentation, https://bot-32142.github.io/ContextWeave/

contextweave-0.3.1/context_aware_translation/core/galgame_document_handler.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+from context_aware_translation.core.cancellation import raise_if_cancelled
+
+if TYPE_CHECKING:
+    from context_aware_translation.core.context_manager import TranslationContextManager
+    from context_aware_translation.core.progress import ProgressCallback
+
+
+class GalgameDocumentHandler:
+    """DocumentTypeHandler for galgame documents using the generic text pipeline."""
+
+    def __init__(
+        self,
+        *,
+        concurrency: int = 5,
+        batch_size: int = 0,
+        max_tokens_per_batch: int = 2000,
+    ) -> None:
+        self._concurrency = max(1, concurrency)
+        self._batch_size = batch_size
+        self._max_tokens_per_batch = max(1, max_tokens_per_batch)
+
+    def add_text(
+        self,
+        text: str,
+        max_token_size_per_chunk: int,
+        document_id: int,
+        manager: TranslationContextManager,
+    ) -> int:
+        """Let the normal semantic chunker group galgame text."""
+        return manager.add_text(text, max_token_size_per_chunk, document_id)
+
+    async def translate_chunks(
+        self,
+        document_ids: list[int],
+        manager: TranslationContextManager,
+        force: bool = False,
+        source_ids: list[int] | None = None,
+        cancel_check: Callable[[], bool] | None = None,
+        progress_callback: ProgressCallback | None = None,
+    ) -> None:
+        """Delegate to generic text translation with configured batching."""
+        _ = source_ids
+        raise_if_cancelled(cancel_check)
+        await manager.translate_chunks(
+            concurrency=self._concurrency,
+            batch_size=self._batch_size,
+            max_tokens_per_batch=self._max_tokens_per_batch,
+            document_ids=document_ids,
+            force=force,
+            cancel_check=cancel_check,
+            progress_callback=progress_callback,
+        )
+
+    def get_translated_lines(
+        self,
+        document_id: int,
+        manager: TranslationContextManager,
+    ) -> list[str]:
+        """Use the normal chunk concatenation and newline split behavior."""
+        return manager.get_translated_lines(document_id)

{contextweave-0.3.0 → contextweave-0.3.1}/context_aware_translation/documents/galgame.py

@@ -1148,11 +1148,7 @@ class GalgameDocument(Document):
         return 0
 
     def get_text(self) -> str:
-        units: list[TranslationUnit] = []
-        for source in self._ordered_sources():
-            relative_path, text_content, adapter = _source_adapter(source)
-            units.extend(adapter.extract_units(relative_path, text_content))
-        return serialize_translation_units(units)
+        return "\n".join(unit.text for unit in self._all_translation_units())
 
     def get_glossary_seeds(self) -> tuple[GalgameGlossarySeed, ...]:
         seeds: list[GalgameGlossarySeed] = []
@@ -1180,7 +1176,7 @@ class GalgameDocument(Document):
     ) -> int:
         _ = progress_callback
         raise_if_cancelled(cancel_check)
-        self._translated_lines = [_coerce_export_line_to_translation(line) for line in lines]
+        self._translated_lines = _expand_galgame_export_lines(lines, self._all_translation_units())
         return len(lines)
 
     async def reembed(
@@ -1251,6 +1247,13 @@ class GalgameDocument(Document):
         sources = cast("list[dict[str, object]]", self.repo.get_document_sources(self.document_id))
         return sorted(sources, key=_source_sequence_number)
 
+    def _all_translation_units(self) -> list[TranslationUnit]:
+        units: list[TranslationUnit] = []
+        for source in self._ordered_sources():
+            relative_path, text_content, adapter = _source_adapter(source)
+            units.extend(adapter.extract_units(relative_path, text_content))
+        return units
+
     def _source_translation_batches(
         self,
     ) -> list[tuple[dict[str, object], GalgameAdapter, list[TranslationUnit], dict[str, str]]]:
@@ -1623,6 +1626,39 @@ def _safe_output_path(output_folder: Path, relative_path: str) -> Path:
     return output_folder.joinpath(*relative.parts)
 
 
+def _expand_galgame_export_lines(lines: list[str], units: list[TranslationUnit]) -> list[str]:
+    translated_lines: list[str] = []
+    for line in lines:
+        translated_lines.extend(_split_galgame_chunk_translation(_coerce_export_line_to_translation(line)))
+
+    expanded: list[str] = []
+    unit_index = 0
+    line_index = 0
+    while unit_index < len(units):
+        source_line_count = _galgame_unit_line_count(units[unit_index].text)
+        next_line_index = line_index + source_line_count
+        if next_line_index > len(translated_lines):
+            raise ValueError("Translated galgame line stream is shorter than the source unit stream.")
+        expanded.append("\n".join(translated_lines[line_index:next_line_index]))
+        unit_index += 1
+        line_index = next_line_index
+
+    extra_lines = [line for line in translated_lines[line_index:] if line]
+    if extra_lines:
+        raise ValueError("Translated galgame line stream is longer than the source unit stream.")
+    return expanded
+
+
+def _split_galgame_chunk_translation(text: str) -> list[str]:
+    normalized = text.replace("\r\n", "\n").replace("\r", "\n")
+    return [decode_compressed_line(line) for line in normalized.split("\n")]
+
+
+def _galgame_unit_line_count(text: str) -> int:
+    normalized = text.replace("\r\n", "\n").replace("\r", "\n")
+    return max(1, len(normalized.split("\n")))
+
+
 def _coerce_export_line_to_translation(line: str) -> str:
     decoded_line = decode_compressed_line(line)
     if not decoded_line.strip().startswith("{"):

{contextweave-0.3.0 → contextweave-0.3.1}/context_aware_translation/workflow/ops/export_ops.py

@@ -26,8 +26,6 @@ def get_lines_with_original_fallback(workflow: WorkflowContext, document: Docume
 
     if document.document_type == "manga":
         return [chunk.translation if chunk.is_translated and chunk.translation is not None else "" for chunk in chunks]
-    if document.document_type == "galgame":
-        return [chunk.translation if chunk.is_translated and chunk.translation is not None else chunk.text for chunk in chunks]
 
     merged_chunks = [
         chunk.translation if chunk.is_translated and chunk.translation is not None else chunk.text for chunk in chunks

{contextweave-0.3.0 → contextweave-0.3.1}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["context_aware_translation"]
 
 [project]
 name = "contextweave"
-version = "0.3.0"
+version = "0.3.1"
 description = "ContextWeave: context-aware document translation with glossary management"
 readme = "README.md"
 license = "GPL-3.0-only"

contextweave-0.3.1/tests/core/test_galgame_document_handler.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from context_aware_translation.core.galgame_document_handler import GalgameDocumentHandler
+
+
+class DummyManager:
+    def __init__(self) -> None:
+        self.add_text_calls: list[tuple[str, int, int]] = []
+        self.translate_calls: list[dict[str, object]] = []
+        self.translated_lines = ["你好", "再见"]
+
+    def add_text(self, text: str, max_token_size_per_chunk: int, document_id: int) -> int:
+        self.add_text_calls.append((text, max_token_size_per_chunk, document_id))
+        return 42
+
+    async def translate_chunks(self, **kwargs):  # noqa: ANN003
+        self.translate_calls.append(kwargs)
+
+    def get_translated_lines(self, document_id: int) -> list[str]:
+        assert document_id == 7
+        return self.translated_lines
+
+
+def test_galgame_handler_delegates_add_text_to_generic_pipeline() -> None:
+    manager = DummyManager()
+    handler = GalgameDocumentHandler()
+
+    last_chunk_id = handler.add_text("こんにちは\nまたね", 500, 7, manager)
+
+    assert last_chunk_id == 42
+    assert manager.add_text_calls == [("こんにちは\nまたね", 500, 7)]
+
+
+def test_galgame_handler_delegates_get_translated_lines_to_generic_pipeline() -> None:
+    manager = DummyManager()
+    handler = GalgameDocumentHandler()
+
+    assert handler.get_translated_lines(7, manager) == ["你好", "再见"]
+
+
+async def test_galgame_handler_delegates_translation_with_configured_batching() -> None:
+    manager = DummyManager()
+    handler = GalgameDocumentHandler(concurrency=3, batch_size=0, max_tokens_per_batch=1234)
+
+    await handler.translate_chunks([7], manager, force=True)
+
+    assert manager.translate_calls == [
+        {
+            "concurrency": 3,
+            "batch_size": 0,
+            "max_tokens_per_batch": 1234,
+            "document_ids": [7],
+            "force": True,
+            "cancel_check": None,
+            "progress_callback": None,
+        }
+    ]

{contextweave-0.3.0 → contextweave-0.3.1}/tests/documents/test_galgame.py

@@ -20,7 +20,6 @@ from context_aware_translation.documents.galgame import (
     TranslatorPlusPlusTransAdapter,
     VnTextJsonAdapter,
     WolfRpgXlsxAdapter,
-    deserialize_translation_unit_stream,
     get_galgame_bridge_formats,
     get_galgame_external_tool_specs,
     inspect_galgame_external_helpers,
@@ -462,8 +461,7 @@ async def test_galgame_document_preserve_export_uses_imported_json_adapter_mime_
     sources = repo.get_document_sources(row["document_id"])
     assert sources[0]["mime_type"] == VnTextJsonAdapter.mime_type
 
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.text, unit.speaker) for unit in units] == [("こんにちは", "Alice")]
+    assert document.get_text().splitlines() == ["こんにちは"]
 
     await document.set_text(["你好"])
     output_folder = tmp_path / "out"
@@ -492,8 +490,7 @@ async def test_galgame_document_preserve_export_writes_imported_xlsx_adapter(tmp
     sources = repo.get_document_sources(row["document_id"])
     assert sources[0]["mime_type"] == TppXlsxAdapter.mime_type
 
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.text, unit.speaker) for unit in units] == [("こんにちは", "Alice")]
+    assert document.get_text().splitlines() == ["こんにちは"]
 
     await document.set_text(["你好"])
     output_folder = tmp_path / "out"
@@ -518,8 +515,7 @@ async def test_galgame_document_imports_and_exports_renpy_script(tmp_path: Path)
     sources = repo.get_document_sources(row["document_id"])
     assert sources[0]["mime_type"] == RenPyScriptAdapter.mime_type
 
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.relative_path, unit.text, unit.speaker) for unit in units] == [("script.rpy", "こんにちは", "e")]
+    assert document.get_text().splitlines() == ["こんにちは"]
 
     await document.set_text(["你好"])
     output_folder = tmp_path / "out"
@@ -561,10 +557,7 @@ async def test_galgame_document_imports_and_exports_rpg_maker_folder(tmp_path: P
     sources = repo.get_document_sources(row["document_id"])
     assert sources[0]["mime_type"] == RpgMakerMvMzJsonAdapter.mime_type
 
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.relative_path, unit.text, unit.context) for unit in units] == [
-        ("www/data/Map001.json", "こんにちは", "Map event 1, page 1")
-    ]
+    assert document.get_text().splitlines() == ["こんにちは"]
 
     await document.set_text(["你好"])
     output_folder = tmp_path / "out"
@@ -589,8 +582,7 @@ async def test_galgame_document_imports_single_rpg_maker_map_file(tmp_path: Path
     sources = repo.get_document_sources(row["document_id"])
     assert sources[0]["relative_path"] == "Map001.json"
     assert sources[0]["mime_type"] == RpgMakerMvMzJsonAdapter.mime_type
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.relative_path, unit.text) for unit in units] == [("Map001.json", "こんにちは")]
+    assert document.get_text().splitlines() == ["こんにちは"]
 
 
 def test_galgame_folder_scan_rejects_rpg_maker_named_json_outside_data_folder(tmp_path: Path) -> None:
@@ -649,11 +641,7 @@ async def test_galgame_document_import_get_text_and_preserve_export(tmp_path: Pa
     assert sources[0]["relative_path"] == "data/script.json"
 
     document = GalgameDocument(repo, row["document_id"])
-    units = deserialize_translation_unit_stream(document.get_text())
-    assert [(unit.relative_path, unit.unit_id, unit.text) for unit in units] == [
-        ("data/script.json", "0", "こんにちは"),
-        ("data/script.json", "1", "またね"),
-    ]
+    assert document.get_text().splitlines() == ["こんにちは", "またね"]
 
     await document.set_text(["你好", "再见"])
     output_folder = tmp_path / "out"
@@ -663,6 +651,40 @@ async def test_galgame_document_import_get_text_and_preserve_export(tmp_path: Pa
     assert patched == {"こんにちは": "你好", "またね": "再见"}
 
 
+async def test_galgame_document_splits_grouped_chunk_translation_by_original_unit_lines(tmp_path: Path) -> None:
+    source = tmp_path / "script.json"
+    _write_mtool_json(source, {"こんにちは": "", "またね": ""})
+    repo = _setup_repo(tmp_path)
+    GalgameDocument.do_import(repo, source)
+    row = repo.get_document_row()
+    assert row is not None
+    document = GalgameDocument(repo, row["document_id"])
+
+    await document.set_text(["你好\n再见"])
+    output_folder = tmp_path / "out"
+    document.export_preserve_structure(output_folder)
+
+    patched = json.loads((output_folder / "script.json").read_text(encoding="utf-8"))
+    assert patched == {"こんにちは": "你好", "またね": "再见"}
+
+
+async def test_galgame_document_splits_multiline_units_by_original_line_counts(tmp_path: Path) -> None:
+    source = tmp_path / "script.json"
+    _write_mtool_json(source, {"こんにちは\n世界": "", "またね": ""})
+    repo = _setup_repo(tmp_path)
+    GalgameDocument.do_import(repo, source)
+    row = repo.get_document_row()
+    assert row is not None
+    document = GalgameDocument(repo, row["document_id"])
+
+    await document.set_text(["你好\n世界\n再见"])
+    output_folder = tmp_path / "out"
+    document.export_preserve_structure(output_folder)
+
+    patched = json.loads((output_folder / "script.json").read_text(encoding="utf-8"))
+    assert patched == {"こんにちは\n世界": "你好\n世界", "またね": "再见"}
+
+
 async def test_galgame_document_coerces_serialized_units_for_original_fallback(tmp_path: Path) -> None:
     source = tmp_path / "script.json"
     _write_mtool_json(source, {"こんにちは": "", "またね": ""})
@@ -680,7 +702,7 @@ async def test_galgame_document_coerces_serialized_units_for_original_fallback(t
     assert patched == {"こんにちは": "こんにちは", "またね": "またね"}
 
 
-async def test_galgame_export_raises_on_short_translation_stream(tmp_path: Path) -> None:
+async def test_galgame_export_raises_on_short_translation_stream() -> None:
     mock_repo = MagicMock()
     mock_repo.get_document_sources.return_value = [
         {
@@ -690,7 +712,6 @@ async def test_galgame_export_raises_on_short_translation_stream(tmp_path: Path)
         }
     ]
     document = GalgameDocument(mock_repo, 1)
-    await document.set_text(["你好"])
 
     with pytest.raises(ValueError, match="shorter than the source unit stream"):
-        document.export_preserve_structure(tmp_path / "out")
+        await document.set_text(["你好"])

{contextweave-0.3.0 → contextweave-0.3.1}/tests/workflow/test_import_export.py

@@ -640,10 +640,9 @@ Yes.
                 db.update_all_sources_text_added(doc_id)
 
                 chunks = translator.manager.term_repo.list_chunks(document_id=doc_id)
-                assert [chunk.text for chunk in chunks] == ["こんにちは", "またね"]
-                for chunk, translation in zip(chunks, ["你好", "再见"], strict=True):
-                    chunk.is_translated = True
-                    chunk.translation = translation
+                assert [chunk.text for chunk in chunks] == ["こんにちは\nまたね"]
+                chunks[0].is_translated = True
+                chunks[0].translation = "你好\n再见"
                 translator.manager.term_repo.apply_batch(BatchUpdate(keyed_context=[], chunk_records=chunks))
 
                 await _export_preserve_structure(translator, output_folder)

{contextweave-0.3.0 → contextweave-0.3.1}/uv.lock

@@ -171,7 +171,7 @@ wheels = [
 
 [[package]]
 name = "contextweave"
-version = "0.3.0"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "defusedxml" },

contextweave-0.3.0/context_aware_translation/core/galgame_document_handler.py

@@ -1,101 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Callable
-from typing import TYPE_CHECKING
-
-from context_aware_translation.core.cancellation import raise_if_cancelled
-from context_aware_translation.documents.galgame import deserialize_translation_unit_stream
-from context_aware_translation.storage.schema.book_db import ChunkRecord
-from context_aware_translation.utils.hashing import compute_chunk_hash
-
-if TYPE_CHECKING:
-    from context_aware_translation.core.context_manager import TranslationContextManager
-    from context_aware_translation.core.progress import ProgressCallback
-
-
-class GalgameDocumentHandler:
-    """DocumentTypeHandler for unit-preserving galgame translation."""
-
-    def __init__(
-        self,
-        *,
-        concurrency: int = 5,
-        batch_size: int = 0,
-        max_tokens_per_batch: int = 2000,
-    ) -> None:
-        self._concurrency = max(1, concurrency)
-        self._batch_size = batch_size
-        self._max_tokens_per_batch = max(1, max_tokens_per_batch)
-
-    def add_text(
-        self,
-        text: str,
-        max_token_size_per_chunk: int,  # noqa: ARG002
-        document_id: int,
-        manager: TranslationContextManager,
-    ) -> int:
-        """Store one chunk per translation unit without deduping repeated dialogue text."""
-        units = deserialize_translation_unit_stream(text)
-        chunk_records: list[ChunkRecord] = []
-        chunk_id = manager.term_repo.get_next_chunk_id()
-        for unit in units:
-            if not unit.text.strip():
-                continue
-            identity_text = f"{unit.relative_path}\0{unit.unit_id}\0{unit.text}"
-            chunk_records.append(
-                ChunkRecord(
-                    chunk_id=chunk_id,
-                    hash=compute_chunk_hash(identity_text, document_id=document_id),
-                    text=unit.text,
-                    document_id=document_id,
-                    is_extracted=False,
-                    is_summarized=False,
-                )
-            )
-            chunk_id += 1
-
-        new_chunk_records = [
-            chunk_record for chunk_record in chunk_records if not manager.term_repo.chunk_exists_by_hash(chunk_record.hash)
-        ]
-        if new_chunk_records:
-            manager._state_update([], new_chunk_records)
-        return chunk_id - 1
-
-    async def translate_chunks(
-        self,
-        document_ids: list[int],
-        manager: TranslationContextManager,
-        force: bool = False,
-        source_ids: list[int] | None = None,
-        cancel_check: Callable[[], bool] | None = None,
-        progress_callback: ProgressCallback | None = None,
-    ) -> None:
-        """Delegate to the generic text translator while preserving one chunk per unit."""
-        _ = source_ids
-        raise_if_cancelled(cancel_check)
-        await manager.translate_chunks(
-            concurrency=self._concurrency,
-            batch_size=self._batch_size,
-            max_tokens_per_batch=self._max_tokens_per_batch,
-            document_ids=document_ids,
-            force=force,
-            cancel_check=cancel_check,
-            progress_callback=progress_callback,
-        )
-
-    def get_translated_lines(
-        self,
-        document_id: int,
-        manager: TranslationContextManager,
-    ) -> list[str]:
-        """Return one translated entry per galgame translation unit."""
-        chunks = manager.term_repo.list_chunks(document_id=document_id)
-        if not chunks:
-            raise ValueError("No chunks found in the database")
-
-        sorted_chunks = sorted(chunks, key=lambda chunk: chunk.chunk_id)
-        untranslated = [chunk for chunk in sorted_chunks if not chunk.is_translated or chunk.translation is None]
-        if untranslated:
-            untranslated_ids = [chunk.chunk_id for chunk in untranslated]
-            raise ValueError(f"Cannot export: chunks {untranslated_ids} are not translated yet")
-        return [chunk.translation for chunk in sorted_chunks if chunk.translation is not None]

contextweave-0.3.0/tests/core/test_galgame_document_handler.py

@@ -1,98 +0,0 @@
-from __future__ import annotations
-
-from context_aware_translation.core.galgame_document_handler import GalgameDocumentHandler
-from context_aware_translation.documents.galgame import TranslationUnit, serialize_translation_units
-from context_aware_translation.storage.schema.book_db import ChunkRecord, TranslationChunkRecord
-
-
-class DummyTermRepo:
-    def __init__(self, chunks: list[ChunkRecord] | None = None) -> None:
-        self._chunks = chunks or []
-
-    def get_next_chunk_id(self) -> int:
-        return max((chunk.chunk_id for chunk in self._chunks), default=-1) + 1
-
-    def chunk_exists_by_hash(self, chunk_hash: str) -> int | None:
-        for chunk in self._chunks:
-            if chunk.hash == chunk_hash:
-                return chunk.chunk_id
-        return None
-
-    def list_chunks(self, document_id: int | None = None) -> list[ChunkRecord]:
-        if document_id is None:
-            return list(self._chunks)
-        return [chunk for chunk in self._chunks if chunk.document_id == document_id]
-
-
-class DummyManager:
-    def __init__(self, chunks: list[ChunkRecord] | None = None) -> None:
-        self.term_repo = DummyTermRepo(chunks)
-        self.translate_calls: list[dict[str, object]] = []
-
-    def _state_update(self, extracted_keyed_context, chunk_records):  # noqa: ANN001, ARG002
-        self.term_repo._chunks.extend(chunk_records)
-
-    async def translate_chunks(self, **kwargs):  # noqa: ANN003
-        self.translate_calls.append(kwargs)
-
-
-def test_galgame_handler_add_text_preserves_repeated_dialogue_units() -> None:
-    manager = DummyManager()
-    handler = GalgameDocumentHandler()
-    text = serialize_translation_units(
-        [
-            TranslationUnit(relative_path="script.json", unit_id="0", text="はい"),
-            TranslationUnit(relative_path="script.json", unit_id="1", text="はい"),
-        ]
-    )
-
-    last_chunk_id = handler.add_text(text, 100, 7, manager)
-
-    chunks = manager.term_repo.list_chunks(document_id=7)
-    assert last_chunk_id == 1
-    assert [chunk.text for chunk in chunks] == ["はい", "はい"]
-    assert chunks[0].hash != chunks[1].hash
-
-
-def test_galgame_handler_get_translated_lines_preserves_multiline_units() -> None:
-    chunks: list[ChunkRecord] = [
-        TranslationChunkRecord(
-            chunk_id=0,
-            hash="h0",
-            text="a",
-            document_id=7,
-            is_translated=True,
-            translation="第一行\n第二行",
-        ),
-        TranslationChunkRecord(
-            chunk_id=1,
-            hash="h1",
-            text="b",
-            document_id=7,
-            is_translated=True,
-            translation="第三行",
-        ),
-    ]
-    manager = DummyManager(chunks)
-    handler = GalgameDocumentHandler()
-
-    assert handler.get_translated_lines(7, manager) == ["第一行\n第二行", "第三行"]
-
-
-async def test_galgame_handler_delegates_translation_with_configured_batching() -> None:
-    manager = DummyManager()
-    handler = GalgameDocumentHandler(concurrency=3, batch_size=0, max_tokens_per_batch=1234)
-
-    await handler.translate_chunks([7], manager, force=True)
-
-    assert manager.translate_calls == [
-        {
-            "concurrency": 3,
-            "batch_size": 0,
-            "max_tokens_per_batch": 1234,
-            "document_ids": [7],
-            "force": True,
-            "cancel_check": None,
-            "progress_callback": None,
-        }
-    ]