wenyi-core 1.4.2__tar.gz

wenyi_core-1.4.2/.gitignore

@@ -0,0 +1,3 @@
+# Bundled language and prompt resources are source files, including .txt templates.
+!wenyi_core/i18n/data/
+!wenyi_core/i18n/data/**

wenyi_core-1.4.2/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.5
+Name: wenyi-core
+Version: 1.4.2
+Summary: Shared translation engine for multilingual long-form documents.
+Project-URL: Repository, https://github.com/BigDawnGhost/wenyi
+Project-URL: Documentation, https://github.com/BigDawnGhost/wenyi/blob/main/docs/architecture.md
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: ebooklib>=0.18
+Requires-Dist: google-genai>=2.13.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: json-repair<1,>=0.61
+Requires-Dist: lxml>=5.0
+Requires-Dist: openai>=1.40
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pypdf>=6.14.2
+Requires-Dist: python-docx>=1.2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: tenacity>=8.2
+Requires-Dist: tiktoken>=0.14.0
+Description-Content-Type: text/markdown
+
+# Wenyi Core
+
+Shared translation engine for multilingual long-form documents. It provides
+document ingestion, terminology management, contextual translation, polishing,
+whole-book review, resumable state, and export services.
+
+For the local command-line application, install **`wenyi-cli`** instead; it
+installs this package automatically. Core does not install the Web API or Desktop
+application.
+
+PDF layout preservation uses an optional, independently installed
+`wenyi-babeldoc-bridge` service over HTTP. BabelDOC is not a dependency of this
+package.
+
+- [Source and license](https://github.com/BigDawnGhost/wenyi)
+- [Architecture](https://github.com/BigDawnGhost/wenyi/blob/main/docs/architecture.md)
+- [CLI guide](https://github.com/BigDawnGhost/wenyi/blob/main/docs/cli.md)
+- [简体中文](https://github.com/BigDawnGhost/wenyi/blob/main/docs/zh/README.md)

wenyi_core-1.4.2/README.md

@@ -0,0 +1,18 @@
+# Wenyi Core
+
+Shared translation engine for multilingual long-form documents. It provides
+document ingestion, terminology management, contextual translation, polishing,
+whole-book review, resumable state, and export services.
+
+For the local command-line application, install **`wenyi-cli`** instead; it
+installs this package automatically. Core does not install the Web API or Desktop
+application.
+
+PDF layout preservation uses an optional, independently installed
+`wenyi-babeldoc-bridge` service over HTTP. BabelDOC is not a dependency of this
+package.
+
+- [Source and license](https://github.com/BigDawnGhost/wenyi)
+- [Architecture](https://github.com/BigDawnGhost/wenyi/blob/main/docs/architecture.md)
+- [CLI guide](https://github.com/BigDawnGhost/wenyi/blob/main/docs/cli.md)
+- [简体中文](https://github.com/BigDawnGhost/wenyi/blob/main/docs/zh/README.md)

wenyi_core-1.4.2/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "wenyi-core"
+dynamic = ["version"]
+description = "Shared translation engine for multilingual long-form documents."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "openai>=1.40",
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+    "tenacity>=8.2",
+    "beautifulsoup4>=4.12",
+    "httpx>=0.27",
+    "lxml>=5.0",
+    "ebooklib>=0.18",
+    "pypdf>=6.14.2",
+    "google-genai>=2.13.0",
+    "json-repair>=0.61,<1",
+    "python-docx>=1.2.0",
+    "tiktoken>=0.14.0",
+]
+
+[project.urls]
+Repository = "https://github.com/BigDawnGhost/wenyi"
+Documentation = "https://github.com/BigDawnGhost/wenyi/blob/main/docs/architecture.md"
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+root = "../.."
+
+[tool.hatch.build.targets.wheel]
+packages = ["wenyi_core"]

wenyi_core-1.4.2/tests/conftest.py

@@ -0,0 +1,31 @@
+"""Shared pytest fixtures for core tests."""
+
+from __future__ import annotations
+
+import re
+
+import pytest
+
+_ANSI_RE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\].*?\x07|\r")
+
+
+def plain_text(value: str) -> str:
+    """Strip ANSI/control sequences so CLI assertions stay stable under Rich."""
+    return _ANSI_RE.sub("", value)
+
+
+@pytest.fixture(autouse=True)
+def _plain_rich_console(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Prefer colorless consoles in tests; Progress may still emit markup."""
+    monkeypatch.setenv("NO_COLOR", "1")
+    monkeypatch.delenv("FORCE_COLOR", raising=False)
+
+    from rich.console import Console as RichConsole
+
+    def console_factory(*args, **kwargs):
+        kwargs = dict(kwargs)
+        kwargs["no_color"] = True
+        kwargs.setdefault("force_terminal", False)
+        return RichConsole(*args, **kwargs)
+
+    monkeypatch.setattr("rich.console.Console", console_factory)

wenyi_core-1.4.2/tests/fake_llm.py

@@ -0,0 +1,117 @@
+"""Offline FakeClient handler routing agent tasks through the complete pipeline."""
+
+from __future__ import annotations
+
+import json
+import re
+
+from wenyi_core.llm.providers.fake import FakeClient
+from wenyi_core.llm.usage import UsageSample
+
+METERED_TOTAL_TOKENS = 8  # prompt 5 + completion 3, recorded by MeteredFakeClient per call.
+
+
+def _count_numbered(text: str) -> int:
+    return len(re.findall(r"^\[(\d+)\]", text, re.MULTILINE))
+
+
+def routing_handler(messages, tier, json_mode):
+    system = messages[0]["content"]
+    user = messages[-1]["content"]
+
+    if "language detector" in system:
+        return json.dumps({"language": "ja"}, ensure_ascii=False)
+
+    if "pre-translation analyst" in system:
+        return json.dumps(
+            {
+                "genre": "校园",
+                "tone": "冷峻",
+                "style_guide": "克制",
+                "characters": [{"source": "綾小路", "gender": "male"}],
+            },
+            ensure_ascii=False,
+        )
+
+    if "chapter title translator" in system:
+        n = _count_numbered(user)
+        return json.dumps({"titles": [f"标题{i}" for i in range(n)]}, ensure_ascii=False)
+
+    if "literary translator" in system:
+        # Polish may append a user turn to the same translation conversation.
+        if "Polish the translations from your previous JSON response" in user:
+            n = None
+            for message in reversed(messages[:-1]):
+                if message.get("role") == "assistant":
+                    try:
+                        payload = json.loads(message["content"])
+                    except json.JSONDecodeError:
+                        payload = {}
+                    translations = payload.get("translations")
+                    if isinstance(translations, list):
+                        n = len(translations)
+                    break
+            if n is None:
+                match = re.search(r"exactly (\d+) items", user)
+                n = int(match.group(1)) if match else 0
+            return json.dumps({"polished": [f"润{i}" for i in range(n)]}, ensure_ascii=False)
+        n = _count_numbered(user)
+        return json.dumps({"translations": [f"译{i}" for i in range(n)]}, ensure_ascii=False)
+
+    if "prose editor" in system:
+        n = _count_numbered(user)
+        return json.dumps({"polished": [f"润{i}" for i in range(n)]}, ensure_ascii=False)
+
+    if "translation reviewer" in system:
+        n = _count_numbered(user)
+        return json.dumps(
+            {
+                "issues": [],
+                "reviewed_segments": n,
+                "complete": True,
+            },
+            ensure_ascii=False,
+        )
+
+    if "terminology" in system and "extractor" in system:
+        return json.dumps(
+            {"terms": [{"source": "堀北", "target": "堀北", "type": "person", "gender": "female"}]},
+            ensure_ascii=False,
+        )
+
+    if "chapter digest writer" in system:
+        return "本章梗概：人物登场，情节推进。"
+
+    if "whole-book synopsis writer" in system:
+        return "全书概览：主线与人物关系，整体基调。"
+
+    return "{}" if json_mode else ""
+
+
+class MeteredFakeClient(FakeClient):
+    """Record fixed small usage per offline call so tests can assert stage-level accounting."""
+
+    def complete(
+        self,
+        messages,
+        *,
+        operation,
+        json_mode=False,
+        max_tokens=None,
+    ):
+        self.usage.record(
+            self.routes[operation].tier or "direct",
+            UsageSample(
+                prompt_tokens=5,
+                completion_tokens=3,
+                total_tokens=8,
+                cache_miss_tokens=5,
+            ),
+            operation,
+        )
+        return super().complete(
+            messages,
+            operation=operation,
+            json_mode=json_mode,
+            max_tokens=max_tokens,
+        )

wenyi_core-1.4.2/tests/model_fixtures.py

@@ -0,0 +1,26 @@
+"""Explicit routing graphs for transport tests with injected SDK responses."""
+
+from wenyi_core.llm.configuration import LLMConfig
+from wenyi_core.llm.registry import provider_spec
+
+
+def model_config(*, kind="deepseek", profiles=None, **connection):
+    factory = getattr(provider_spec(kind)._module(), "preset_models", None)
+    defaults = factory() if factory else {}
+    profiles = profiles or {}
+    models = {}
+    for tier in ("strong", "cheap", "fast"):
+        default = defaults.get(tier)
+        profile = profiles.get(tier, {})
+        models[tier] = {
+            "provider": "default",
+            "model": profile.get("model") or (default.model if default else "test-model"),
+            "options": profile.get("options", default.options.model_dump() if default else {}),
+        }
+    return LLMConfig.model_validate(
+        {
+            "providers": {"default": {"kind": kind, **connection}},
+            "models": models,
+            "tiers": {tier: tier for tier in models},
+        }
+    )

wenyi_core-1.4.2/tests/pipeline_fixtures.py

@@ -0,0 +1,55 @@
+"""Shared configuration and storage narrowing for offline pipeline tests."""
+
+from wenyi_core.config import Config
+from wenyi_core.models.document import Segment
+from wenyi_core.pipeline.translate.translation_batch import BatchPlan
+from wenyi_core.storage.file import FileStorage
+from wenyi_core.storage.protocol import Storage
+
+
+def require_file_storage(store: Storage) -> FileStorage:
+    """CLI/offline tests use the file backend; narrow Storage to FileStorage for path asserts."""
+    if not isinstance(store, FileStorage):
+        raise TypeError(f"expected FileStorage, got {type(store).__name__}")
+    return store
+
+
+def fake_agent_config() -> Config:
+    return Config.from_dict(
+        {
+            "language": {"source": "ja", "target": "zh"},
+            "llm": {
+                "preset": "fake",
+                "models": {
+                    "default_strong": {"provider": "default", "model": "p"},
+                    "default_cheap": {"provider": "default", "model": "f"},
+                },
+            },
+        }
+    )
+
+
+def batch_plan(sources, *, allow_empty=False):
+    return BatchPlan.capture(
+        3,
+        5,
+        [Segment(index=10 + i, source=s) for i, s in enumerate(sources)],
+        [],
+        "previous translation",
+        "style",
+        "synopsis",
+        "digest",
+        [[] for _ in sources],
+        "next source",
+        allow_empty_translations=allow_empty,
+    )
+
+
+def fake_pipeline_config(state_dir: str) -> Config:
+    config = fake_agent_config()
+    config.segment.max_tokens_per_batch = 1800
+    config.pipeline.review = True
+    config.pipeline.review_autofix = False
+    config.pipeline.polish = True
+    config.state_dir = state_dir
+    return config

wenyi_core-1.4.2/tests/precision_fixtures.py

@@ -0,0 +1,106 @@
+"""Shared fixed-book fixtures and fake responses for precision tests."""
+
+import json
+from collections import Counter
+from pathlib import Path
+from threading import Lock
+
+from wenyi_core.config import Config
+from wenyi_core.models.document import Chapter, Segment
+from wenyi_core.pipeline.translate.translation_batch import BatchPlan
+from wenyi_core.storage.file import FileStorage
+from wenyi_core.storage.run_store import STATUS_PENDING
+
+from tests.fake_llm import routing_handler
+
+
+def precision_config(tmp_path: Path) -> Config:
+    return Config.from_dict(
+        {
+            "language": {"source": "en", "target": "zh"},
+            "llm": {"preset": "fake"},
+            "pipeline": {
+                "translation_mode": "best_of_three",
+                "review": False,
+                "book_understanding": False,
+                "annotation_alignment": False,
+            },
+            "paths": {"state_dir": str(tmp_path)},
+        }
+    )
+
+
+def precision_store(tmp_path: Path, sources: tuple[str, ...] = ("one", "two")) -> FileStorage:
+    store = FileStorage(str(tmp_path / "book"))
+    store.save_chapter(
+        Chapter(
+            index=0,
+            segments=[Segment(index=i + 10, source=source) for i, source in enumerate(sources)],
+        )
+    )
+    store.save_manifest(
+        {
+            "title": "book",
+            "source_lang": "en",
+            "target_lang": "zh",
+            "source_sha256": "0" * 64,
+            "chapters": [{"index": 0, "status": STATUS_PENDING}],
+        }
+    )
+    return store
+
+
+def artifact(store: FileStorage, key: str) -> dict:
+    value = store.read_artifact(key)
+    assert isinstance(value, dict), f"Expected a JSON object at {key}"
+    return value
+
+
+def precision_plan(store: FileStorage, allow_empty: bool = False) -> BatchPlan:
+    segments = store.load_chapter(0).text_segments
+    return BatchPlan.capture(
+        0,
+        0,
+        segments,
+        [],
+        "prior",
+        "style",
+        "synopsis",
+        "digest",
+        [[] for _ in segments],
+        "following",
+        allow_empty_translations=allow_empty,
+    )
+
+
+class PrecisionHandler:
+    def __init__(self, invalid=False, blank=False):
+        self.counts, self.lock = Counter(), Lock()
+        self.invalid, self.blank = invalid, blank
+
+    def __call__(self, messages, tier, json_mode):
+        task_message = next(
+            (message for message in messages if "Task (JSON):\n" in message["content"]), None
+        )
+        if task_message is None:
+            return routing_handler(messages, tier, json_mode)
+        sources = json.loads(messages[1]["content"].split("\n", 1)[1])["sources"]
+        task = json.loads(task_message["content"].split("Task (JSON):\n")[1])
+        kind = "synthesis" if "drafts" in task else "translate"
+        with self.lock:
+            self.counts[kind] += 1
+            sample = self.counts[kind]
+        if kind == "synthesis" and self.invalid:
+            return '{"translations":[]}'
+        return json.dumps(
+            {
+                "translations": [
+                    source
+                    if not any(character.isalpha() for character in source)
+                    else ""
+                    if self.blank
+                    else f"{'润' if kind == 'synthesis' else '译'}{sample}:{i}"
+                    for i, source in enumerate(sources)
+                ]
+            }
+        )

wenyi_core-1.4.2/tests/review_fixtures.py

@@ -0,0 +1,130 @@
+"""Small book and fake-model configuration builders for review tests."""
+
+import json
+import re
+from pathlib import Path
+
+from wenyi_core.config import Config
+from wenyi_core.models.document import Chapter, Segment
+from wenyi_core.models.glossary import GlossaryTerm
+from wenyi_core.review.evidence import BookEvidenceIndex
+from wenyi_core.storage.file import FileStorage
+from wenyi_core.storage.run_store import STATUS_DONE
+
+
+def _config() -> Config:
+    return Config.from_dict(
+        {
+            "language": {"source": "en", "target": "zh"},
+            "llm": {
+                "preset": "fake",
+                "models": {
+                    "default_strong": {"provider": "default", "model": "strong"},
+                    "default_cheap": {"provider": "default", "model": "cheap"},
+                },
+            },
+            "pipeline": {
+                "review_agent_max_evidence_rounds": 2,
+            },
+        }
+    )
+
+
+def _chapter(index: int, texts: list[tuple[str, str]]) -> Chapter:
+    return Chapter(
+        index=index,
+        title=f"Chapter {index}",
+        segments=[
+            Segment(index=segment_index, source=source, target=target)
+            for segment_index, (source, target) in enumerate(texts)
+        ],
+        meta={"source_digest": f"Digest {index}"},
+    )
+
+
+def review_evidence() -> BookEvidenceIndex:
+    return BookEvidenceIndex(
+        [
+            _chapter(
+                0,
+                [
+                    ("Ann arrived.", "安到了。"),
+                    ("Ann spoke.", "安开口了。"),
+                ],
+            )
+        ],
+        [GlossaryTerm(source="Ann", target="安", type="person")],
+        {},
+    )
+
+
+def autofix_config(state_dir: str) -> Config:
+    return Config.from_dict(
+        {
+            "language": {"source": "ja", "target": "zh"},
+            "llm": {
+                "preset": "fake",
+                "models": {"default_strong": {"provider": "default", "model": "p"}},
+            },
+            "pipeline": {
+                "review_autofix": True,
+                "review_concurrency": 1,
+            },
+            "output": {"punctuation_normalize": False},
+            "paths": {"state_dir": state_dir},
+        }
+    )
+
+
+def autofix_store(directory: str, target: str = "正式译文。") -> FileStorage:
+    store = FileStorage(str(Path(directory, "state", "book")))
+    store.save_chapter(
+        Chapter(
+            index=0,
+            title="第一章",
+            segments=[Segment(index=0, source="原文。", target=target)],
+        )
+    )
+    store.save_manifest(
+        {
+            "title": "book",
+            "source_lang": "ja",
+            "target_lang": "zh",
+            "source_sha256": "0" * 64,
+            "chapters": [{"index": 0, "status": STATUS_DONE}],
+        }
+    )
+    return store
+
+
+def review_json(user: str, issues: list[dict]) -> str:
+    """Build a reviewer response with its completeness receipt."""
+    return json.dumps(
+        {
+            "issues": issues,
+            "reviewed_segments": len(re.findall(r"^\[(\d+)\]", user, re.MULTILINE)),
+            "complete": True,
+        },
+        ensure_ascii=False,
+    )
+
+
+def fix_json(user: str, replacement: str) -> str:
+    """Echo identity fields from a fixer request into a complete temporary replacement."""
+
+    def field(name: str) -> str:
+        match = re.search(rf"^{name}:\s*(.+)$", user, re.MULTILINE)
+        if match is None:
+            raise AssertionError(f"Fixer prompt missing {name}")
+        return match.group(1).strip()
+
+    return json.dumps(
+        {
+            "segment_ref": field("segment_ref"),
+            "before_hash": field("before_hash"),
+            "issue_ids": json.loads(field("issue_ids")),
+            "replacement": replacement,
+            "complete": True,
+        },
+        ensure_ascii=False,
+    )