deepresearch-flow 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

deepresearch_flow/translator/cli.py

@@ -7,6 +7,7 @@ import json
 import logging
 from pathlib import Path
 import time
+from typing import Any
 
 import click
 import coloredlogs
@@ -24,7 +25,7 @@ from deepresearch_flow.paper.utils import (
     short_hash,
 )
 from deepresearch_flow.translator.config import TranslateConfig
-from deepresearch_flow.translator.engine import MarkdownTranslator, RequestThrottle
+from deepresearch_flow.translator.engine import DumpSnapshot, MarkdownTranslator, RequestThrottle
 
 
 logger = logging.getLogger(__name__)
@@ -126,6 +127,14 @@ def translator() -> None:
 @click.option("--fix-level", "fix_level", default="moderate", type=click.Choice(["off", "moderate", "aggressive"]))
 @click.option("--max-chunk-chars", "max_chunk_chars", default=4000, show_default=True, type=int)
 @click.option("--max-concurrency", "max_concurrency", default=4, show_default=True, type=int)
+@click.option(
+    "--group-concurrency",
+    "group_concurrency",
+    default=1,
+    show_default=True,
+    type=int,
+    help="Concurrent translation groups per document",
+)
 @click.option("--timeout", "timeout", default=120.0, show_default=True, type=float)
 @click.option("--retry-times", "retry_times", default=3, show_default=True, type=int)
 @click.option("--fallback-model", "fallback_model_ref", default=None, help="Fallback provider/model")
@@ -155,6 +164,12 @@ def translator() -> None:
 @click.option("--dump-protected", "dump_protected", is_flag=True, help="Write protected markdown")
 @click.option("--dump-placeholders", "dump_placeholders", is_flag=True, help="Write placeholder mapping JSON")
 @click.option("--dump-nodes", "dump_nodes", is_flag=True, help="Write per-node translation JSON")
+@click.option(
+    "--dump-requests-log",
+    "dump_requests_log",
+    is_flag=True,
+    help="Write request/response attempts to JSON log",
+)
 @click.option("--no-format", "no_format", is_flag=True, help="Disable rumdl formatting")
 @click.option("--dry-run", "dry_run", is_flag=True, help="Discover inputs without calling providers")
 @click.option("--force", "force", is_flag=True, help="Overwrite existing outputs")
@@ -171,6 +186,7 @@ def translate(
     fix_level: str,
     max_chunk_chars: int,
     max_concurrency: int,
+    group_concurrency: int,
     timeout: float,
     retry_times: int,
     fallback_model_ref: str | None,
@@ -183,6 +199,7 @@ def translate(
     dump_protected: bool,
     dump_placeholders: bool,
     dump_nodes: bool,
+    dump_requests_log: bool,
     no_format: bool,
     dry_run: bool,
     force: bool,
@@ -240,6 +257,8 @@ def translate(
         raise click.ClickException("--max-chunk-chars must be positive")
     if max_concurrency <= 0:
         raise click.ClickException("--max-concurrency must be positive")
+    if group_concurrency <= 0:
+        raise click.ClickException("--group-concurrency must be positive")
     if timeout <= 0:
         raise click.ClickException("--timeout must be positive")
     if retry_times <= 0:
@@ -288,7 +307,9 @@ def translate(
         output_root.mkdir(parents=True, exist_ok=True)
 
     debug_root = Path(debug_dir) if debug_dir else None
-    if debug_root is None and (dump_protected or dump_placeholders or dump_nodes):
+    if debug_root is None and (
+        dump_protected or dump_placeholders or dump_nodes or dump_requests_log
+    ):
         debug_root = output_root or Path.cwd()
     if debug_root is not None:
         debug_root.mkdir(parents=True, exist_ok=True)
@@ -360,6 +381,43 @@ def translate(
         progress: ProgressTracker,
     ) -> None:
         content = read_text(path)
+        request_log: list[dict[str, Any]] = []
+        debug_tag = None
+        protected_path = None
+        placeholders_path = None
+        nodes_path = None
+        requests_path = None
+        if debug_root is not None:
+            debug_tag = f"{path.stem}.{short_hash(str(path))}"
+            protected_path = debug_root / f"{debug_tag}.protected.md"
+            placeholders_path = debug_root / f"{debug_tag}.placeholders.json"
+            nodes_path = debug_root / f"{debug_tag}.nodes.json"
+            requests_path = debug_root / f"{debug_tag}.requests.json"
+
+        def write_dump(snapshot: DumpSnapshot) -> None:
+            if debug_root is None or debug_tag is None:
+                return
+            if dump_protected and snapshot.protected_text is not None and protected_path:
+                protected_path.write_text(snapshot.protected_text, encoding="utf-8")
+            if dump_placeholders and snapshot.placeholder_store is not None and placeholders_path:
+                snapshot.placeholder_store.save(str(placeholders_path))
+            if dump_nodes and snapshot.nodes is not None and nodes_path:
+                node_payload = {
+                    str(node_id): {
+                        "origin_text": node.origin_text,
+                        "translated_text": node.translated_text,
+                    }
+                    for node_id, node in snapshot.nodes.items()
+                }
+                nodes_path.write_text(
+                    json.dumps(node_payload, ensure_ascii=False, indent=2),
+                    encoding="utf-8",
+                )
+            if dump_requests_log and snapshot.request_log is not None and requests_path:
+                requests_path.write_text(
+                    json.dumps(snapshot.request_log, ensure_ascii=False, indent=2),
+                    encoding="utf-8",
+                )
         result = await translator.translate(
             content,
             provider,
@@ -381,6 +439,9 @@ def translate(
             fallback_retry_times=fallback_retry_times,
             fallback_retry_times_2=fallback_retry_times_2,
             format_enabled=not no_format,
+            request_log=request_log if dump_requests_log else None,
+            dump_callback=write_dump if debug_root is not None else None,
+            group_concurrency=group_concurrency,
         )
         output_path = output_map[path]
         output_path.write_text(result.translated_text, encoding="utf-8")
@@ -401,25 +462,15 @@ def translate(
         )
 
         if debug_root is not None:
-            debug_tag = f"{path.stem}.{short_hash(str(path))}"
-            if dump_protected:
-                (debug_root / f"{debug_tag}.protected.md").write_text(
-                    result.protected_text, encoding="utf-8"
-                )
-            if dump_placeholders:
-                result.placeholder_store.save(str(debug_root / f"{debug_tag}.placeholders.json"))
-            if dump_nodes:
-                node_payload = {
-                    str(node_id): {
-                        "origin_text": node.origin_text,
-                        "translated_text": node.translated_text,
-                    }
-                    for node_id, node in result.nodes.items()
-                }
-                (debug_root / f"{debug_tag}.nodes.json").write_text(
-                    json.dumps(node_payload, ensure_ascii=False, indent=2),
-                    encoding="utf-8",
+            write_dump(
+                DumpSnapshot(
+                    stage="final",
+                    nodes=result.nodes,
+                    protected_text=result.protected_text,
+                    placeholder_store=result.placeholder_store,
+                    request_log=request_log if dump_requests_log else None,
                 )
+            )
         await progress.advance_docs(1)
 
     async def run() -> None:

deepresearch_flow/translator/engine.py

@@ -9,7 +9,8 @@ import logging
 import re
 import shutil
 import subprocess
-from typing import Optional, Protocol
+import time
+from typing import Any, Callable, Optional, Protocol
 
 import httpx
 
@@ -47,6 +48,15 @@ class TranslationResult:
     stats: "TranslationStats"
 
 
+@dataclass
+class DumpSnapshot:
+    stage: str
+    nodes: dict[int, Node] | None = None
+    protected_text: str | None = None
+    placeholder_store: PlaceHolderStore | None = None
+    request_log: list[dict[str, Any]] | None = None
+
+
 @dataclass
 class TranslationStats:
     total_nodes: int
@@ -116,7 +126,8 @@ class MarkdownTranslator:
         )
 
         self._rx_node_unpack = re.compile(
-            r"<NODE_START_(\d{4})>(.*?)</NODE_END_\1>", re.DOTALL
+            r"(?:<|@@)NODE_START_(\d{4})(?:>|@@)(.*?)(?:</NODE_END_\1>|@@NODE_END_\1@@)",
+            re.DOTALL,
         )
 
     def _strip_untranslatables(self, s: str) -> str:
@@ -167,6 +178,10 @@ class MarkdownTranslator:
         text = s.strip()
         if not text:
             return False
+        if "__PH_" in text:
+            core = self._strip_untranslatables(text)
+            if len(core) <= 2:
+                return True
         if re.search(r"\b(?:isbn|issn|doi|arxiv)\b", text, flags=re.IGNORECASE):
             return True
         if re.search(r"\b(?:acm|ieee)\b", text, flags=re.IGNORECASE):
@@ -341,7 +356,10 @@ class MarkdownTranslator:
             return text
         if not self._rumdl_path:
             if not self._rumdl_warned:
-                logger.warning("rumdl not available; skip markdown formatting")
+                logger.warning(
+                    "rumdl not available; skip markdown formatting (stage=%s)",
+                    stage,
+                )
                 self._rumdl_warned = True
             return text
 
@@ -356,8 +374,9 @@ class MarkdownTranslator:
         result = await asyncio.to_thread(run)
         if result.returncode != 0:
             logger.warning(
-                "rumdl fmt failed (%s): %s",
+                "rumdl fmt failed (stage=%s, rc=%s): %s",
                 stage,
+                result.returncode,
                 (result.stderr or "").strip() or "unknown error",
             )
             return text
@@ -530,6 +549,10 @@ class MarkdownTranslator:
         throttle: RequestThrottle | None,
         max_tokens: int | None,
         max_retries: int,
+        request_log: list[dict[str, Any]] | None,
+        stage: str,
+        group_index: int,
+        dump_callback: Callable[[DumpSnapshot], None] | None,
     ) -> str:
         attempts = 0
         while True:
@@ -539,9 +562,10 @@ class MarkdownTranslator:
             messages = build_translation_messages(
                 self.cfg.source_lang, self.cfg.target_lang, group_text
             )
+            start_time = time.time()
             try:
                 async with semaphore:
-                    return await call_provider(
+                    response = await call_provider(
                         provider,
                         model,
                         messages,
@@ -552,7 +576,60 @@ class MarkdownTranslator:
                         client,
                         max_tokens=max_tokens,
                     )
+                elapsed_ms = int((time.time() - start_time) * 1000)
+                if request_log is not None:
+                    request_log.append(
+                        {
+                            "stage": stage,
+                            "group_index": group_index,
+                            "attempt": attempts,
+                            "provider": provider.name,
+                            "model": model,
+                            "messages": messages,
+                            "response": response,
+                            "elapsed_ms": elapsed_ms,
+                        }
+                    )
+                    if dump_callback is not None:
+                        dump_callback(DumpSnapshot(stage=stage, request_log=request_log))
+                if logger.isEnabledFor(logging.DEBUG):
+                    logger.debug(
+                        "Group translated: stage=%s group=%d attempt=%d chars=%d elapsed_ms=%d",
+                        stage,
+                        group_index,
+                        attempts,
+                        len(group_text),
+                        elapsed_ms,
+                    )
+                return response
             except ProviderError as exc:
+                elapsed_ms = int((time.time() - start_time) * 1000)
+                if request_log is not None:
+                    request_log.append(
+                        {
+                            "stage": stage,
+                            "group_index": group_index,
+                            "attempt": attempts,
+                            "provider": provider.name,
+                            "model": model,
+                            "messages": messages,
+                            "error": str(exc),
+                            "retryable": exc.retryable,
+                            "elapsed_ms": elapsed_ms,
+                        }
+                    )
+                    if dump_callback is not None:
+                        dump_callback(DumpSnapshot(stage=stage, request_log=request_log))
+                if logger.isEnabledFor(logging.DEBUG):
+                    logger.debug(
+                        "Group failed: stage=%s group=%d attempt=%d retryable=%s elapsed_ms=%d error=%s",
+                        stage,
+                        group_index,
+                        attempts,
+                        exc.retryable,
+                        elapsed_ms,
+                        exc,
+                    )
                 if exc.retryable and attempts < max_retries:
                     await asyncio.sleep(backoff_delay(1.0, attempts, 20.0))
                     continue
@@ -580,6 +657,9 @@ class MarkdownTranslator:
         fallback_retry_times: int | None = None,
         fallback_retry_times_2: int | None = None,
         format_enabled: bool = True,
+        request_log: list[dict[str, Any]] | None = None,
+        dump_callback: Callable[[DumpSnapshot], None] | None = None,
+        group_concurrency: int = 1,
     ) -> TranslationResult:
         if fix_level != "off":
             text = fix_markdown(text, fix_level)
@@ -588,6 +668,15 @@ class MarkdownTranslator:
 
         store = PlaceHolderStore()
         protected = self.protector.protect(text, self.cfg, store)
+        if dump_callback is not None:
+            dump_callback(
+                DumpSnapshot(
+                    stage="protected",
+                    protected_text=protected,
+                    placeholder_store=store,
+                    request_log=request_log,
+                )
+            )
         segments, nodes = split_to_segments(protected, self.cfg.max_chunk_chars)
         total_nodes = len(nodes)
         if logger.isEnabledFor(logging.DEBUG):
@@ -607,31 +696,111 @@ class MarkdownTranslator:
         rotator = KeyRotator(resolve_api_keys(api_keys))
         max_retries = max(self.cfg.retry_times, 1)
 
-        groups = self._group_nodes(nodes)
-        initial_groups = len(groups)
-        if logger.isEnabledFor(logging.DEBUG):
-            logger.debug("Groups: %d", len(groups))
-        if progress:
-            await progress.add_groups(len(groups))
-        outputs: list[str] = []
-        for group in groups:
-            api_key = await rotator.next_key()
-            outputs.append(
-                await self._translate_group(
-                    group,
-                    provider,
-                    model,
+        nodes_progress: dict[int, Node] | None = None
+        if dump_callback is not None:
+            nodes_progress = {
+                nid: Node(
+                    nid=nid,
+                    origin_text=node.origin_text,
+                    translated_text=node.translated_text,
+                )
+                for nid, node in nodes.items()
+            }
+
+        async def run_groups(
+            groups: list[str],
+            rotator: KeyRotator,
+            stage: str,
+            max_tokens_value: int | None,
+            retry_limit_value: int,
+            provider_value: ProviderConfig,
+            model_value: str,
+        ) -> list[str]:
+            if not groups:
+                return []
+            outputs: list[str] = [""] * len(groups)
+
+            async def run_one(idx: int, group_text: str) -> tuple[int, str]:
+                api_key = await rotator.next_key()
+                response = await self._translate_group(
+                    group_text,
+                    provider_value,
+                    model_value,
                     client,
                     api_key,
                     timeout,
                     semaphore,
                     throttle,
-                    max_tokens,
-                    max_retries,
+                    max_tokens_value,
+                    retry_limit_value,
+                    request_log,
+                    stage,
+                    idx,
+                    dump_callback,
                 )
-            )
-            if progress:
-                await progress.advance_groups(1)
+                return idx, response
+
+            if group_concurrency <= 1:
+                for idx, group_text in enumerate(groups):
+                    idx_out, response = await run_one(idx, group_text)
+                    outputs[idx_out] = response
+                    if nodes_progress is not None:
+                        nodes_progress.update(self._ungroup_nodes(response, nodes))
+                        dump_callback(
+                            DumpSnapshot(
+                                stage=stage,
+                                nodes=nodes_progress,
+                                request_log=request_log,
+                            )
+                        )
+                    if progress:
+                        await progress.advance_groups(1)
+                return outputs
+
+            guard = asyncio.Semaphore(group_concurrency)
+
+            async def guarded(idx: int, group_text: str) -> tuple[int, str]:
+                async with guard:
+                    return await run_one(idx, group_text)
+
+            tasks = [asyncio.create_task(guarded(i, g)) for i, g in enumerate(groups)]
+            try:
+                for task in asyncio.as_completed(tasks):
+                    idx_out, response = await task
+                    outputs[idx_out] = response
+                    if nodes_progress is not None:
+                        nodes_progress.update(self._ungroup_nodes(response, nodes))
+                        dump_callback(
+                            DumpSnapshot(
+                                stage=stage,
+                                nodes=nodes_progress,
+                                request_log=request_log,
+                            )
+                        )
+                    if progress:
+                        await progress.advance_groups(1)
+            except Exception:
+                for task in tasks:
+                    task.cancel()
+                raise
+
+            return outputs
+
+        groups = self._group_nodes(nodes)
+        initial_groups = len(groups)
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug("Groups: %d", len(groups))
+        if progress:
+            await progress.add_groups(len(groups))
+        outputs = await run_groups(
+            groups,
+            rotator,
+            "initial",
+            max_tokens,
+            max_retries,
+            provider,
+            model,
+        )
 
         translated_nodes = self._ungroup_groups(outputs, nodes)
         valid_placeholders = set(store.snapshot().values())
@@ -692,25 +861,15 @@ class MarkdownTranslator:
                 )
                 if progress:
                     await progress.add_groups(len(retry_groups))
-                retry_outputs: list[str] = []
-                for group in retry_groups:
-                    api_key = await rotator.next_key()
-                    retry_outputs.append(
-                        await self._translate_group(
-                            group,
-                            provider,
-                            model,
-                            client,
-                            api_key,
-                            timeout,
-                            semaphore,
-                            throttle,
-                            max_tokens,
-                            retry_limit,
-                        )
-                    )
-                    if progress:
-                        await progress.advance_groups(1)
+                retry_outputs = await run_groups(
+                    retry_groups,
+                    rotator,
+                    f"retry-{attempt}",
+                    max_tokens,
+                    retry_limit,
+                    provider,
+                    model,
+                )
                 retry_nodes = self._ungroup_groups(
                     retry_outputs, failed_nodes, fill_missing=False
                 )
@@ -777,25 +936,15 @@ class MarkdownTranslator:
                 )
                 if progress:
                     await progress.add_groups(len(retry_groups))
-                retry_outputs: list[str] = []
-                for group in retry_groups:
-                    api_key = await fallback_rotator.next_key()
-                    retry_outputs.append(
-                        await self._translate_group(
-                            group,
-                            fallback_provider,
-                            fallback_model,
-                            client,
-                            api_key,
-                            timeout,
-                            semaphore,
-                            throttle,
-                            fallback_max_tokens,
-                            fallback_retry_limit,
-                        )
-                    )
-                    if progress:
-                        await progress.advance_groups(1)
+                retry_outputs = await run_groups(
+                    retry_groups,
+                    fallback_rotator,
+                    f"fallback-{attempt}",
+                    fallback_max_tokens,
+                    fallback_retry_limit,
+                    fallback_provider,
+                    fallback_model,
+                )
                 retry_nodes = self._ungroup_groups(
                     retry_outputs, failed_nodes, fill_missing=False
                 )
@@ -862,25 +1011,15 @@ class MarkdownTranslator:
                 )
                 if progress:
                     await progress.add_groups(len(retry_groups))
-                retry_outputs: list[str] = []
-                for group in retry_groups:
-                    api_key = await fallback_rotator.next_key()
-                    retry_outputs.append(
-                        await self._translate_group(
-                            group,
-                            fallback_provider_2,
-                            fallback_model_2,
-                            client,
-                            api_key,
-                            timeout,
-                            semaphore,
-                            throttle,
-                            fallback_max_tokens_2,
-                            fallback_retry_limit,
-                        )
-                    )
-                    if progress:
-                        await progress.advance_groups(1)
+                retry_outputs = await run_groups(
+                    retry_groups,
+                    fallback_rotator,
+                    f"fallback2-{attempt}",
+                    fallback_max_tokens_2,
+                    fallback_retry_limit,
+                    fallback_provider_2,
+                    fallback_model_2,
+                )
                 retry_nodes = self._ungroup_groups(
                     retry_outputs, failed_nodes, fill_missing=False
                 )

deepresearch_flow/translator/fixers.py

@@ -448,4 +448,19 @@ def fix_markdown(text: str, level: str) -> str:
     if level == "aggressive":
         text = title_processor.fix_titles(text)
 
+    try:
+        from deepresearch_flow.paper.web.markdown import (
+            normalize_fenced_code_blocks,
+            normalize_footnote_definitions,
+            normalize_mermaid_blocks,
+            normalize_unbalanced_fences,
+        )
+    except Exception:
+        return text
+
+    text = normalize_fenced_code_blocks(text)
+    text = normalize_mermaid_blocks(text)
+    text = normalize_unbalanced_fences(text)
+    text = normalize_footnote_definitions(text)
+
     return text

deepresearch_flow/translator/prompts.py

@@ -28,7 +28,7 @@ You are a professional translation engine. Follow these invariant rules:
 
 TRANSLATE_XML_TEMPLATE = Template(
     dedent(
-        """\
+        r"""\
 <TranslationTask version="1.0">
   <meta>
     <source_lang>$SOURCE_LANG</source_lang>
@@ -36,14 +36,23 @@ TRANSLATE_XML_TEMPLATE = Template(
     <visibility_note>Sections with visibility="internal" are instructions and MUST NOT appear in the final output.</visibility_note>
   </meta>
 
+  <task>
+    You are a professional $SOURCE_LANG_NAME ($SOURCE_LANG_CODE) to $TARGET_LANG_NAME ($TARGET_LANG_CODE) translator.
+    Your goal is to accurately convey the meaning and nuances of the original $SOURCE_LANG_NAME text while adhering to $TARGET_LANG_NAME grammar, vocabulary, and cultural sensitivities.
+    Produce only the $TARGET_LANG_NAME translation, without any additional explanations or commentary.
+    Please translate the following $SOURCE_LANG_NAME text into $TARGET_LANG_NAME:
+    Important: There are two blank lines before the text to translate.
+  </task>
+
   <constraints visibility="internal">
     <rule id="fmt-1">Preserve ALL original formatting exactly: Markdown, whitespace, line breaks, paragraph spacing.</rule>
-    <rule id="fmt-2">Do NOT translate any content inside LaTeX ($$...$$, $$$$...$$$$, \\( ... \\), \\[ ... \\]) or LaTeX commands/environments.</rule>
+    <rule id="fmt-2">Do NOT translate any content inside LaTeX ($$...$$, $$$$...$$$$, \( ... \), \[ ... \]) or LaTeX commands/environments.</rule>
     <rule id="fmt-3">Keep ALL HTML tags intact.</rule>
     <rule id="fmt-4">Do NOT alter abbreviations, technical terms, or code identifiers; translate surrounding prose only.</rule>
     <rule id="fmt-5">Document structure must be preserved, including blank lines (double newlines) between blocks.</rule>
   </constraints>
 
+
   <markers visibility="internal">
     <preserve>
       <open>@@PRESERVE_{n}@@</open>
@@ -91,6 +100,8 @@ TRANSLATE_XML_TEMPLATE = Template(
   <io>
     <input>
       <![CDATA[
+
+
 $TEXT_TO_TRANSLATE
       ]]>
     </input>
@@ -105,9 +116,15 @@ $TEXT_TO_TRANSLATE
 
 
 def build_translation_messages(source_lang: str | None, target_lang: str, text: str) -> list[dict[str, str]]:
+    source_name = source_lang or "auto"
+    target_name = target_lang
     user_xml = TRANSLATE_XML_TEMPLATE.substitute(
         SOURCE_LANG=source_lang or "auto",
         TARGET_LANG=target_lang,
+        SOURCE_LANG_NAME=source_name,
+        SOURCE_LANG_CODE=source_name,
+        TARGET_LANG_NAME=target_name,
+        TARGET_LANG_CODE=target_name,
         TEXT_TO_TRANSLATE=_cdata_wrap(text),
     )
     return [