deepresearch-flow 0.1.1__py3-none-any.whl

deepresearch_flow/paper/llm.py

@@ -0,0 +1,115 @@
+"""Shared LLM call helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+import httpx
+
+from deepresearch_flow.paper.config import ProviderConfig
+from deepresearch_flow.paper.providers.base import ProviderError
+from deepresearch_flow.paper.providers.azure_openai import chat as azure_openai_chat
+from deepresearch_flow.paper.providers.claude import chat as claude_chat
+from deepresearch_flow.paper.providers.dashscope import chat as dashscope_chat
+from deepresearch_flow.paper.providers.gemini import (
+    chat_ai_studio as gemini_ai_studio_chat,
+    chat_vertex as gemini_vertex_chat,
+)
+from deepresearch_flow.paper.providers.ollama import chat as ollama_chat
+from deepresearch_flow.paper.providers.openai_compatible import chat as openai_chat
+
+
+def backoff_delay(base: float, attempt: int, max_delay: float) -> float:
+    delay = base * (2 ** max(attempt - 1, 0))
+    return min(delay, max_delay)
+
+
+async def call_provider(
+    provider: ProviderConfig,
+    model: str,
+    messages: list[dict[str, str]],
+    schema: dict[str, Any],
+    api_key: str | None,
+    timeout: float,
+    structured_mode: str,
+    client: httpx.AsyncClient,
+) -> str:
+    headers = dict(provider.extra_headers)
+    if api_key and provider.type == "openai_compatible":
+        headers.setdefault("Authorization", f"Bearer {api_key}")
+    elif api_key and provider.type == "azure_openai":
+        headers.setdefault("api-key", api_key)
+
+    if provider.type == "ollama":
+        return await ollama_chat(
+            client,
+            provider.base_url,
+            model,
+            messages,
+            structured_mode,
+            headers,
+            timeout,
+        )
+
+    if provider.type == "dashscope":
+        if not api_key:
+            raise ProviderError("dashscope provider requires api_key")
+        return await dashscope_chat(api_key=api_key, model=model, messages=messages)
+
+    if provider.type == "gemini_ai_studio":
+        if not api_key:
+            raise ProviderError("gemini_ai_studio provider requires api_key")
+        return await gemini_ai_studio_chat(api_key=api_key, model=model, messages=messages)
+
+    if provider.type == "gemini_vertex":
+        if not provider.project_id or not provider.location:
+            raise ProviderError("gemini_vertex provider requires project_id and location")
+        return await gemini_vertex_chat(
+            project_id=provider.project_id,
+            location=provider.location,
+            credentials_path=provider.credentials_path,
+            model=model,
+            messages=messages,
+        )
+
+    if provider.type == "azure_openai":
+        if not api_key:
+            raise ProviderError("azure_openai provider requires api_key")
+        if not provider.deployment or not provider.api_version:
+            raise ProviderError("azure_openai provider requires deployment and api_version")
+        return await azure_openai_chat(
+            client,
+            provider.base_url,
+            provider.deployment,
+            provider.api_version,
+            messages,
+            structured_mode,
+            headers,
+            timeout,
+            schema,
+        )
+
+    if provider.type == "claude":
+        if not api_key:
+            raise ProviderError("claude provider requires api_key")
+        if not provider.anthropic_version:
+            raise ProviderError("claude provider requires anthropic_version")
+        return await claude_chat(
+            api_key=api_key,
+            model=model,
+            messages=messages,
+            anthropic_version=provider.anthropic_version,
+        )
+
+    if provider.type == "openai_compatible":
+        return await openai_chat(
+            client,
+            provider.base_url,
+            model,
+            messages,
+            structured_mode,
+            headers,
+            timeout,
+            schema,
+        )
+
+    raise ProviderError(f"Unsupported provider type: {provider.type}")

deepresearch_flow/paper/prompt_templates/__init__.py

@@ -0,0 +1 @@
+"""Built-in prompt templates."""

deepresearch_flow/paper/prompt_templates/deep_read_system.j2

@@ -0,0 +1,6 @@
+You are an information extraction assistant.
+Output language: {{ output_language }}.
+Extract structured data from the provided markdown document.
+Return ONLY valid JSON that conforms to the given JSON Schema.
+The field "paper_authors" MUST be an array of strings (each author name as one item).
+If a field is unknown, use an empty string or empty list per schema.

deepresearch_flow/paper/prompt_templates/deep_read_user.j2

@@ -0,0 +1,82 @@
+Template: deep_read
+
+{% if stage_name %}
+当前阶段：{{ stage_name }}。
+请仅输出 JSON，包含字段：{{ stage_fields | join(", ") }}。
+输出语言：{{ output_language }}。请使用该语言输出（zh 时使用中文，保留必要英文术语）。
+如需引用此前结果，可参考以下已完成模块（JSON）：{{ previous_outputs }}
+{% else %}
+注意：本工具是单次输出。请一次性完成所有模块内容，并填入 JSON 字段 module_a, module_b, module_c1..module_c8, module_d, module_e；可选 module_f, module_g, module_h 也请填入（若无法完成则填空字符串）。
+每个模块字段内部使用 Markdown 编排（标题、列表等）。
+禁止第一人称（不出现“我/我们/本人”）。
+关键结论需要注明来自论文的定位（章节/图/表/公式编号）。
+{% endif %}
+
+---
+
+# 论文精读 Prompt（多轮输出｜由浅入深｜术语表｜联网核验引用）
+
+你是一个高级语言模型，任务是帮助用户高效、可复现地阅读和理解一篇算法/工程研究论文。方法以 “How to Read a Paper: A Three-Pass Approach（Keshav）” 为骨架，并加入审稿式 checklist、术语解释与联网核验引用。
+
+## 0) 总体硬约束（必须遵守）
+1. 输出语言：全程使用 {{ output_language }}（保留必要英文术语/公式符号）。
+2. 写作人称：禁止第一人称。
+3. 输出形式：使用 Markdown；不省略推理过程；对关键结论给出“来自论文哪里”的定位（章节/图/表/公式编号）。
+4. 由浅入深：每个模块内按浅层/中层/深层组织。
+
+## 1) 模块表{% if not stage_name %}（本次单次输出需要全部完成）{% endif %}
+- 模块 A：阅读对齐与输入校验
+- 模块 B：第一遍鸟瞰扫描
+- 模块 C1~C8：八个问题逐个回答
+- 模块 D：第二遍内容理解
+- 模块 E：第三遍深度审视
+- 可选模块 F：术语表 + 概念依赖图
+- 可选模块 G：联网核验与引用校对
+- 模块 H：图表与表格的客观抽取
+
+## 2) 模块内容规范
+### 模块 A：阅读对齐与输入校验
+- 明确用户目的与背景
+- 检查输入材料：PDF/附录/代码链接/补充材料/数据集
+- 输出阅读路线图
+
+### 模块 B：第一遍（鸟瞰扫描）
+必须给出 5 项产出：类别、上下文、正确性直觉检查、贡献、清晰度，并给出继续/放弃建议。
+
+### 模块 C1~C8：八个问题
+问题1：论文试图解决什么问题？
+问题2：有哪些相关研究？（最强基线/最相近思路/正交方向）
+问题3：论文如何解决这个问题？（流程图文字版）
+问题4：实验数据集与实现设置：用了哪些数据集/场景？为什么选它们？验证哪些关键特性？
+问题5：指标与结果：有哪些指标？实验结果是什么？这些指标分别体现了哪些创新点？尽力抽取全部结果指标。
+问题6：有什么可以进一步探索的点？（理论/工程/数据/评测）
+问题7：现实场景如何体现/实现？（部署约束、算力、延迟、鲁棒性）
+问题8：总结主要内容（一段摘要 + 5 条要点 + 1 个核心公式/机制）
+
+### 模块 D：第二遍（内容理解）
+- 给出结构树
+- 拆解核心方法：输入/输出/模块/损失/训练或推理流程
+- 关键图表说明
+- 实验部分：数据集、指标、对比方法、公平性、消融充分性
+
+### 模块 E：第三遍（深度审视）
+必须包含：可复现重建（伪代码）、假设清单、失败模式、强弱点评估、下一步研究。
+
+### 模块 F：术语表（可选）
+- 15-40 个关键词，包含中英文名、定义、直觉解释、作用、前置知识
+- 术语依赖图（文字版）
+
+### 模块 G：联网核验（可选）
+- 关键事实提供至少 2 个独立来源
+- 冲突要显式说明，无法核验要标记
+- 输出引用列表
+
+### 模块 H：图表与表格抽取
+- 全量索引、打分、TOP 深读
+- 需要明确图表位置、图注、类型、支撑的 claim
+
+Document content:
+{{ content }}
+
+JSON Schema:
+{{ schema }}

deepresearch_flow/paper/prompt_templates/eight_questions_system.j2

@@ -0,0 +1,6 @@
+You are an information extraction assistant.
+Output language: {{ output_language }}.
+Extract structured data from the provided markdown document.
+Return ONLY valid JSON that conforms to the given JSON Schema.
+The field "paper_authors" MUST be an array of strings (each author name as one item).
+If a field is unknown, use an empty string or empty list per schema.

deepresearch_flow/paper/prompt_templates/eight_questions_user.j2

@@ -0,0 +1,28 @@
+Template: eight_questions
+
+{% if stage_name %}
+Current stage: {{ stage_name }}.
+Return JSON with keys: {{ stage_fields | join(", ") }}.
+Output language: {{ output_language }}. Use that language in all answers (if zh, use Chinese).
+Previously completed outputs (JSON): {{ previous_outputs }}
+{% else %}
+Answer each question in the corresponding JSON field question1 through question8.
+Use Markdown in each answer if helpful.
+Output language: {{ output_language }}. Use that language in all answers (if zh, use Chinese).
+{% endif %}
+
+Questions:
+1) What problem does the paper try to solve?
+2) What related research exists? (separate: strongest baseline, closest approach, orthogonal directions)
+3) How does the paper solve the problem? (include a text flowchart)
+4) What datasets and implementation settings are used? Why these datasets/scenarios, and which key properties do they validate?
+5) What are the metrics and results? List metrics, summarize results, and explain which innovation each metric reflects.
+6) What can be further explored? (split by theory/engineering/data/evaluation)
+7) How would this work in real-world scenarios? (deployment constraints, compute, latency, robustness)
+8) Summarize the main content (one paragraph + 5 bullets + one core formula/mechanism)
+
+Document content:
+{{ content }}
+
+JSON Schema:
+{{ schema }}

deepresearch_flow/paper/prompt_templates/simple_system.j2

@@ -0,0 +1,6 @@
+You are an information extraction assistant.
+Output language: {{ output_language }}.
+Extract structured data from the provided markdown document.
+Return ONLY valid JSON that conforms to the given JSON Schema.
+The field "paper_authors" MUST be an array of strings (each author name as one item).
+If a field is unknown, use an empty string or empty list per schema.

deepresearch_flow/paper/prompt_templates/simple_user.j2

@@ -0,0 +1,24 @@
+Template: simple
+
+{% if stage_name %}
+Current stage: {{ stage_name }}.
+Return JSON with keys: {{ stage_fields | join(", ") }}.
+Previously completed outputs (JSON): {{ previous_outputs }}
+{% else %}
+Write a single-paragraph summary that covers:
+- the problem being solved
+- related research context
+- the proposed solution
+- datasets and implementation settings (why chosen, what they validate)
+- metrics and results (what metrics, key results, which innovations they support)
+- further exploration directions
+- real-world scenarios and constraints
+- the main content summary with one core formula/mechanism
+Output language: {{ output_language }}. Use that language in all answers (if zh, use Chinese).
+{% endif %}
+
+Document content:
+{{ content }}
+
+JSON Schema:
+{{ schema }}

deepresearch_flow/paper/prompt_templates/three_pass_system.j2

@@ -0,0 +1,6 @@
+You are an information extraction assistant.
+Output language: {{ output_language }}.
+Extract structured data from the provided markdown document.
+Return ONLY valid JSON that conforms to the given JSON Schema.
+The field "paper_authors" MUST be an array of strings (each author name as one item).
+If a field is unknown, use an empty string or empty list per schema.

deepresearch_flow/paper/prompt_templates/three_pass_user.j2

@@ -0,0 +1,44 @@
+Template: three_pass
+
+{% if stage_name %}
+Current stage: {{ stage_name }}.
+Return JSON with keys: {{ stage_fields | join(", ") }}.
+Output language: {{ output_language }}. Use that language in all answers (if zh, use Chinese).
+Previously completed outputs (JSON): {{ previous_outputs }}
+{% else %}
+Follow the three-pass reading method. Fill the JSON fields:
+- step1_summary: "Step 1" output (overview + innovation highlights + structure + figures + conclusion + references + open source).
+- step2_analysis: "Step 2" output (content comprehension + experiments + evidence + related work weaknesses).
+- step3_analysis: "Step 3" output (reconstruction, assumptions, failure modes, strengths/weaknesses, future work).
+Use Markdown inside each field to format lists and sections.
+Output language: {{ output_language }}. Use that language in all answers (if zh, use Chinese).
+
+Step 1 guidance:
+- Summarize title/abstract/introduction.
+- List 3-5 innovations.
+- List authors and affiliations if available.
+- List section and subsection titles (translated if needed).
+- Summarize key figures/tables.
+- Summarize conclusions.
+- List key references.
+- State whether code is open-sourced and where.
+
+Step 2 guidance:
+- Explain each innovation: source, prior limitations, and solution details.
+- Analyze experiments: datasets, objectives, baselines, results, ablations.
+- Analyze key figures/tables.
+- Summarize important related work and weaknesses.
+
+Step 3 guidance:
+- Provide a reconstruction outline and pseudocode if possible.
+- Challenge assumptions and statements.
+- Identify failure modes and reasons.
+- Assess strengths/weaknesses and possible fixes.
+- Suggest future research directions.
+{% endif %}
+
+Document content:
+{{ content }}
+
+JSON Schema:
+{{ schema }}

deepresearch_flow/paper/prompts.py

@@ -0,0 +1,11 @@
+"""Prompt templates for paper extraction."""
+
+DEFAULT_SYSTEM_PROMPT = (
+    "You are an information extraction assistant. "
+    "Extract structured data from the provided markdown document. "
+    "Return ONLY valid JSON that conforms to the given JSON Schema. "
+    "The field 'paper_authors' MUST be an array of strings (each author name as one item). "
+    "If a field is unknown, use an empty string or empty list per schema." 
+)
+
+DEFAULT_USER_PROMPT = """Document content:\n{content}\n\nJSON Schema:\n{schema}\n"""

deepresearch_flow/paper/providers/__init__.py

@@ -0,0 +1 @@
+"""Provider implementations."""

deepresearch_flow/paper/providers/azure_openai.py

@@ -0,0 +1,66 @@
+"""Azure OpenAI provider implementation."""
+
+from __future__ import annotations
+
+from typing import Any
+import httpx
+
+from deepresearch_flow.paper.providers.base import ProviderError
+
+
+async def chat(
+    client: httpx.AsyncClient,
+    endpoint: str,
+    deployment: str,
+    api_version: str,
+    messages: list[dict[str, str]],
+    structured_mode: str,
+    headers: dict[str, str] | None,
+    timeout: float,
+    schema: dict[str, Any] | None = None,
+) -> str:
+    url = endpoint.rstrip("/") + f"/openai/deployments/{deployment}/chat/completions"
+    payload: dict[str, Any] = {
+        "messages": messages,
+        "temperature": 0,
+    }
+
+    if structured_mode == "json_schema" and schema is not None:
+        payload["response_format"] = {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "paper_extract",
+                "schema": schema,
+            },
+        }
+    elif structured_mode == "json_object":
+        payload["response_format"] = {"type": "json_object"}
+
+    try:
+        response = await client.post(
+            url,
+            json=payload,
+            params={"api-version": api_version},
+            headers=headers,
+            timeout=timeout,
+        )
+    except httpx.RequestError as exc:
+        raise ProviderError(str(exc), retryable=True) from exc
+
+    if response.status_code == 429:
+        raise ProviderError(response.text, status_code=429, retryable=True)
+    if response.status_code >= 500:
+        raise ProviderError(response.text, status_code=response.status_code, retryable=True)
+    if response.status_code >= 400:
+        structured_error = structured_mode in ("json_schema", "json_object")
+        raise ProviderError(response.text, status_code=response.status_code, structured_error=structured_error)
+
+    data = response.json()
+    choices = data.get("choices") or []
+    if not choices:
+        raise ProviderError("Azure OpenAI response missing choices", status_code=response.status_code)
+    message = choices[0].get("message", {})
+    content = message.get("content")
+    if not content:
+        raise ProviderError("Azure OpenAI response missing content", status_code=response.status_code)
+    return content

deepresearch_flow/paper/providers/base.py

@@ -0,0 +1,19 @@
+"""Provider base helpers."""
+
+from __future__ import annotations
+
+
+class ProviderError(RuntimeError):
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        retryable: bool = False,
+        structured_error: bool = False,
+        error_type: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.retryable = retryable
+        self.structured_error = structured_error
+        self.error_type = error_type or "provider_error"

deepresearch_flow/paper/providers/claude.py

@@ -0,0 +1,71 @@
+"""Claude provider implementation (Anthropic SDK)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from deepresearch_flow.paper.providers.base import ProviderError
+
+
+def _extract_text_blocks(content: Any) -> str | None:
+    if isinstance(content, str):
+        return content
+    if not content:
+        return None
+    parts: list[str] = []
+    for block in content:
+        if isinstance(block, str):
+            parts.append(block)
+            continue
+        text = getattr(block, "text", None)
+        if text:
+            parts.append(text)
+            continue
+        if isinstance(block, dict) and block.get("text"):
+            parts.append(str(block["text"]))
+    return "".join(parts) if parts else None
+
+
+async def chat(
+    api_key: str,
+    model: str,
+    messages: list[dict[str, str]],
+    anthropic_version: str,
+    max_tokens: int = 2048,
+) -> str:
+    try:
+        from anthropic import AsyncAnthropic
+    except ImportError as exc:
+        raise ProviderError("anthropic package is not installed") from exc
+
+    system_parts: list[str] = []
+    claude_messages: list[dict[str, str]] = []
+    for message in messages:
+        role = message.get("role")
+        content = message.get("content", "")
+        if role == "system":
+            system_parts.append(content)
+        else:
+            claude_messages.append({"role": role or "user", "content": content})
+
+    system_prompt = "\n\n".join(system_parts).strip() or None
+
+    client = AsyncAnthropic(
+        api_key=api_key,
+        default_headers={"anthropic-version": anthropic_version},
+    )
+
+    try:
+        response = await client.messages.create(
+            model=model,
+            max_tokens=max_tokens,
+            system=system_prompt,
+            messages=claude_messages,
+        )
+    except Exception as exc:  # pragma: no cover - SDK error
+        raise ProviderError(str(exc), retryable=True) from exc
+
+    content = _extract_text_blocks(response.content)
+    if not content:
+        raise ProviderError("Claude response missing content")
+    return content

deepresearch_flow/paper/providers/dashscope.py

@@ -0,0 +1,58 @@
+"""DashScope provider implementation."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from deepresearch_flow.paper.providers.base import ProviderError
+
+
+async def chat(
+    api_key: str,
+    model: str,
+    messages: list[dict[str, str]],
+) -> str:
+    try:
+        from dashscope.aigc.generation import AioGeneration, Message
+    except ImportError as exc:
+        raise ProviderError("dashscope package is not installed") from exc
+
+    ds_messages = [Message(role=item["role"], content=item["content"]) for item in messages]
+
+    try:
+        response = await AioGeneration.call(
+            api_key=api_key,
+            model=model,
+            messages=ds_messages,
+            result_format="message",
+        )
+    except Exception as exc:  # pragma: no cover - SDK error
+        raise ProviderError(str(exc), retryable=True) from exc
+
+    data = _normalize_response(response)
+    status_code = data.get("status_code")
+    if status_code and status_code != 200:
+        retryable = status_code in (429, 500, 502, 503, 504)
+        raise ProviderError(data.get("message") or "DashScope error", status_code=status_code, retryable=retryable)
+
+    content = (
+        data.get("output", {})
+        .get("choices", [{}])[0]
+        .get("message", {})
+        .get("content")
+    )
+    if not content:
+        raise ProviderError("DashScope response missing content")
+    return content
+
+
+def _normalize_response(response: Any) -> dict[str, Any]:
+    if isinstance(response, dict):
+        return response
+    if hasattr(response, "to_dict"):
+        return response.to_dict()
+    try:
+        return json.loads(str(response))
+    except json.JSONDecodeError:
+        return {"message": str(response)}

deepresearch_flow/paper/providers/gemini.py

@@ -0,0 +1,116 @@
+"""Gemini provider implementation (AI Studio and Vertex)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+
+from deepresearch_flow.paper.providers.base import ProviderError
+
+
+def _build_prompt(messages: list[dict[str, str]]) -> str:
+    parts: list[str] = []
+    for message in messages:
+        role = message.get("role", "user")
+        content = message.get("content", "")
+        parts.append(f"[{role}]\n{content}")
+    return "\n\n".join(parts)
+
+
+def _normalize_response(response: Any) -> dict[str, Any]:
+    if isinstance(response, dict):
+        return response
+    if hasattr(response, "to_dict"):
+        return response.to_dict()
+    try:
+        return json.loads(str(response))
+    except json.JSONDecodeError:
+        return {"message": str(response)}
+
+
+def _extract_text(response: Any) -> str | None:
+    text = getattr(response, "text", None)
+    if text:
+        return text
+
+    data = _normalize_response(response)
+    for candidate in data.get("candidates", []):
+        content = candidate.get("content", {})
+        parts = content.get("parts") or []
+        for part in parts:
+            part_text = part.get("text")
+            if part_text:
+                return part_text
+    return None
+
+
+async def chat_ai_studio(
+    api_key: str,
+    model: str,
+    messages: list[dict[str, str]],
+) -> str:
+    try:
+        from google import genai
+    except ImportError as exc:
+        raise ProviderError("google-genai package is not installed") from exc
+
+    prompt = _build_prompt(messages)
+    client = genai.Client(api_key=api_key)
+
+    try:
+        response = await asyncio.to_thread(
+            client.models.generate_content,
+            model=model,
+            contents=prompt,
+        )
+    except Exception as exc:  # pragma: no cover - SDK error
+        raise ProviderError(str(exc), retryable=True) from exc
+
+    content = _extract_text(response)
+    if not content:
+        raise ProviderError("Gemini response missing content")
+    return content
+
+
+async def chat_vertex(
+    project_id: str,
+    location: str,
+    credentials_path: str | None,
+    model: str,
+    messages: list[dict[str, str]],
+) -> str:
+    try:
+        from google import genai
+    except ImportError as exc:
+        raise ProviderError("google-genai package is not installed") from exc
+
+    credentials = None
+    if credentials_path:
+        try:
+            from google.auth import load_credentials_from_file
+        except ImportError as exc:
+            raise ProviderError("google-auth package is not installed") from exc
+        credentials, _ = load_credentials_from_file(credentials_path)
+
+    prompt = _build_prompt(messages)
+    client = genai.Client(
+        vertexai=True,
+        project=project_id,
+        location=location,
+        credentials=credentials,
+    )
+
+    try:
+        response = await asyncio.to_thread(
+            client.models.generate_content,
+            model=model,
+            contents=prompt,
+        )
+    except Exception as exc:  # pragma: no cover - SDK error
+        raise ProviderError(str(exc), retryable=True) from exc
+
+    content = _extract_text(response)
+    if not content:
+        raise ProviderError("Gemini response missing content")
+    return content