b-sort 0.1.0__py3-none-any.whl

b_sort/blocks.py

@@ -0,0 +1,22 @@
+import json
+from pathlib import Path
+
+BLOCKS_FILENAME = "blocks.json"
+
+
+def load_blocks(root: Path | None = None) -> list:
+    """加载blocks"""
+    root = root or Path.cwd()
+    blocks_path = root / BLOCKS_FILENAME
+    if not blocks_path.exists():
+        return []
+
+    try:
+        overrides = json.loads(blocks_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+        raise TypeError(f"{BLOCKS_FILENAME} 不是合法 JSON: {exc}") from exc
+
+    if not isinstance(overrides, list):
+        raise TypeError(f"{BLOCKS_FILENAME} 顶层必须是 LIST 对象")
+
+    return overrides

b_sort/config.py

@@ -0,0 +1,76 @@
+import json
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from b_sort.states.sort_state import PipelineConfig, SortState
+
+
+def resolve_config(state: SortState) -> PipelineConfig:
+    """从图状态中取出 PipelineConfig,自动适配三种来源:
+
+    1. CLI 入口(`src/graphs/main.py`)通过 ``ainvoke({"config": config})`` 传入真实实例;
+    2. LangGraph Studio 把 ``__start__`` 的 raw input 作为 ``dict`` 透传,``config`` 也是 dict;
+    3. ``__start__`` 未传 ``config`` 时字段缺失。
+
+    第 2/3 种情况下回退到 ``PipelineConfig()`` 默认值;若 ``config`` 是 dict,
+    按 ``PipelineConfig.model_fields`` 校验字段并实例化,非法字段让 Pydantic 报错
+    上抛 —— 与 ``src/config.py:load_config`` 的语义一致。
+    """
+    value = state.get("config")
+    if isinstance(value, PipelineConfig):
+        return value
+    if value is None:
+        return PipelineConfig()
+    if isinstance(value, dict):
+        unknown = set(value) - set(PipelineConfig.model_fields)
+        if unknown:
+            raise ConfigError(f"{CONFIG_FILENAME} 包含未知字段: {sorted(unknown)}")
+        try:
+            return PipelineConfig(**value)
+        except ValidationError as exc:
+            raise ConfigError(f"{CONFIG_FILENAME} 字段非法: {exc}") from exc
+    raise ConfigError(
+        f"state['config'] 类型不受支持: {type(value).__name__},"
+        "应为 PipelineConfig 实例、dict 覆盖或 None"
+    )
+
+
+CONFIG_FILENAME = "config.json"
+
+
+class ConfigError(Exception):
+    """配置文件非法(非法 JSON / 字段类型错误 / 非法取值)。"""
+
+
+def load_config(root: Path | None = None) -> PipelineConfig:
+    """加载流水线配置。
+
+    - 根目录不存在 config.json 时,返回全默认配置;
+    - 存在时,仅覆盖文件中出现的字段;
+    - 文件非法时抛出 ConfigError,不静默回退默认值。
+
+    未知字段校验直接读 PipelineConfig.model_fields,故新增配置项(如 skills_dir /
+    cluster_skill / topics_skill)无需改动本函数即可被覆盖与校验。
+    """
+    root = root or Path.cwd()
+    config_path = root / CONFIG_FILENAME
+    if not config_path.exists():
+        return PipelineConfig()
+
+    try:
+        overrides = json.loads(config_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+        raise ConfigError(f"{CONFIG_FILENAME} 不是合法 JSON: {exc}") from exc
+
+    if not isinstance(overrides, dict):
+        raise ConfigError(f"{CONFIG_FILENAME} 顶层必须是 JSON 对象")
+
+    unknown = set(overrides) - set(PipelineConfig.model_fields)
+    if unknown:
+        raise ConfigError(f"{CONFIG_FILENAME} 包含未知字段: {sorted(unknown)}")
+
+    try:
+        return PipelineConfig(**overrides)
+    except ValidationError as exc:
+        raise ConfigError(f"{CONFIG_FILENAME} 字段非法: {exc}") from exc

b_sort/fault_tolerance.py

@@ -0,0 +1,71 @@
+from typing import Any
+
+from langgraph.errors import NodeError
+from langgraph.types import Command, RetryPolicy, TimeoutPolicy, default_retry_on
+from pydantic import ValidationError
+
+from b_sort.config import ConfigError
+from b_sort.skills import SkillError
+
+# 黑名单优先:重试 N 次也必然失败的确定性错误,直接进入降级路径
+_NON_RETRYABLE: tuple[type[Exception], ...] = (
+    ConfigError,
+    SkillError,
+    TypeError,
+    ImportError,
+    NameError,
+    SyntaxError,
+)
+
+# 项目白名单:default_retry_on 默认排除这些类型,但在本项目中它们恰是
+# 「LLM 输出不合格(ValidationError)/ 网络抖动(OSError)/ 结构化输出异常(RuntimeError)」
+# 这三类最该重试的瞬时失败
+_RETRYABLE: tuple[type[Exception], ...] = (
+    ValidationError,
+    RuntimeError,
+    OSError,
+    TimeoutError,
+)
+
+
+def pipeline_retry_on(exc: Exception) -> bool:
+    """先黑名单、再白名单、最后回落 default_retry_on(NodeTimeoutError 默认可重试)。"""
+    if isinstance(exc, _NON_RETRYABLE):
+        return False
+    if isinstance(exc, _RETRYABLE):
+        return True
+    return default_retry_on(exc)
+
+
+DEFAULT_RETRY = RetryPolicy(
+    max_attempts=2,
+    initial_interval=0.5,
+    backoff_factor=2.0,
+    max_interval=8.0,
+    jitter=True,
+    retry_on=pipeline_retry_on,
+)
+DEFAULT_TIMEOUT = TimeoutPolicy(run_timeout=30.0)
+
+
+LLM_RETRY = RetryPolicy(
+    max_attempts=3,
+    initial_interval=2.0,
+    backoff_factor=2.0,
+    max_interval=30.0,
+    jitter=True,
+    retry_on=pipeline_retry_on,
+)
+
+LLM_TIMEOUT = TimeoutPolicy(run_timeout=120.0)
+
+
+def make_degrade_handler(fallback: dict[str, Any], goto: str):
+    """降级处理器工厂:重试耗尽后写告警 + 安全空值产出,并路由到 goto(design D7)。"""
+
+    def degrade(state: dict[str, Any], error: NodeError) -> Command:
+        warning = f"[{error.node}] 重试耗尽后降级: {error.error}"
+        print(f"[fault][warn] {warning}")
+        return Command(update={**fallback, "warnings": [warning]}, goto=goto)
+
+    return degrade

b_sort/graphs/images_sort.py

@@ -0,0 +1,30 @@
+from langgraph.graph import END, START, StateGraph
+from langgraph.types import TimeoutPolicy
+
+from b_sort.fault_tolerance import (
+    DEFAULT_RETRY,
+    LLM_RETRY,
+    make_degrade_handler,
+)
+from b_sort.nodes.images_assign import images_assign
+from b_sort.states.sort_state import SortState
+
+builder = StateGraph(SortState)
+builder.set_node_defaults(retry_policy=DEFAULT_RETRY)
+
+
+builder.add_node(
+    "images_assign",
+    images_assign,
+    retry_policy=LLM_RETRY,
+    timeout=TimeoutPolicy(run_timeout=300.0),
+    error_handler=make_degrade_handler(
+        {"image_assigns": []}, goto=END
+    ),  # 错误handler后面考虑
+)
+
+
+builder.add_edge(START, "images_assign")
+builder.add_edge("images_assign", END)
+
+images_sort_graph = builder.compile()

b_sort/graphs/paper_cluster.py

@@ -0,0 +1,101 @@
+import asyncio
+
+from langgraph.graph import END, START, StateGraph
+
+from b_sort.fault_tolerance import (
+    DEFAULT_RETRY,
+    DEFAULT_TIMEOUT,
+    make_degrade_handler,
+)
+from b_sort.graphs.images_sort import images_sort_graph
+from b_sort.graphs.paper_sort import paper_sort
+from b_sort.graphs.texts_sort import texts_sort_graph
+from b_sort.nodes.blocks_split import blocks_split
+from b_sort.nodes.data_resolve import data_resolve
+from b_sort.nodes.output_file_json import output_file_json
+from b_sort.nodes.result_clusters import result_clusters
+from b_sort.nodes.texts_cluster_resolve import texts_cluster_resolve
+from b_sort.states.sort_state import SortState
+
+builder = StateGraph(SortState)
+builder.set_node_defaults(retry_policy=DEFAULT_RETRY)
+builder.add_node(
+    "data_resolve",
+    data_resolve,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler(
+        {
+            "blocks": [],
+        },
+        goto=END,
+    ),  # 错误handler后面考虑策略
+)
+builder.add_node(
+    "blocks_split",
+    blocks_split,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler(
+        {"texts": [], "images": []}, goto=END
+    ),  # 错误handler后面考虑策略
+)
+builder.add_node(
+    "texts_sort_graph",
+    texts_sort_graph,
+    error_handler=make_degrade_handler(
+        {"text_clusters": []}, goto=END
+    ),  # 错误handler后面考虑策略，也通过edge表现出来
+)
+builder.add_node(
+    "images_sort_graph",
+    images_sort_graph,
+    error_handler=make_degrade_handler({"image_assigns": []}, goto=END),
+)
+builder.add_node(
+    "texts_cluster_resolve",
+    texts_cluster_resolve,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({"text_clusters_block": []}, goto=END),
+)
+builder.add_node(
+    "result_clusters",
+    result_clusters,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({"result_clusters": []}, goto=END),
+)
+
+builder.add_node(
+    "output_file_json",
+    output_file_json,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({}, goto=END),
+)
+builder.add_edge(START, "data_resolve")
+builder.add_edge("data_resolve", "blocks_split")
+builder.add_edge("blocks_split", "texts_sort_graph")
+builder.add_edge("texts_sort_graph", "texts_cluster_resolve")
+builder.add_edge("texts_cluster_resolve", "images_sort_graph")
+builder.add_edge("images_sort_graph", "result_clusters")
+builder.add_edge("result_clusters", END)
+
+
+def build_agent(checkpointer):
+    """CLI 场景用:注入检查点存储,支持断点续跑。"""
+    return builder.compile(checkpointer=checkpointer)
+
+
+async def paper_cluster(blocks: list | None = None) -> SortState | None:
+    result = await paper_sort(blocks=blocks, build_call=build_agent)
+    return result if result else None
+
+
+if __name__ == "__main__":
+    from b_sort.blocks import load_blocks
+
+    blocks = load_blocks()
+    result = asyncio.run(paper_cluster(blocks=blocks))
+    print("[paper_cluster]", result)

b_sort/graphs/paper_sort.py

@@ -0,0 +1,176 @@
+from langgraph.graph import END, START, StateGraph
+
+from b_sort.fault_tolerance import (
+    DEFAULT_RETRY,
+    DEFAULT_TIMEOUT,
+    make_degrade_handler,
+)
+from b_sort.graphs.images_sort import images_sort_graph
+from b_sort.graphs.texts_sort import texts_sort_graph
+from b_sort.nodes.blocks_split import blocks_split
+from b_sort.nodes.data_resolve import data_resolve
+from b_sort.nodes.output_file_json import output_file_json
+from b_sort.nodes.result_clusters import result_clusters
+from b_sort.nodes.texts_cluster_resolve import texts_cluster_resolve
+from b_sort.states.sort_state import SortState
+
+builder = StateGraph(SortState)
+builder.set_node_defaults(retry_policy=DEFAULT_RETRY)
+builder.add_node(
+    "data_resolve",
+    data_resolve,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler(
+        {
+            "blocks": [],
+        },
+        goto=END,
+    ),  # 错误handler后面考虑策略
+)
+builder.add_node(
+    "blocks_split",
+    blocks_split,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler(
+        {"texts": [], "images": []}, goto=END
+    ),  # 错误handler后面考虑策略
+)
+builder.add_node(
+    "texts_sort_graph",
+    texts_sort_graph,
+    error_handler=make_degrade_handler(
+        {"text_clusters": []}, goto=END
+    ),  # 错误handler后面考虑策略，也通过edge表现出来
+)
+builder.add_node(
+    "images_sort_graph",
+    images_sort_graph,
+    error_handler=make_degrade_handler({"image_assigns": []}, goto=END),
+)
+builder.add_node(
+    "texts_cluster_resolve",
+    texts_cluster_resolve,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({"text_clusters_block": []}, goto=END),
+)
+builder.add_node(
+    "result_clusters",
+    result_clusters,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({"result_clusters": []}, goto=END),
+)
+
+builder.add_node(
+    "output_file_json",
+    output_file_json,
+    retry_policy=DEFAULT_RETRY,
+    timeout=DEFAULT_TIMEOUT,
+    error_handler=make_degrade_handler({}, goto=END),
+)
+builder.add_edge(START, "data_resolve")
+builder.add_edge("data_resolve", "blocks_split")
+builder.add_edge("blocks_split", "texts_sort_graph")
+builder.add_edge("texts_sort_graph", "texts_cluster_resolve")
+builder.add_edge("texts_cluster_resolve", "images_sort_graph")
+builder.add_edge("images_sort_graph", "result_clusters")
+builder.add_edge("result_clusters", "output_file_json")
+builder.add_edge("output_file_json", END)
+
+agent = builder.compile()
+
+import asyncio
+import os
+import signal
+from datetime import UTC, datetime
+
+from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+from langgraph.errors import GraphDrained
+from langgraph.runtime import RunControl
+
+from b_sort.blocks import load_blocks
+from b_sort.config import load_config
+from b_sort.skills import load_skill
+
+
+def _resolve_thread_id(config) -> str:
+    """优先 PipelineConfig.thread_id,其次环境变量 TOPIC_THREAD_ID,回落当日日期。"""
+    if config.thread_id:
+        return config.thread_id
+    # 使用本地时间，精确到秒
+    now = datetime.now(tz=UTC).astimezone().strftime("%Y-%m-%d-%H-%M-%S")
+    return os.getenv("TOPIC_THREAD_ID") or f"paper-sort-{now}"
+
+
+def build_agent(checkpointer):
+    """CLI 场景用:注入检查点存储,支持断点续跑。"""
+    return builder.compile(checkpointer=checkpointer)
+
+
+from collections.abc import Callable
+
+from langgraph.graph.state import CompiledStateGraph
+
+from b_sort.states.sort_state import PipelineConfig
+
+
+async def paper_sort(
+    blocks: list | None = None,
+    config: PipelineConfig | None = None,
+    build_call: Callable[[AsyncSqliteSaver], CompiledStateGraph] | None = None,
+) -> SortState | None:
+    if config is None:
+        config = load_config()
+    if blocks is None:
+        blocks = load_blocks()
+
+    skill = load_skill(config.skills_dir, config.texts_cluster_skill)
+    print(f"[main] skill 就绪 {skill.name}({len(skill.prompt_block)} 字符)")
+
+    thread_id = _resolve_thread_id(config)
+    print(f"[main] thread_id={thread_id} 检查点库={config.checkpoint_db}")
+
+    control = RunControl()
+
+    def _on_signal(signum, frame):
+        control.request_drain(f"收到信号 {signal.Signals(signum).name}")
+
+    signal.signal(signal.SIGTERM, _on_signal)
+    signal.signal(signal.SIGINT, _on_signal)
+
+    async with AsyncSqliteSaver.from_conn_string(config.checkpoint_db) as saver:
+        if build_call is None:
+            graph = build_agent(saver)
+        else:
+            graph = build_call(saver)
+        run_config = {"configurable": {"thread_id": thread_id}}
+        snapshot = await graph.aget_state(run_config)
+        if snapshot.values:
+            # 命中已有检查点:传 None 续跑(已跑完则直接返回旧结果,design D9)
+            pending = ", ".join(snapshot.next) if snapshot.next else "无(已跑完)"
+            print(
+                f"[main] 命中已有检查点(待执行: {pending}),续跑该会话;"
+                "如需强制重跑请设置 TOPIC_THREAD_ID"
+            )
+            input_data = None
+        else:
+            input_data = {"config": config, "blocks": blocks}
+
+        try:
+            result = await graph.ainvoke(input_data, config=run_config, control=control)
+        except GraphDrained:
+            print(f"[main] 优雅停机: {control.drain_reason}")
+            print(f"[main] 检查点已保存,保持 thread_id={thread_id} 重跑即可续跑")
+            return None
+
+        if control.drain_requested:
+            print(f"[main] 停机请求({control.drain_reason})到达时流程恰好已完成")
+        print("[main] 完成")
+        return result
+
+
+if __name__ == "__main__":
+    asyncio.run(paper_sort())

b_sort/graphs/texts_sort.py

@@ -0,0 +1,30 @@
+from langgraph.graph import END, START, StateGraph
+from langgraph.types import TimeoutPolicy
+
+from b_sort.fault_tolerance import (
+    DEFAULT_RETRY,
+    LLM_RETRY,
+    make_degrade_handler,
+)
+from b_sort.nodes.texts_cluster import texts_cluster
+from b_sort.states.sort_state import SortState
+
+builder = StateGraph(SortState)
+builder.set_node_defaults(retry_policy=DEFAULT_RETRY)
+
+
+builder.add_node(
+    "texts_cluster",
+    texts_cluster,
+    retry_policy=LLM_RETRY,
+    timeout=TimeoutPolicy(run_timeout=300.0),
+    error_handler=make_degrade_handler(
+        {"text_clusters": []}, goto=END
+    ),  # 错误handler后面考虑
+)
+
+
+builder.add_edge(START, "texts_cluster")
+builder.add_edge("texts_cluster", END)
+
+texts_sort_graph = builder.compile()

b_sort/llm.py

@@ -0,0 +1,55 @@
+"""LLM 模型工厂:ChatOpenAI 指向阿里云 DashScope 兼容模式。"""
+
+import os
+
+from langchain_core.messages import BaseMessage
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel
+
+DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+
+# SDK 客户端超时取节点 run_timeout(120s)的约 80%,让 SDK 先于节点超时
+# 抛出可归因的网络错误
+# CLIENT_TIMEOUT_SECONDS = 96
+
+
+def make_chat_model(model: str) -> ChatOpenAI:
+    """创建指向 DashScope 兼容模式的模型,API key 只读 BL_TOKEN 环境变量。
+
+    max_retries=0:重试职责单一归属图层 RetryPolicy,避免 SDK 重试叠乘(design D4)。
+    """
+    api_key = os.getenv("BL_TOKEN", "").strip()
+    if not api_key:
+        raise RuntimeError("环境变量 BL_TOKEN 未设置,无法调用 LLM")
+    return ChatOpenAI(
+        api_key=api_key,
+        base_url=DASHSCOPE_BASE_URL,
+        model=model,
+        # temperature=0.3,
+        # timeout=CLIENT_TIMEOUT_SECONDS,
+        # max_retries=0,
+    )
+
+
+Prompt = str | list[BaseMessage]
+from typing import TypeVar
+
+T = TypeVar("T", bound=BaseModel)
+
+
+async def ainvoke_structured(model_name: str, schema: type[T], prompt: Prompt) -> T:
+    """structured output 单次调用;失败直接抛异常,重试由图层 RetryPolicy 承担。"""
+    model = make_chat_model(model_name).with_structured_output(schema)
+    result = await model.ainvoke(prompt)
+
+    # model = make_chat_model(model_name)
+
+    # async for chunk in model.astream(prompt):
+    #     print(chunk.content, end="", flush=True)
+
+    if not isinstance(result, schema):
+        # TRY004 不适用:TypeError 在重试黑名单内,此处需 RuntimeError 走图层重试
+        raise RuntimeError(  # noqa: TRY004
+            f"{model_name} 结构化输出类型异常: {type(result)}"
+        )
+    return result

b_sort/nodes/blocks_split.py

@@ -0,0 +1,23 @@
+from b_sort.states.sort_state import BlockImage, BlockText, SortState
+
+
+async def blocks_split(state: SortState) -> SortState:
+    blocks = state["blocks"]
+
+    texts: list[BlockText] = []
+    images: list[BlockImage] = []
+
+    print("[blocks_split] start")
+    for block in blocks:
+        if block.type == "text":
+            texts.append(block)
+        elif block.type == "image":
+            images.append(block)
+        else:
+            raise ValueError(f"Illegal block: {block}")
+
+    print("[blocks_split] end")
+    return {
+        "texts": texts,
+        "images": images,
+    }

b_sort/nodes/data_resolve.py

@@ -0,0 +1,28 @@
+from b_sort.config import resolve_config
+from b_sort.states.sort_state import BlockImage, BlockText, SortState
+
+
+async def data_resolve(state: SortState) -> SortState:
+    blocks = state["blocks"]
+
+    loaded_blocks: list[BlockText | BlockImage] = []
+
+    print("[data_resolve] start")
+    for block in blocks:
+        if isinstance(block, dict):
+            if block["type"] == "text":
+                block = BlockText.model_validate(block)
+            elif block["type"] == "image":
+                block = BlockImage.model_validate(block)
+            else:
+                raise ValueError(f"Illegal block: {block}")
+
+        loaded_blocks.append(block)
+
+    config = resolve_config(state)
+
+    print("[data_resolve] end")
+    return {
+        "config": config,
+        "blocks": loaded_blocks,
+    }