AstrBot 4.3.5__py3-none-any.whl → 4.5.1__py3-none-any.whl

astrbot/core/db/vec_db/faiss_impl/vec_db.py

@@ -1,11 +1,12 @@
 import uuid
-import json
+import time
 import numpy as np
 from .document_storage import DocumentStorage
 from .embedding_storage import EmbeddingStorage
 from ..base import Result, BaseVecDB
 from astrbot.core.provider.provider import EmbeddingProvider
 from astrbot.core.provider.provider import RerankProvider
+from astrbot import logger
 
 
 class FaissVecDB(BaseVecDB):
@@ -44,18 +45,56 @@ class FaissVecDB(BaseVecDB):
 
         vector = await self.embedding_provider.get_embedding(content)
         vector = np.array(vector, dtype=np.float32)
-        async with self.document_storage.connection.cursor() as cursor:
-            await cursor.execute(
-                "INSERT INTO documents (doc_id, text, metadata) VALUES (?, ?, ?)",
-                (str_id, content, json.dumps(metadata)),
-            )
-            await self.document_storage.connection.commit()
-            result = await self.document_storage.get_document_by_doc_id(str_id)
-            int_id = result["id"]
 
-            # 插入向量到 FAISS
-            await self.embedding_storage.insert(vector, int_id)
-            return int_id
+        # 使用 DocumentStorage 的方法插入文档
+        int_id = await self.document_storage.insert_document(str_id, content, metadata)
+
+        # 插入向量到 FAISS
+        await self.embedding_storage.insert(vector, int_id)
+        return int_id
+
+    async def insert_batch(
+        self,
+        contents: list[str],
+        metadatas: list[dict] | None = None,
+        ids: list[str] | None = None,
+        batch_size: int = 32,
+        tasks_limit: int = 3,
+        max_retries: int = 3,
+        progress_callback=None,
+    ) -> list[int]:
+        """
+        批量插入文本和其对应向量，自动生成 ID 并保持一致性。
+
+        Args:
+            progress_callback: 进度回调函数，接收参数 (current, total)
+        """
+        metadatas = metadatas or [{} for _ in contents]
+        ids = ids or [str(uuid.uuid4()) for _ in contents]
+
+        start = time.time()
+        logger.debug(f"Generating embeddings for {len(contents)} contents...")
+        vectors = await self.embedding_provider.get_embeddings_batch(
+            contents,
+            batch_size=batch_size,
+            tasks_limit=tasks_limit,
+            max_retries=max_retries,
+            progress_callback=progress_callback,
+        )
+        end = time.time()
+        logger.debug(
+            f"Generated embeddings for {len(contents)} contents in {end - start:.2f} seconds."
+        )
+
+        # 使用 DocumentStorage 的批量插入方法
+        int_ids = await self.document_storage.insert_documents_batch(
+            ids, contents, metadatas
+        )
+
+        # 批量插入向量到 FAISS
+        vectors_array = np.array(vectors).astype("float32")
+        await self.embedding_storage.insert_batch(vectors_array, int_ids)
+        return int_ids
 
     async def retrieve(
         self,
@@ -119,23 +158,42 @@ class FaissVecDB(BaseVecDB):
 
         return top_k_results
 
-    async def delete(self, doc_id: int):
+    async def delete(self, doc_id: str):
         """
-        删除一条文档
+        删除一条文档块（chunk）
         """
-        await self.document_storage.connection.execute(
-            "DELETE FROM documents WHERE doc_id = ?", (doc_id,)
-        )
-        await self.document_storage.connection.commit()
+        # 获得对应的 int id
+        result = await self.document_storage.get_document_by_doc_id(doc_id)
+        int_id = result["id"] if result else None
+        if int_id is None:
+            return
+
+        # 使用 DocumentStorage 的删除方法
+        await self.document_storage.delete_document_by_doc_id(doc_id)
+        await self.embedding_storage.delete([int_id])
 
     async def close(self):
         await self.document_storage.close()
 
-    async def count_documents(self) -> int:
+    async def count_documents(self, metadata_filter: dict | None = None) -> int:
         """
         计算文档数量
+
+        Args:
+            metadata_filter (dict | None): 元数据过滤器
         """
-        async with self.document_storage.connection.cursor() as cursor:
-            await cursor.execute("SELECT COUNT(*) FROM documents")
-            count = await cursor.fetchone()
-            return count[0] if count else 0
+        count = await self.document_storage.count_documents(
+            metadata_filters=metadata_filter or {}
+        )
+        return count
+
+    async def delete_documents(self, metadata_filters: dict):
+        """
+        根据元数据过滤器删除文档
+        """
+        docs = await self.document_storage.get_documents(
+            metadata_filters=metadata_filters, offset=None, limit=None
+        )
+        doc_ids: list[int] = [doc["id"] for doc in docs]
+        await self.embedding_storage.delete(doc_ids)
+        await self.document_storage.delete_documents(metadata_filters=metadata_filters)

astrbot/core/file_token_service.py

@@ -23,7 +23,12 @@ class FileTokenService:
         for token in expired_tokens:
             self.staged_files.pop(token, None)
 
-    async def register_file(self, file_path: str, timeout: float = None) -> str:
+    async def check_token_expired(self, file_token: str) -> bool:
+        async with self.lock:
+            await self._cleanup_expired_tokens()
+            return file_token not in self.staged_files
+
+    async def register_file(self, file_path: str, timeout: float | None = None) -> str:
         """向令牌服务注册一个文件。
 
         Args:

astrbot/core/initial_loader.py

@@ -41,10 +41,13 @@ class InitialLoader:
         self.dashboard_server = AstrBotDashboard(
             core_lifecycle, self.db, core_lifecycle.dashboard_shutdown_event, webui_dir
         )
-        task = asyncio.gather(
-            core_task, self.dashboard_server.run()
-        )  # 启动核心任务和仪表板服务器
 
+        coro = self.dashboard_server.run()
+        if coro:
+            # 启动核心任务和仪表板服务器
+            task = asyncio.gather(core_task, coro)
+        else:
+            task = core_task
         try:
             await task  # 整个AstrBot在这里运行
         except asyncio.CancelledError:

astrbot/core/knowledge_base/chunking/__init__.py

@@ -0,0 +1,11 @@
+"""
+文档分块模块
+"""
+
+from .base import BaseChunker
+from .fixed_size import FixedSizeChunker
+
+__all__ = [
+    "BaseChunker",
+    "FixedSizeChunker",
+]

astrbot/core/knowledge_base/chunking/base.py

@@ -0,0 +1,24 @@
+"""文档分块器基类
+
+定义了文档分块处理的抽象接口。
+"""
+
+from abc import ABC, abstractmethod
+
+
+class BaseChunker(ABC):
+    """分块器基类
+
+    所有分块器都应该继承此类并实现 chunk 方法。
+    """
+
+    @abstractmethod
+    async def chunk(self, text: str, **kwargs) -> list[str]:
+        """将文本分块
+
+        Args:
+            text: 输入文本
+
+        Returns:
+            list[str]: 分块后的文本列表
+        """

astrbot/core/knowledge_base/chunking/fixed_size.py

@@ -0,0 +1,57 @@
+"""固定大小分块器
+
+按照固定的字符数将文本分块,支持重叠区域。
+"""
+
+from .base import BaseChunker
+
+
+class FixedSizeChunker(BaseChunker):
+    """固定大小分块器
+
+    按照固定的字符数分块,并支持块之间的重叠。
+    """
+
+    def __init__(self, chunk_size: int = 512, chunk_overlap: int = 50):
+        """初始化分块器
+
+        Args:
+            chunk_size: 块的大小(字符数)
+            chunk_overlap: 块之间的重叠字符数
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+
+    async def chunk(self, text: str, **kwargs) -> list[str]:
+        """固定大小分块
+
+        Args:
+            text: 输入文本
+            chunk_size: 每个文本块的最大大小
+            chunk_overlap: 每个文本块之间的重叠部分大小
+
+        Returns:
+            list[str]: 分块后的文本列表
+        """
+        chunk_size = kwargs.get("chunk_size", self.chunk_size)
+        chunk_overlap = kwargs.get("chunk_overlap", self.chunk_overlap)
+
+        chunks = []
+        start = 0
+        text_len = len(text)
+
+        while start < text_len:
+            end = start + chunk_size
+            chunk = text[start:end]
+
+            if chunk:
+                chunks.append(chunk)
+
+            # 移动窗口,保留重叠部分
+            start = end - chunk_overlap
+
+            # 防止无限循环: 如果重叠过大,直接移到end
+            if start >= end or chunk_overlap >= chunk_size:
+                start = end
+
+        return chunks

astrbot/core/knowledge_base/chunking/recursive.py

@@ -0,0 +1,155 @@
+from collections.abc import Callable
+from .base import BaseChunker
+
+
+class RecursiveCharacterChunker(BaseChunker):
+    def __init__(
+        self,
+        chunk_size: int = 500,
+        chunk_overlap: int = 100,
+        length_function: Callable[[str], int] = len,
+        is_separator_regex: bool = False,
+        separators: list[str] | None = None,
+    ):
+        """
+        初始化递归字符文本分割器
+
+        Args:
+            chunk_size: 每个文本块的最大大小
+            chunk_overlap: 每个文本块之间的重叠部分大小
+            length_function: 计算文本长度的函数
+            is_separator_regex: 分隔符是否为正则表达式
+            separators: 用于分割文本的分隔符列表，按优先级排序
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.length_function = length_function
+        self.is_separator_regex = is_separator_regex
+
+        # 默认分隔符列表，按优先级从高到低
+        self.separators = separators or [
+            "\n\n",  # 段落
+            "\n",  # 换行
+            "。",  # 中文句子
+            "，",  # 中文逗号
+            ". ",  # 句子
+            ", ",  # 逗号分隔
+            " ",  # 单词
+            "",  # 字符
+        ]
+
+    async def chunk(self, text: str, **kwargs) -> list[str]:
+        """
+        递归地将文本分割成块
+
+        Args:
+            text: 要分割的文本
+            chunk_size: 每个文本块的最大大小
+            chunk_overlap: 每个文本块之间的重叠部分大小
+
+        Returns:
+            分割后的文本块列表
+        """
+        if not text:
+            return []
+
+        overlap = kwargs.get("chunk_overlap", self.chunk_overlap)
+        chunk_size = kwargs.get("chunk_size", self.chunk_size)
+
+        text_length = self.length_function(text)
+        if text_length <= chunk_size:
+            return [text]
+
+        for separator in self.separators:
+            if separator == "":
+                return self._split_by_character(text, chunk_size, overlap)
+
+            if separator in text:
+                splits = text.split(separator)
+                # 重新添加分隔符（除了最后一个片段）
+                splits = [s + separator for s in splits[:-1]] + [splits[-1]]
+                splits = [s for s in splits if s]
+                if len(splits) == 1:
+                    continue
+
+                # 递归合并分割后的文本块
+                final_chunks = []
+                current_chunk = []
+                current_chunk_length = 0
+
+                for split in splits:
+                    split_length = self.length_function(split)
+
+                    # 如果单个分割部分已经超过了chunk_size，需要递归分割
+                    if split_length > chunk_size:
+                        # 先处理当前积累的块
+                        if current_chunk:
+                            combined_text = "".join(current_chunk)
+                            final_chunks.extend(
+                                await self.chunk(
+                                    combined_text,
+                                    chunk_size=chunk_size,
+                                    chunk_overlap=overlap,
+                                )
+                            )
+                            current_chunk = []
+                            current_chunk_length = 0
+
+                        # 递归分割过大的部分
+                        final_chunks.extend(
+                            await self.chunk(
+                                split, chunk_size=chunk_size, chunk_overlap=overlap
+                            )
+                        )
+                    # 如果添加这部分会使当前块超过chunk_size
+                    elif current_chunk_length + split_length > chunk_size:
+                        # 合并当前块并添加到结果中
+                        combined_text = "".join(current_chunk)
+                        final_chunks.append(combined_text)
+
+                        # 处理重叠部分
+                        overlap_start = max(0, len(combined_text) - overlap)
+                        if overlap_start > 0:
+                            overlap_text = combined_text[overlap_start:]
+                            current_chunk = [overlap_text, split]
+                            current_chunk_length = (
+                                self.length_function(overlap_text) + split_length
+                            )
+                        else:
+                            current_chunk = [split]
+                            current_chunk_length = split_length
+                    else:
+                        # 添加到当前块
+                        current_chunk.append(split)
+                        current_chunk_length += split_length
+
+                # 处理剩余的块
+                if current_chunk:
+                    final_chunks.append("".join(current_chunk))
+
+                return final_chunks
+
+        return [text]
+
+    def _split_by_character(
+        self, text: str, chunk_size: int | None = None, overlap: int | None = None
+    ) -> list[str]:
+        """
+        按字符级别分割文本
+
+        Args:
+            text: 要分割的文本
+
+        Returns:
+            分割后的文本块列表
+        """
+        chunk_size = chunk_size or self.chunk_size
+        overlap = overlap or self.chunk_overlap
+        result = []
+        for i in range(0, len(text), chunk_size - overlap):
+            end = min(i + chunk_size, len(text))
+            result.append(text[i:end])
+            if end == len(text):
+                break
+
+        return result