MaiBot 1.3.8__py3-none-any.whl

src/A_memorix/scripts/process_knowledge.py

@@ -0,0 +1,831 @@
+#!/usr/bin/env python3
+"""
+知识库自动导入脚本 (Strategy-Aware Version)
+
+功能：
+1. 扫描 A_Memorix 数据目录 imports/source/raw 下的 .txt 文件
+2. 检查 imports/manifest.json 确认是否已导入
+3. 使用 Strategy 模式处理文件 (Narrative/Factual/Quote)
+4. 将生成的数据直接存入 VectorStore/GraphStore/MetadataStore
+5. 更新 manifest
+"""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Optional
+import argparse
+import asyncio
+import hashlib
+import json
+import os
+import sys
+import time
+
+from rich.console import Console
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+import tomlkit
+
+from _bootstrap import DEFAULT_CONFIG_PATH, DEFAULT_DATA_DIR, resolve_repo_path
+
+console = Console()
+
+
+class LLMGenerationError(Exception):
+    pass
+
+
+def _build_arg_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="A_Memorix Knowledge Importer (Strategy-Aware)")
+    parser.add_argument("--data-dir", default=str(DEFAULT_DATA_DIR), help="A_Memorix 数据目录")
+    parser.add_argument("--force", action="store_true", help="Force re-import")
+    parser.add_argument("--clear-manifest", action="store_true", help="Clear manifest")
+    parser.add_argument(
+        "--type",
+        "-t",
+        default="auto",
+        help="Target import strategy override (auto/narrative/factual/quote)",
+    )
+    parser.add_argument("--concurrency", "-c", type=int, default=5)
+    parser.add_argument(
+        "--chat-log",
+        action="store_true",
+        help="聊天记录导入模式：强制 narrative 策略，并使用 LLM 语义抽取 event_time/event_time_range",
+    )
+    parser.add_argument(
+        "--chat-reference-time",
+        default=None,
+        help="chat_log 模式的相对时间参考点（如 2026/02/12 10:30）；不传则使用当前本地时间",
+    )
+    return parser
+
+
+# --help/-h 快速路径：避免加载较重的宿主和插件运行时
+if any(arg in {"-h", "--help"} for arg in sys.argv[1:]):
+    _build_arg_parser().print_help()
+    sys.exit(0)
+
+
+try:
+    import A_memorix.core as core_module
+    import A_memorix.core.storage as storage_module
+    from src.common.logger import get_logger
+    from src.services import llm_service as llm_api
+    from src.config.config import global_config
+
+    VectorStore = core_module.VectorStore
+    GraphStore = core_module.GraphStore
+    MetadataStore = core_module.MetadataStore
+    ImportStrategy = core_module.ImportStrategy
+    create_embedding_api_adapter = core_module.create_embedding_api_adapter
+    RelationWriteService = getattr(core_module, "RelationWriteService", None)
+
+    looks_like_quote_text = storage_module.looks_like_quote_text
+    parse_import_strategy = storage_module.parse_import_strategy
+    resolve_stored_knowledge_type = storage_module.resolve_stored_knowledge_type
+    select_import_strategy = storage_module.select_import_strategy
+
+    from A_memorix.core.utils.import_payloads import (
+        ImportPayloadValidationError,
+        is_probable_hash_token,
+        normalize_entity_import_item,
+        normalize_paragraph_import_item,
+        normalize_relation_import_item,
+    )
+    from A_memorix.core.utils.model_routing import (
+        ResolvedLLMModel,
+        generate_with_resolved_model,
+        get_text_generation_model_tasks,
+        pick_text_generation_task,
+        resolve_text_generation_model_selector,
+    )
+    from A_memorix.core.utils.time_parser import normalize_time_meta
+    from A_memorix.core.strategies.base import BaseStrategy, ProcessedChunk, KnowledgeType as StratKnowledgeType
+    from A_memorix.core.strategies.narrative import NarrativeStrategy
+    from A_memorix.core.strategies.factual import FactualStrategy
+    from A_memorix.core.strategies.quote import QuoteStrategy
+
+except ImportError as e:
+    print(f"❌ 无法导入模块: {e}")
+    import traceback
+
+    traceback.print_exc()
+    sys.exit(1)
+
+logger = get_logger("A_Memorix.AutoImport")
+
+
+def _log_before_retry(retry_state) -> None:
+    """使用项目统一日志风格记录重试信息。"""
+    exc = None
+    if getattr(retry_state, "outcome", None) is not None and retry_state.outcome.failed:
+        exc = retry_state.outcome.exception()
+    next_sleep = getattr(getattr(retry_state, "next_action", None), "sleep", None)
+    logger.warning(
+        f"LLM 调用即将重试: attempt={getattr(retry_state, 'attempt_number', '?')} next_sleep={next_sleep} error={exc}"
+    )
+
+
+class AutoImporter:
+    def __init__(
+        self,
+        force: bool = False,
+        clear_manifest: bool = False,
+        target_type: str = "auto",
+        concurrency: int = 5,
+        chat_log: bool = False,
+        chat_reference_time: Optional[str] = None,
+        data_dir: Optional[Path] = None,
+    ):
+        self.vector_store: Optional[VectorStore] = None
+        self.graph_store: Optional[GraphStore] = None
+        self.metadata_store: Optional[MetadataStore] = None
+        self.embedding_manager = None
+        self.relation_write_service = None
+        self.plugin_config = {}
+        self.manifest = {}
+        self.data_dir = resolve_repo_path(data_dir, fallback=DEFAULT_DATA_DIR)
+        self.import_dir = self.data_dir / "imports"
+        self.raw_dir = self.import_dir / "source" / "raw"
+        self.processed_dir = self.import_dir / "processed"
+        self.manifest_path = self.import_dir / "manifest.json"
+        self.force = force
+        self.clear_manifest = clear_manifest
+        self.chat_log = chat_log
+        parsed_target_type = parse_import_strategy(target_type, default=ImportStrategy.AUTO)
+        self.target_type = ImportStrategy.NARRATIVE.value if chat_log else parsed_target_type.value
+        self.chat_reference_dt = self._parse_reference_time(chat_reference_time)
+        if self.chat_log and parsed_target_type not in {ImportStrategy.AUTO, ImportStrategy.NARRATIVE}:
+            logger.warning(f"chat_log 模式已启用，target_type={target_type} 将被覆盖为 narrative")
+        self.concurrency_limit = concurrency
+        self.semaphore = None
+        self.storage_lock = None
+
+    async def initialize(self):
+        logger.info(f"正在初始化... (并发数: {self.concurrency_limit})")
+        self.semaphore = asyncio.Semaphore(self.concurrency_limit)
+        self.storage_lock = asyncio.Lock()
+
+        self.raw_dir.mkdir(parents=True, exist_ok=True)
+        self.processed_dir.mkdir(parents=True, exist_ok=True)
+        legacy_manifest_path = self.data_dir / "import_manifest.json"
+        if legacy_manifest_path.is_file() and not self.manifest_path.exists():
+            try:
+                os.replace(legacy_manifest_path, self.manifest_path)
+                logger.info(f"已迁移旧导入清单: {legacy_manifest_path} -> {self.manifest_path}")
+            except OSError as exc:
+                logger.warning(f"旧导入清单迁移失败，保留原文件继续执行: {legacy_manifest_path}, error={exc}")
+
+        if self.clear_manifest:
+            logger.info("🧹 清理 Mainfest")
+            self.manifest = {}
+            self._save_manifest()
+        elif self.manifest_path.exists():
+            try:
+                with open(self.manifest_path, "r", encoding="utf-8") as f:
+                    self.manifest = json.load(f)
+            except Exception:
+                self.manifest = {}
+
+        config_path = DEFAULT_CONFIG_PATH
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                self.plugin_config = tomlkit.load(f)
+        except Exception as e:
+            logger.error(f"加载 A_Memorix 配置失败: {e}")
+            return False
+
+        try:
+            await self._init_stores()
+        except Exception as e:
+            logger.error(f"初始化存储失败: {e}")
+            return False
+
+        return True
+
+    async def _init_stores(self):
+        # 使用与主运行时一致的存储初始化流程。
+        self.embedding_manager = create_embedding_api_adapter(
+            batch_size=self.plugin_config.get("embedding", {}).get("batch_size", 32),
+            default_dimension=self.plugin_config.get("embedding", {}).get("dimension", 384),
+            model_name=self.plugin_config.get("embedding", {}).get("model_name", "auto"),
+            dimension_request_mode=self.plugin_config.get("embedding", {}).get("dimension_request_mode", "explicit"),
+            retry_config=self.plugin_config.get("embedding", {}).get("retry", {}),
+        )
+        try:
+            dim = await self.embedding_manager._detect_dimension()
+        except Exception:
+            dim = self.embedding_manager.default_dimension
+
+        q_type_str = str(self.plugin_config.get("embedding", {}).get("quantization_type", "int8") or "int8").lower()
+        # QuantizationType 通过 storage_module 延迟获取，避免脚本启动阶段提前导入。
+        QuantizationType = storage_module.QuantizationType
+        if q_type_str != "int8":
+            raise ValueError(
+                "embedding.quantization_type 在 vNext 仅允许 int8(SQ8)。"
+                " 请先执行 scripts/release_vnext_migrate.py migrate。"
+            )
+
+        self.vector_store = VectorStore(
+            dimension=dim, quantization_type=QuantizationType.INT8, data_dir=self.data_dir / "vectors"
+        )
+
+        SparseMatrixFormat = storage_module.SparseMatrixFormat
+        m_fmt_str = self.plugin_config.get("graph", {}).get("sparse_matrix_format", "csr")
+        m_map = {"csr": SparseMatrixFormat.CSR, "csc": SparseMatrixFormat.CSC}
+
+        self.graph_store = GraphStore(
+            matrix_format=m_map.get(m_fmt_str, SparseMatrixFormat.CSR), data_dir=self.data_dir / "graph"
+        )
+
+        self.metadata_store = MetadataStore(data_dir=self.data_dir / "metadata")
+        self.metadata_store.connect()
+
+        if RelationWriteService is not None:
+            self.relation_write_service = RelationWriteService(
+                metadata_store=self.metadata_store,
+                graph_store=self.graph_store,
+                vector_store=self.vector_store,
+                embedding_manager=self.embedding_manager,
+            )
+
+        if self.vector_store.has_data():
+            self.vector_store.load()
+        if self.graph_store.has_data():
+            self.graph_store.load()
+
+    def _should_write_relation_vectors(self) -> bool:
+        retrieval_cfg = self.plugin_config.get("retrieval", {})
+        if not isinstance(retrieval_cfg, dict):
+            return False
+        rv_cfg = retrieval_cfg.get("relation_vectorization", {})
+        if not isinstance(rv_cfg, dict):
+            return False
+        return bool(rv_cfg.get("enabled", False)) and bool(rv_cfg.get("write_on_import", True))
+
+    def load_file(self, file_path: Path) -> str:
+        with open(file_path, "r", encoding="utf-8") as f:
+            return f.read()
+
+    def get_file_hash(self, content: str) -> str:
+        return hashlib.md5(content.encode("utf-8")).hexdigest()
+
+    def _parse_reference_time(self, value: Optional[str]) -> datetime:
+        """解析 chat_log 模式的参考时间（用于相对时间语义解析）。"""
+        if not value:
+            return datetime.now()
+        formats = [
+            "%Y/%m/%d %H:%M:%S",
+            "%Y/%m/%d %H:%M",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d %H:%M",
+            "%Y/%m/%d",
+            "%Y-%m-%d",
+        ]
+        text = str(value).strip()
+        for fmt in formats:
+            try:
+                return datetime.strptime(text, fmt)
+            except ValueError:
+                continue
+        logger.warning(f"无法解析 chat_reference_time={value}，将回退为当前本地时间")
+        return datetime.now()
+
+    async def _extract_chat_time_meta_with_llm(
+        self,
+        text: str,
+        *,
+        resolved_model: Any,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        使用 LLM 从聊天文本语义中抽取时间信息。
+        支持将相对时间表达转换为绝对时间。
+        """
+        if not text.strip():
+            return None
+
+        reference_now = self.chat_reference_dt.strftime("%Y/%m/%d %H:%M")
+        prompt = f"""You are a time extraction engine for chat logs.
+Extract temporal information from the following chat paragraph.
+
+Rules:
+1. Use semantic understanding, not regex matching.
+2. Convert relative expressions (e.g., yesterday evening, last Friday morning) to absolute local datetime using reference_now.
+3. If a time span exists, return event_time_start/event_time_end.
+4. If only one point in time exists, return event_time.
+5. If no reliable time can be inferred, return all time fields as null.
+6. Output ONLY valid JSON. No markdown, no explanation.
+
+reference_now: {reference_now}
+timezone: local system timezone
+
+Allowed output formats for time values:
+- "YYYY/MM/DD"
+- "YYYY/MM/DD HH:mm"
+
+JSON schema:
+{{
+  "event_time": null,
+  "event_time_start": null,
+  "event_time_end": null,
+  "time_range": null,
+  "time_granularity": "day",
+  "time_confidence": 0.0
+}}
+
+Chat paragraph:
+\"\"\"{text}\"\"\"
+"""
+        try:
+            result = await self._llm_call(prompt, resolved_model)
+        except Exception as e:
+            logger.warning(f"chat_log 时间语义抽取失败: {e}")
+            return None
+
+        if not isinstance(result, dict):
+            return None
+
+        raw_time_meta = {
+            "event_time": result.get("event_time"),
+            "event_time_start": result.get("event_time_start"),
+            "event_time_end": result.get("event_time_end"),
+            "time_range": result.get("time_range"),
+            "time_granularity": result.get("time_granularity"),
+            "time_confidence": result.get("time_confidence"),
+        }
+        try:
+            normalized = normalize_time_meta(raw_time_meta)
+        except Exception as e:
+            logger.warning(f"chat_log 时间语义抽取结果不可用，已忽略: {e}")
+            return None
+
+        has_effective_time = any(key in normalized for key in ("event_time", "event_time_start", "event_time_end"))
+        if not has_effective_time:
+            return None
+
+        return normalized
+
+    def _determine_strategy(self, filename: str, content: str) -> BaseStrategy:
+        """第一层：全局导入策略路由。"""
+        strategy = select_import_strategy(
+            content,
+            override=self.target_type,
+            chat_log=self.chat_log,
+        )
+        if self.chat_log:
+            logger.info(f"chat_log 模式: {filename} 强制使用 NarrativeStrategy")
+        elif strategy == ImportStrategy.QUOTE:
+            logger.info(f"Auto-detected Quote/Lyric type for {filename}")
+
+        if strategy == ImportStrategy.FACTUAL:
+            return FactualStrategy(filename)
+        if strategy == ImportStrategy.QUOTE:
+            return QuoteStrategy(filename)
+        return NarrativeStrategy(filename)
+
+    def _chunk_rescue(self, chunk: ProcessedChunk, filename: str) -> Optional[BaseStrategy]:
+        """第二层：分块级策略修正。"""
+        # Quote 分块已经满足目标格式，不再重复修正。
+        if chunk.type == StratKnowledgeType.QUOTE:
+            return None
+
+        if looks_like_quote_text(chunk.chunk.text):
+            logger.info(f"  > Rescuing chunk {chunk.chunk.index} as Quote")
+            return QuoteStrategy(filename)
+
+        return None
+
+    async def process_and_import(self):
+        if not await self.initialize():
+            return
+
+        files = list(self.raw_dir.glob("*.txt"))
+        logger.info(f"扫描到 {len(files)} 个文件 in {self.raw_dir}")
+
+        if not files:
+            return
+
+        tasks = []
+        for file_path in files:
+            tasks.append(asyncio.create_task(self._process_single_file(file_path)))
+
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        success_count = sum(1 for r in results if r is True)
+        logger.info(f"本次主处理完成，共成功处理 {success_count}/{len(files)} 个文件")
+
+        if self.vector_store:
+            self.vector_store.save()
+        if self.graph_store:
+            self.graph_store.save()
+
+    async def _process_single_file(self, file_path: Path) -> bool:
+        filename = file_path.name
+        async with self.semaphore:
+            try:
+                content = self.load_file(file_path)
+                file_hash = self.get_file_hash(content)
+
+                if not self.force and filename in self.manifest:
+                    record = self.manifest[filename]
+                    if record.get("hash") == file_hash and record.get("imported"):
+                        logger.info(f"跳过已导入文件: {filename}")
+                        return False
+
+                logger.info(f">>> 开始处理: {filename}")
+
+                # 1. 选择导入策略
+                strategy = self._determine_strategy(filename, content)
+                logger.info(f"  策略: {strategy.__class__.__name__}")
+
+                # 2. 按策略分块
+                initial_chunks = strategy.split(content)
+                logger.info(f"  初步分块: {len(initial_chunks)}")
+
+                processed_data = {"paragraphs": [], "entities": [], "relations": []}
+
+                # 3. 逐块抽取
+                resolved_model = await self._select_model()
+
+                for i, chunk in enumerate(initial_chunks):
+                    current_strategy = strategy
+                    # 第二层：分块级策略修正
+                    rescue_strategy = self._chunk_rescue(chunk, filename)
+                    if rescue_strategy:
+                        # 修正后不再重新切分，直接把当前分块作为完整 Quote 交给新策略。
+                        chunk.type = StratKnowledgeType.QUOTE
+                        chunk.flags.verbatim = True
+                        chunk.flags.requires_llm = False  # Quote 通常不需要 LLM
+                        current_strategy = rescue_strategy
+
+                    # 抽取内容
+                    if chunk.flags.requires_llm:
+                        result_chunk = await current_strategy.extract(
+                            chunk,
+                            lambda p: self._llm_call(p, resolved_model),
+                        )
+                    else:
+                        # QuoteStrategy 通常透传文本，并保留统一抽取接口。
+                        result_chunk = await current_strategy.extract(chunk)
+
+                    time_meta = None
+                    if self.chat_log:
+                        time_meta = await self._extract_chat_time_meta_with_llm(
+                            result_chunk.chunk.text,
+                            resolved_model=resolved_model,
+                        )
+
+                    # 归一化结果
+                    self._normalize_and_aggregate(
+                        result_chunk,
+                        processed_data,
+                        time_meta=time_meta,
+                    )
+
+                    logger.info(f"  已处理块 {i + 1}/{len(initial_chunks)}")
+
+                # 4. 保存 JSON
+                json_path = self.processed_dir / f"{file_path.stem}.json"
+                with open(json_path, "w", encoding="utf-8") as f:
+                    json.dump(processed_data, f, ensure_ascii=False, indent=2)
+
+                # 5. 导入数据库
+                async with self.storage_lock:
+                    await self._import_to_db(processed_data)
+
+                    self.manifest[filename] = {"hash": file_hash, "timestamp": time.time(), "imported": True}
+                    self._save_manifest()
+                    self.vector_store.save()
+                    self.graph_store.save()
+                    logger.info(f"✅ 文件 {filename} 处理并导入完成")
+                    return True
+
+            except Exception as e:
+                logger.error(f"❌ 处理失败 {filename}: {e}")
+                import traceback
+
+                traceback.print_exc()
+                return False
+
+    def _normalize_and_aggregate(
+        self,
+        chunk: ProcessedChunk,
+        all_data: Dict,
+        time_meta: Optional[Dict[str, Any]] = None,
+    ):
+        """将策略特有结果转换为存储层统一格式。"""
+        # 通用字段
+        para_item = {
+            "content": chunk.chunk.text,
+            "source": chunk.source.file,
+            "knowledge_type": resolve_stored_knowledge_type(
+                chunk.type.value,
+                content=chunk.chunk.text,
+            ).value,
+            "entities": [],
+            "relations": [],
+        }
+
+        data = chunk.data
+
+        # 1. 事实三元组（Factual）
+        if "triples" in data:
+            for t in data["triples"]:
+                para_item["relations"].append(
+                    {"subject": t.get("subject"), "predicate": t.get("predicate"), "object": t.get("object")}
+                )
+                # 自动收集三元组中的实体。
+                para_item["entities"].extend([t.get("subject"), t.get("object")])
+
+        # 2. 事件与关系（Narrative）
+        if "events" in data:
+            # 当前将事件作为实体写入，保留既有检索语义。
+            para_item["entities"].extend(data["events"])
+
+        if "relations" in data:  # Narrative 同时输出关系列表
+            para_item["relations"].extend(data["relations"])
+            for r in data["relations"]:
+                para_item["entities"].extend([r.get("subject"), r.get("object")])
+
+        # 3. 逐字实体（Quote）
+        if "verbatim_entities" in data:
+            para_item["entities"].extend(data["verbatim_entities"])
+
+        # 在每个段落内去重。
+        para_item["entities"] = list(set([e for e in para_item["entities"] if e]))
+
+        if time_meta:
+            para_item["time_meta"] = time_meta
+
+        all_data["paragraphs"].append(para_item)
+        all_data["entities"].extend(para_item["entities"])
+        if "relations" in para_item:
+            all_data["relations"].extend(para_item["relations"])
+
+    @retry(
+        retry=retry_if_exception_type((LLMGenerationError, json.JSONDecodeError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        before_sleep=_log_before_retry,
+    )
+    async def _llm_call(self, prompt: str, resolved_model: Any) -> Dict:
+        """统一的 LLM 调用入口。"""
+        result = await generate_with_resolved_model(
+            resolved_model,
+            request_type="Script.ProcessKnowledge",
+            prompt=prompt,
+            temperature=getattr(resolved_model.task_config, "temperature", None),
+            max_tokens=getattr(resolved_model.task_config, "max_tokens", None),
+        )
+        success = bool(result.success)
+        response = str(result.completion.response or "")
+        if success:
+            txt = response.strip()
+            if "```" in txt:
+                txt = txt.split("```json")[-1].split("```")[0].strip()
+            try:
+                return json.loads(txt)
+            except json.JSONDecodeError:
+                # 回退解析：截取首个左花括号到最后一个右花括号。
+                start = txt.find("{")
+                end = txt.rfind("}")
+                if start != -1 and end != -1:
+                    return json.loads(txt[start : end + 1])
+                raise
+        else:
+            raise LLMGenerationError("LLM generation failed")
+
+    async def _select_model(self) -> "ResolvedLLMModel":
+        models = get_text_generation_model_tasks(llm_api)
+        if not models:
+            raise ValueError("No LLM models")
+
+        config_model = str(self.plugin_config.get("advanced", {}).get("extraction_model", "auto") or "auto").strip()
+        if config_model != "auto":
+            task_name, task_config, selected_model_name = resolve_text_generation_model_selector(models, config_model)
+            if task_name and task_config:
+                return ResolvedLLMModel(
+                    task_name=task_name,
+                    task_config=task_config,
+                    selected_model_name=selected_model_name,
+                )
+            logger.warning(f"advanced.extraction_model={config_model!r} 不可用于文本生成，已回退自动选择")
+
+        task_name, task_config = pick_text_generation_task(
+            models,
+            preferred=("memory", "utils", "lpmm_entity_extract", "lpmm_rdf_build", "replyer", "planner"),
+        )
+        if task_name and task_config:
+            return ResolvedLLMModel(task_name=task_name, task_config=task_config)
+        raise ValueError("No LLM models")
+
+    # 复用现有写入方法。
+    async def _add_entity_with_vector(self, name: str, source_paragraph: Optional[str] = None) -> str:
+        # 最后一道守卫：防止旁路把 hash 写入实体名
+        entity_name = str(name or "").strip()
+        if not entity_name:
+            return ""
+        if is_probable_hash_token(entity_name):
+            logger.warning(f"脚本导入跳过疑似哈希实体: {entity_name[:32]}")
+            return ""
+
+        hash_value = self.metadata_store.add_entity(entity_name, source_paragraph=source_paragraph)
+        self.graph_store.add_nodes([entity_name])
+        try:
+            emb = await self.embedding_manager.encode(entity_name)
+            self.vector_store.add(emb.reshape(1, -1), [hash_value])
+        except (RuntimeError, ValueError) as exc:
+            logger.warning(f"实体向量写入失败: entity={entity_name}, hash={hash_value}, error={exc}")
+        return hash_value
+
+    async def import_json_data(self, data: Dict, filename: str = "script_import", progress_callback=None):
+        """预处理 JSON 载荷的公开导入入口。"""
+        if not self.storage_lock:
+            raise RuntimeError("Importer is not initialized. Call initialize() first.")
+
+        async with self.storage_lock:
+            await self._import_to_db(data, progress_callback=progress_callback)
+            self.manifest[filename] = {
+                "hash": self.get_file_hash(json.dumps(data, ensure_ascii=False, sort_keys=True)),
+                "timestamp": time.time(),
+                "imported": True,
+            }
+            self._save_manifest()
+            self.vector_store.save()
+            self.graph_store.save()
+
+    async def _import_to_db(self, data: Dict, progress_callback=None):
+        # 沿用统一导入逻辑，并逐项记录非法数据。
+        warning_count = 0
+
+        def append_warning(message: str) -> None:
+            nonlocal warning_count
+            warning_count += 1
+            logger.warning(message)
+
+        with self.graph_store.batch_update():
+            for paragraph_index, item in enumerate(data.get("paragraphs", [])):
+                try:
+                    paragraph = normalize_paragraph_import_item(
+                        item,
+                        default_source="script",
+                    )
+                except ImportPayloadValidationError as exc:
+                    append_warning(f"脚本导入跳过段落[{paragraph_index}]：{exc} (code={exc.code})")
+                    if progress_callback:
+                        progress_callback(1)
+                    continue
+
+                content = paragraph["content"]
+                if is_probable_hash_token(content):
+                    append_warning(f"脚本导入跳过段落[{paragraph_index}]：段落内容疑似哈希值")
+                    if progress_callback:
+                        progress_callback(1)
+                    continue
+
+                source = paragraph["source"]
+                k_type_val = paragraph["knowledge_type"]
+
+                h_val = self.metadata_store.add_paragraph(
+                    content=content,
+                    source=source,
+                    knowledge_type=k_type_val,
+                    time_meta=paragraph["time_meta"],
+                )
+
+                if h_val not in self.vector_store:
+                    try:
+                        emb = await self.embedding_manager.encode(content)
+                        self.vector_store.add(emb.reshape(1, -1), [h_val])
+                    except Exception as e:
+                        logger.error(f"  Vector fail: {e}")
+
+                para_entities = paragraph["entities"]
+                for entity in para_entities:
+                    name = normalize_entity_import_item(entity)
+                    if not name:
+                        append_warning(f"脚本导入跳过段落[{paragraph_index}]中的实体：无效名称或疑似哈希值")
+                        continue
+                    await self._add_entity_with_vector(name, source_paragraph=h_val)
+
+                para_relations = paragraph["relations"]
+                for rel in para_relations:
+                    normalized_relation = normalize_relation_import_item(rel)
+                    if normalized_relation is None:
+                        append_warning(f"脚本导入跳过段落[{paragraph_index}]中的关系：字段无效或疑似哈希值")
+                        continue
+
+                    s = normalized_relation["subject"]
+                    p = normalized_relation["predicate"]
+                    o = normalized_relation["object"]
+                    await self._add_entity_with_vector(s, source_paragraph=h_val)
+                    await self._add_entity_with_vector(o, source_paragraph=h_val)
+
+                    confidence_value = rel.get("confidence", 1.0) if isinstance(rel, dict) else 1.0
+                    confidence = float(1.0 if confidence_value is None else confidence_value)
+                    rel_meta = rel.get("metadata", {}) if isinstance(rel, dict) else {}
+                    write_vector = self._should_write_relation_vectors()
+                    if self.relation_write_service is not None:
+                        await self.relation_write_service.upsert_relation_with_vector(
+                            subject=s,
+                            predicate=p,
+                            obj=o,
+                            confidence=confidence,
+                            source_paragraph=h_val,
+                            metadata=rel_meta if isinstance(rel_meta, dict) else {},
+                            write_vector=write_vector,
+                        )
+                    else:
+                        rel_hash = self.metadata_store.add_relation(
+                            s,
+                            p,
+                            o,
+                            confidence=confidence,
+                            source_paragraph=h_val,
+                            metadata=rel_meta if isinstance(rel_meta, dict) else {},
+                        )
+                        self.graph_store.add_edges([(s, o)], relation_hashes=[rel_hash])
+
+                if progress_callback:
+                    progress_callback(1)
+
+            for entity_index, raw_entity in enumerate(data.get("entities", []) or []):
+                entity_name = normalize_entity_import_item(raw_entity)
+                if not entity_name:
+                    append_warning(f"脚本导入跳过顶层实体[{entity_index}]：无效名称或疑似哈希值")
+                    continue
+                await self._add_entity_with_vector(entity_name)
+
+            for relation_index, raw_relation in enumerate(data.get("relations", []) or []):
+                relation = normalize_relation_import_item(raw_relation)
+                if relation is None:
+                    append_warning(f"脚本导入跳过顶层关系[{relation_index}]：字段无效或疑似哈希值")
+                    continue
+
+                subject = relation["subject"]
+                predicate = relation["predicate"]
+                obj = relation["object"]
+                await self._add_entity_with_vector(subject)
+                await self._add_entity_with_vector(obj)
+
+                confidence_value = raw_relation.get("confidence", 1.0) if isinstance(raw_relation, dict) else 1.0
+                confidence = float(1.0 if confidence_value is None else confidence_value)
+                rel_meta = raw_relation.get("metadata", {}) if isinstance(raw_relation, dict) else {}
+                write_vector = self._should_write_relation_vectors()
+                if self.relation_write_service is not None:
+                    await self.relation_write_service.upsert_relation_with_vector(
+                        subject=subject,
+                        predicate=predicate,
+                        obj=obj,
+                        confidence=confidence,
+                        source_paragraph="",
+                        metadata=rel_meta if isinstance(rel_meta, dict) else {},
+                        write_vector=write_vector,
+                    )
+                else:
+                    rel_hash = self.metadata_store.add_relation(
+                        subject,
+                        predicate,
+                        obj,
+                        confidence=confidence,
+                        source_paragraph="",
+                        metadata=rel_meta if isinstance(rel_meta, dict) else {},
+                    )
+                    self.graph_store.add_edges([(subject, obj)], relation_hashes=[rel_hash])
+
+        if warning_count > 0:
+            logger.warning(f"脚本导入完成，跳过异常项 {warning_count} 条")
+
+    async def close(self):
+        if self.metadata_store:
+            self.metadata_store.close()
+
+    def _save_manifest(self):
+        self.manifest_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.manifest_path, "w", encoding="utf-8") as f:
+            json.dump(self.manifest, f, ensure_ascii=False, indent=2)
+
+
+async def main():
+    parser = _build_arg_parser()
+    args = parser.parse_args()
+
+    if not global_config:
+        return
+
+    importer = AutoImporter(
+        force=args.force,
+        clear_manifest=args.clear_manifest,
+        target_type=args.type,
+        concurrency=args.concurrency,
+        chat_log=args.chat_log,
+        chat_reference_time=args.chat_reference_time,
+        data_dir=Path(args.data_dir),
+    )
+    await importer.process_and_import()
+    await importer.close()
+
+
+if __name__ == "__main__":
+    if sys.platform == "win32":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+    asyncio.run(main())