AstrBot 3.5.6__py3-none-any.whl → 4.7.0__py3-none-any.whl

astrbot/core/db/migration/migra_3_to_4.py

@@ -0,0 +1,357 @@
+import datetime
+import json
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from astrbot.api import logger, sp
+from astrbot.core.config import AstrBotConfig
+from astrbot.core.config.default import DB_PATH
+from astrbot.core.db.po import ConversationV2, PlatformMessageHistory
+from astrbot.core.platform.astr_message_event import MessageSesion
+
+from .. import BaseDatabase
+from .shared_preferences_v3 import sp as sp_v3
+from .sqlite_v3 import SQLiteDatabase as SQLiteV3DatabaseV3
+
+"""
+1. 迁移旧的 webchat_conversation 表到新的 conversation 表。
+2. 迁移旧的 platform 到新的 platform_stats 表。
+"""
+
+
+def get_platform_id(
+    platform_id_map: dict[str, dict[str, str]],
+    old_platform_name: str,
+) -> str:
+    return platform_id_map.get(
+        old_platform_name,
+        {"platform_id": old_platform_name, "platform_type": old_platform_name},
+    ).get("platform_id", old_platform_name)
+
+
+def get_platform_type(
+    platform_id_map: dict[str, dict[str, str]],
+    old_platform_name: str,
+) -> str:
+    return platform_id_map.get(
+        old_platform_name,
+        {"platform_id": old_platform_name, "platform_type": old_platform_name},
+    ).get("platform_type", old_platform_name)
+
+
+async def migration_conversation_table(
+    db_helper: BaseDatabase,
+    platform_id_map: dict[str, dict[str, str]],
+):
+    db_helper_v3 = SQLiteV3DatabaseV3(
+        db_path=DB_PATH.replace("data_v4.db", "data_v3.db"),
+    )
+    conversations, total_cnt = db_helper_v3.get_all_conversations(
+        page=1,
+        page_size=10000000,
+    )
+    logger.info(f"迁移 {total_cnt} 条旧的会话数据到新的表中...")
+
+    async with db_helper.get_db() as dbsession:
+        dbsession: AsyncSession
+        async with dbsession.begin():
+            for idx, conversation in enumerate(conversations):
+                if total_cnt > 0 and (idx + 1) % max(1, total_cnt // 10) == 0:
+                    progress = int((idx + 1) / total_cnt * 100)
+                    if progress % 10 == 0:
+                        logger.info(f"进度: {progress}% ({idx + 1}/{total_cnt})")
+                try:
+                    conv = db_helper_v3.get_conversation_by_user_id(
+                        user_id=conversation.get("user_id", "unknown"),
+                        cid=conversation.get("cid", "unknown"),
+                    )
+                    if not conv:
+                        logger.info(
+                            f"未找到该条旧会话对应的具体数据: {conversation}, 跳过。",
+                        )
+                    if ":" not in conv.user_id:
+                        continue
+                    session = MessageSesion.from_str(session_str=conv.user_id)
+                    platform_id = get_platform_id(
+                        platform_id_map,
+                        session.platform_name,
+                    )
+                    session.platform_id = platform_id  # 更新平台名称为新的 ID
+                    conv_v2 = ConversationV2(
+                        user_id=str(session),
+                        content=json.loads(conv.history) if conv.history else [],
+                        platform_id=platform_id,
+                        title=conv.title,
+                        persona_id=conv.persona_id,
+                        conversation_id=conv.cid,
+                        created_at=datetime.datetime.fromtimestamp(conv.created_at),
+                        updated_at=datetime.datetime.fromtimestamp(conv.updated_at),
+                    )
+                    dbsession.add(conv_v2)
+                except Exception as e:
+                    logger.error(
+                        f"迁移旧会话 {conversation.get('cid', 'unknown')} 失败: {e}",
+                        exc_info=True,
+                    )
+    logger.info(f"成功迁移 {total_cnt} 条旧的会话数据到新表。")
+
+
+async def migration_platform_table(
+    db_helper: BaseDatabase,
+    platform_id_map: dict[str, dict[str, str]],
+):
+    db_helper_v3 = SQLiteV3DatabaseV3(
+        db_path=DB_PATH.replace("data_v4.db", "data_v3.db"),
+    )
+    secs_from_2023_4_10_to_now = (
+        datetime.datetime.now(datetime.timezone.utc)
+        - datetime.datetime(2023, 4, 10, tzinfo=datetime.timezone.utc)
+    ).total_seconds()
+    offset_sec = int(secs_from_2023_4_10_to_now)
+    logger.info(f"迁移旧平台数据，offset_sec: {offset_sec} 秒。")
+    stats = db_helper_v3.get_base_stats(offset_sec=offset_sec)
+    logger.info(f"迁移 {len(stats.platform)} 条旧的平台数据到新的表中...")
+    platform_stats_v3 = stats.platform
+
+    if not platform_stats_v3:
+        logger.info("没有找到旧平台数据，跳过迁移。")
+        return
+
+    first_time_stamp = platform_stats_v3[0].timestamp
+    end_time_stamp = platform_stats_v3[-1].timestamp
+    start_time = first_time_stamp - (first_time_stamp % 3600)  # 向下取整到小时
+    end_time = end_time_stamp + (3600 - (end_time_stamp % 3600))  # 向上取整到小时
+
+    idx = 0
+
+    async with db_helper.get_db() as dbsession:
+        dbsession: AsyncSession
+        async with dbsession.begin():
+            total_buckets = (end_time - start_time) // 3600
+            for bucket_idx, bucket_end in enumerate(range(start_time, end_time, 3600)):
+                if bucket_idx % 500 == 0:
+                    progress = int((bucket_idx + 1) / total_buckets * 100)
+                    logger.info(f"进度: {progress}% ({bucket_idx + 1}/{total_buckets})")
+                cnt = 0
+                while (
+                    idx < len(platform_stats_v3)
+                    and platform_stats_v3[idx].timestamp < bucket_end
+                ):
+                    cnt += platform_stats_v3[idx].count
+                    idx += 1
+                if cnt == 0:
+                    continue
+                platform_id = get_platform_id(
+                    platform_id_map,
+                    platform_stats_v3[idx].name,
+                )
+                platform_type = get_platform_type(
+                    platform_id_map,
+                    platform_stats_v3[idx].name,
+                )
+                try:
+                    await dbsession.execute(
+                        text("""
+                        INSERT INTO platform_stats (timestamp, platform_id, platform_type, count)
+                        VALUES (:timestamp, :platform_id, :platform_type, :count)
+                        ON CONFLICT(timestamp, platform_id, platform_type) DO UPDATE SET
+                            count = platform_stats.count + EXCLUDED.count
+                        """),
+                        {
+                            "timestamp": datetime.datetime.fromtimestamp(
+                                bucket_end,
+                                tz=datetime.timezone.utc,
+                            ),
+                            "platform_id": platform_id,
+                            "platform_type": platform_type,
+                            "count": cnt,
+                        },
+                    )
+                except Exception:
+                    logger.error(
+                        f"迁移平台统计数据失败: {platform_id}, {platform_type}, 时间戳: {bucket_end}",
+                        exc_info=True,
+                    )
+    logger.info(f"成功迁移 {len(platform_stats_v3)} 条旧的平台数据到新表。")
+
+
+async def migration_webchat_data(
+    db_helper: BaseDatabase,
+    platform_id_map: dict[str, dict[str, str]],
+):
+    """迁移 WebChat 的历史记录到新的 PlatformMessageHistory 表中"""
+    db_helper_v3 = SQLiteV3DatabaseV3(
+        db_path=DB_PATH.replace("data_v4.db", "data_v3.db"),
+    )
+    conversations, total_cnt = db_helper_v3.get_all_conversations(
+        page=1,
+        page_size=10000000,
+    )
+    logger.info(f"迁移 {total_cnt} 条旧的 WebChat 会话数据到新的表中...")
+
+    async with db_helper.get_db() as dbsession:
+        dbsession: AsyncSession
+        async with dbsession.begin():
+            for idx, conversation in enumerate(conversations):
+                if total_cnt > 0 and (idx + 1) % max(1, total_cnt // 10) == 0:
+                    progress = int((idx + 1) / total_cnt * 100)
+                    if progress % 10 == 0:
+                        logger.info(f"进度: {progress}% ({idx + 1}/{total_cnt})")
+                try:
+                    conv = db_helper_v3.get_conversation_by_user_id(
+                        user_id=conversation.get("user_id", "unknown"),
+                        cid=conversation.get("cid", "unknown"),
+                    )
+                    if not conv:
+                        logger.info(
+                            f"未找到该条旧会话对应的具体数据: {conversation}, 跳过。",
+                        )
+                    if ":" in conv.user_id:
+                        continue
+                    platform_id = "webchat"
+                    history = json.loads(conv.history) if conv.history else []
+                    for msg in history:
+                        type_ = msg.get("type")  # user type, "bot" or "user"
+                        new_history = PlatformMessageHistory(
+                            platform_id=platform_id,
+                            user_id=conv.cid,  # we use conv.cid as user_id for webchat
+                            content=msg,
+                            sender_id=type_,
+                            sender_name=type_,
+                        )
+                        dbsession.add(new_history)
+
+                except Exception:
+                    logger.error(
+                        f"迁移旧 WebChat 会话 {conversation.get('cid', 'unknown')} 失败",
+                        exc_info=True,
+                    )
+
+    logger.info(f"成功迁移 {total_cnt} 条旧的 WebChat 会话数据到新表。")
+
+
+async def migration_persona_data(
+    db_helper: BaseDatabase,
+    astrbot_config: AstrBotConfig,
+):
+    """迁移 Persona 数据到新的表中。
+    旧的 Persona 数据存储在 preference 中，新的 Persona 数据存储在 persona 表中。
+    """
+    v3_persona_config: list[dict] = astrbot_config.get("persona", [])
+    total_personas = len(v3_persona_config)
+    logger.info(f"迁移 {total_personas} 个 Persona 配置到新表中...")
+
+    for idx, persona in enumerate(v3_persona_config):
+        if total_personas > 0 and (idx + 1) % max(1, total_personas // 10) == 0:
+            progress = int((idx + 1) / total_personas * 100)
+            if progress % 10 == 0:
+                logger.info(f"进度: {progress}% ({idx + 1}/{total_personas})")
+        try:
+            begin_dialogs = persona.get("begin_dialogs", [])
+            mood_imitation_dialogs = persona.get("mood_imitation_dialogs", [])
+            parts = []
+            user_turn = True
+            for mood_dialog in mood_imitation_dialogs:
+                if user_turn:
+                    parts.append(f"A: {mood_dialog}\n")
+                else:
+                    parts.append(f"B: {mood_dialog}\n")
+                user_turn = not user_turn
+            mood_prompt = "".join(parts)
+            system_prompt = persona.get("prompt", "")
+            if mood_prompt:
+                system_prompt += f"Here are few shots of dialogs, you need to imitate the tone of 'B' in the following dialogs to respond:\n {mood_prompt}"
+            persona_new = await db_helper.insert_persona(
+                persona_id=persona["name"],
+                system_prompt=system_prompt,
+                begin_dialogs=begin_dialogs,
+            )
+            logger.info(
+                f"迁移 Persona {persona['name']}({persona_new.system_prompt[:30]}...) 到新表成功。",
+            )
+        except Exception as e:
+            logger.error(f"解析 Persona 配置失败：{e}")
+
+
+async def migration_preferences(
+    db_helper: BaseDatabase,
+    platform_id_map: dict[str, dict[str, str]],
+):
+    # 1. global scope migration
+    keys = [
+        "inactivated_llm_tools",
+        "inactivated_plugins",
+        "curr_provider",
+        "curr_provider_tts",
+        "curr_provider_stt",
+        "alter_cmd",
+    ]
+    for key in keys:
+        value = sp_v3.get(key)
+        if value is not None:
+            await sp.put_async("global", "global", key, value)
+            logger.info(f"迁移全局偏好设置 {key} 成功，值: {value}")
+
+    # 2. umo scope migration
+    session_conversation = sp_v3.get("session_conversation", default={})
+    for umo, conversation_id in session_conversation.items():
+        if not umo or not conversation_id:
+            continue
+        try:
+            session = MessageSesion.from_str(session_str=umo)
+            platform_id = get_platform_id(platform_id_map, session.platform_name)
+            session.platform_id = platform_id
+            await sp.put_async("umo", str(session), "sel_conv_id", conversation_id)
+            logger.info(f"迁移会话 {umo} 的对话数据到新表成功，平台 ID: {platform_id}")
+        except Exception as e:
+            logger.error(f"迁移会话 {umo} 的对话数据失败: {e}", exc_info=True)
+
+    session_service_config = sp_v3.get("session_service_config", default={})
+    for umo, config in session_service_config.items():
+        if not umo or not config:
+            continue
+        try:
+            session = MessageSesion.from_str(session_str=umo)
+            platform_id = get_platform_id(platform_id_map, session.platform_name)
+            session.platform_id = platform_id
+
+            await sp.put_async("umo", str(session), "session_service_config", config)
+
+            logger.info(f"迁移会话 {umo} 的服务配置到新表成功，平台 ID: {platform_id}")
+        except Exception as e:
+            logger.error(f"迁移会话 {umo} 的服务配置失败: {e}", exc_info=True)
+
+    session_variables = sp_v3.get("session_variables", default={})
+    for umo, variables in session_variables.items():
+        if not umo or not variables:
+            continue
+        try:
+            session = MessageSesion.from_str(session_str=umo)
+            platform_id = get_platform_id(platform_id_map, session.platform_name)
+            session.platform_id = platform_id
+            await sp.put_async("umo", str(session), "session_variables", variables)
+        except Exception as e:
+            logger.error(f"迁移会话 {umo} 的变量失败: {e}", exc_info=True)
+
+    session_provider_perf = sp_v3.get("session_provider_perf", default={})
+    for umo, perf in session_provider_perf.items():
+        if not umo or not perf:
+            continue
+        try:
+            session = MessageSesion.from_str(session_str=umo)
+            platform_id = get_platform_id(platform_id_map, session.platform_name)
+            session.platform_id = platform_id
+
+            for provider_type, provider_id in perf.items():
+                await sp.put_async(
+                    "umo",
+                    str(session),
+                    f"provider_perf_{provider_type}",
+                    provider_id,
+                )
+            logger.info(
+                f"迁移会话 {umo} 的提供商偏好到新表成功，平台 ID: {platform_id}",
+            )
+        except Exception as e:
+            logger.error(f"迁移会话 {umo} 的提供商偏好失败: {e}", exc_info=True)

astrbot/core/db/migration/migra_45_to_46.py

@@ -0,0 +1,44 @@
+from astrbot.api import logger, sp
+from astrbot.core.astrbot_config_mgr import AstrBotConfigManager
+from astrbot.core.umop_config_router import UmopConfigRouter
+
+
+async def migrate_45_to_46(acm: AstrBotConfigManager, ucr: UmopConfigRouter):
+    abconf_data = acm.abconf_data
+
+    if not isinstance(abconf_data, dict):
+        # should be unreachable
+        logger.warning(
+            f"migrate_45_to_46: abconf_data is not a dict (type={type(abconf_data)}). Value: {abconf_data!r}",
+        )
+        return
+
+    # 如果任何一项带有 umop，则说明需要迁移
+    need_migration = False
+    for conf_id, conf_info in abconf_data.items():
+        if isinstance(conf_info, dict) and "umop" in conf_info:
+            need_migration = True
+            break
+
+    if not need_migration:
+        return
+
+    logger.info("Starting migration from version 4.5 to 4.6")
+
+    # extract umo->conf_id mapping
+    umo_to_conf_id = {}
+    for conf_id, conf_info in abconf_data.items():
+        if isinstance(conf_info, dict) and "umop" in conf_info:
+            umop_ls = conf_info.pop("umop")
+            if not isinstance(umop_ls, list):
+                continue
+            for umo in umop_ls:
+                if isinstance(umo, str) and umo not in umo_to_conf_id:
+                    umo_to_conf_id[umo] = conf_id
+
+    # update the abconf data
+    await sp.global_put("abconf_mapping", abconf_data)
+    # update the umop config router
+    await ucr.update_routing_data(umo_to_conf_id)
+
+    logger.info("Migration from version 45 to 46 completed successfully")

astrbot/core/db/migration/migra_webchat_session.py

@@ -0,0 +1,131 @@
+"""Migration script for WebChat sessions.
+
+This migration creates PlatformSession from existing platform_message_history records.
+
+Changes:
+- Creates platform_sessions table
+- Adds platform_id field (default: 'webchat')
+- Adds display_name field
+- Session_id format: {platform_id}_{uuid}
+"""
+
+from sqlalchemy import func, select
+from sqlmodel import col
+
+from astrbot.api import logger, sp
+from astrbot.core.db import BaseDatabase
+from astrbot.core.db.po import ConversationV2, PlatformMessageHistory, PlatformSession
+
+
+async def migrate_webchat_session(db_helper: BaseDatabase):
+    """Create PlatformSession records from platform_message_history.
+
+    This migration extracts all unique user_ids from platform_message_history
+    where platform_id='webchat' and creates corresponding PlatformSession records.
+    """
+    # 检查是否已经完成迁移
+    migration_done = await db_helper.get_preference(
+        "global", "global", "migration_done_webchat_session_1"
+    )
+    if migration_done:
+        return
+
+    logger.info("开始执行数据库迁移（WebChat 会话迁移）...")
+
+    try:
+        async with db_helper.get_db() as session:
+            # 从 platform_message_history 创建 PlatformSession
+            query = (
+                select(
+                    col(PlatformMessageHistory.user_id),
+                    col(PlatformMessageHistory.sender_name),
+                    func.min(PlatformMessageHistory.created_at).label("earliest"),
+                    func.max(PlatformMessageHistory.updated_at).label("latest"),
+                )
+                .where(col(PlatformMessageHistory.platform_id) == "webchat")
+                .where(col(PlatformMessageHistory.sender_id) != "bot")
+                .group_by(col(PlatformMessageHistory.user_id))
+            )
+
+            result = await session.execute(query)
+            webchat_users = result.all()
+
+            if not webchat_users:
+                logger.info("没有找到需要迁移的 WebChat 数据")
+                await sp.put_async(
+                    "global", "global", "migration_done_webchat_session_1", True
+                )
+                return
+
+            logger.info(f"找到 {len(webchat_users)} 个 WebChat 会话需要迁移")
+
+            # 检查已存在的会话
+            existing_query = select(col(PlatformSession.session_id))
+            existing_result = await session.execute(existing_query)
+            existing_session_ids = {row[0] for row in existing_result.fetchall()}
+
+            # 查询 Conversations 表中的 title，用于设置 display_name
+            # 对于每个 user_id，对应的 conversation user_id 格式为: webchat:FriendMessage:webchat!astrbot!{user_id}
+            user_ids_to_query = [
+                f"webchat:FriendMessage:webchat!astrbot!{user_id}"
+                for user_id, _, _, _ in webchat_users
+            ]
+            conv_query = select(
+                col(ConversationV2.user_id), col(ConversationV2.title)
+            ).where(col(ConversationV2.user_id).in_(user_ids_to_query))
+            conv_result = await session.execute(conv_query)
+            # 创建 user_id -> title 的映射字典
+            title_map = {
+                user_id.replace("webchat:FriendMessage:webchat!astrbot!", ""): title
+                for user_id, title in conv_result.fetchall()
+            }
+
+            # 批量创建 PlatformSession 记录
+            sessions_to_add = []
+            skipped_count = 0
+
+            for user_id, sender_name, created_at, updated_at in webchat_users:
+                # user_id 就是 webchat_conv_id (session_id)
+                session_id = user_id
+
+                # sender_name 通常是 username，但可能为 None
+                creator = sender_name if sender_name else "guest"
+
+                # 检查是否已经存在该会话
+                if session_id in existing_session_ids:
+                    logger.debug(f"会话 {session_id} 已存在，跳过")
+                    skipped_count += 1
+                    continue
+
+                # 从 Conversations 表中获取 display_name
+                display_name = title_map.get(user_id)
+
+                # 创建新的 PlatformSession（保留原有的时间戳）
+                new_session = PlatformSession(
+                    session_id=session_id,
+                    platform_id="webchat",
+                    creator=creator,
+                    is_group=0,
+                    created_at=created_at,
+                    updated_at=updated_at,
+                    display_name=display_name,
+                )
+                sessions_to_add.append(new_session)
+
+            # 批量插入
+            if sessions_to_add:
+                session.add_all(sessions_to_add)
+                await session.commit()
+
+                logger.info(
+                    f"WebChat 会话迁移完成！成功迁移: {len(sessions_to_add)}, 跳过: {skipped_count}",
+                )
+            else:
+                logger.info("没有新会话需要迁移")
+
+        # 标记迁移完成
+        await sp.put_async("global", "global", "migration_done_webchat_session_1", True)
+
+    except Exception as e:
+        logger.error(f"迁移过程中发生错误: {e}", exc_info=True)
+        raise

astrbot/core/db/migration/shared_preferences_v3.py

@@ -0,0 +1,48 @@
+import json
+import os
+from typing import TypeVar
+
+from astrbot.core.utils.astrbot_path import get_astrbot_data_path
+
+_VT = TypeVar("_VT")
+
+
+class SharedPreferences:
+    def __init__(self, path=None):
+        if path is None:
+            path = os.path.join(get_astrbot_data_path(), "shared_preferences.json")
+        self.path = path
+        self._data = self._load_preferences()
+
+    def _load_preferences(self):
+        if os.path.exists(self.path):
+            try:
+                with open(self.path) as f:
+                    return json.load(f)
+            except json.JSONDecodeError:
+                os.remove(self.path)
+        return {}
+
+    def _save_preferences(self):
+        with open(self.path, "w") as f:
+            json.dump(self._data, f, indent=4, ensure_ascii=False)
+            f.flush()
+
+    def get(self, key, default: _VT = None) -> _VT:
+        return self._data.get(key, default)
+
+    def put(self, key, value):
+        self._data[key] = value
+        self._save_preferences()
+
+    def remove(self, key):
+        if key in self._data:
+            del self._data[key]
+            self._save_preferences()
+
+    def clear(self):
+        self._data.clear()
+        self._save_preferences()
+
+
+sp = SharedPreferences()