consumer-sdk 1.0.0__py3-none-any.whl

consumer_sdk/__init__.py

@@ -0,0 +1,9 @@
+from .consumer import DiscordEventConsumer
+from .snapshot import fetch_snapshot, format_snapshot, format_snapshot_summary
+
+__all__ = [
+    "DiscordEventConsumer",
+    "fetch_snapshot",
+    "format_snapshot",
+    "format_snapshot_summary",
+]

consumer_sdk/consumer.py

@@ -0,0 +1,144 @@
+import asyncio
+import json
+import logging
+import os
+from typing import Awaitable, Callable, Optional
+
+import redis
+import redis.asyncio as aioredis
+
+logger = logging.getLogger("consumer")
+
+MetaHandler = Callable[[dict], Awaitable[None]]
+
+
+class DiscordEventConsumer:
+    def __init__(
+        self,
+        redis_url: str,
+        consumer_name: str,
+        stream_key: str = "discord:events",
+        group_name: Optional[str] = None,
+        block_ms: int = 5000,
+        batch: int = 50,
+        state_file: Optional[str] = None,
+    ):
+        self.redis_url = redis_url
+        self.consumer_name = consumer_name
+        self.stream_key = stream_key
+        self.group_name = group_name or f"group_{consumer_name}"
+        self.block_ms = block_ms
+        self.batch = batch
+        self.state_file = state_file or f".consumer_state_{consumer_name}.json"
+
+        self.rds: Optional[aioredis.Redis] = None
+        self._stop = False
+        self._seen: set = set()
+        self._load_state()
+
+    def _load_state(self):
+        if os.path.exists(self.state_file):
+            try:
+                with open(self.state_file, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                self._seen = set(data.get("seen_event_ids", []))
+                if len(self._seen) > 20000:
+                    self._seen = set(list(self._seen)[-20000:])
+                logger.info(f"加载去重状态: {len(self._seen)} 条")
+            except Exception as e:
+                logger.warning(f"加载状态失败: {e}")
+
+    def _save_state(self):
+        try:
+            with open(self.state_file + ".tmp", "w", encoding="utf-8") as f:
+                json.dump({"seen_event_ids": list(self._seen)[-20000:]}, f)
+            os.replace(self.state_file + ".tmp", self.state_file)
+        except Exception as e:
+            logger.warning(f"保存状态失败: {e}")
+
+    def _is_dup(self, event_id: str) -> bool:
+        if not event_id:
+            return False
+        if event_id in self._seen:
+            return True
+        self._seen.add(event_id)
+        if len(self._seen) % 200 == 0:
+            self._save_state()
+        return False
+
+    async def _connect(self):
+        self.rds = aioredis.from_url(self.redis_url, decode_responses=True)
+        try:
+            await self.rds.xgroup_create(
+                self.stream_key, self.group_name, id="$", mkstream=True
+            )
+            logger.info(f"创建 group {self.group_name} (从最新开始)")
+        except Exception as e:
+            if "BUSYGROUP" in str(e):
+                logger.info(f"group {self.group_name} 已存在")
+            else:
+                raise
+
+    async def listen(self, handler: MetaHandler):
+        await self._connect()
+        logger.info(
+            f"[{self.consumer_name}] 开始监听 stream={self.stream_key} group={self.group_name}"
+        )
+
+        while not self._stop:
+            try:
+                resp = await self.rds.xreadgroup(
+                    groupname=self.group_name,
+                    consumername=self.consumer_name,
+                    streams={self.stream_key: ">"},
+                    count=self.batch,
+                    block=self.block_ms,
+                )
+                if not resp:
+                    continue
+                for _stream, entries in resp:
+                    for entry_id, fields in entries:
+                        await self._dispatch(entry_id, fields, handler)
+
+            except asyncio.CancelledError:
+                break
+            except asyncio.TimeoutError:
+                continue
+            except redis.exceptions.TimeoutError:
+                continue
+            except redis.exceptions.ConnectionError as e:
+                logger.warning(f"Redis 连接异常，1 秒后重连: {e}")
+                await asyncio.sleep(1)
+            except Exception as e:
+                logger.error(f"循环异常: {e}")
+                await asyncio.sleep(1)
+
+        self._save_state()
+        if self.rds:
+            await self.rds.close()
+        logger.info(f"[{self.consumer_name}] 已停止")
+
+    async def _dispatch(self, entry_id, fields, handler: MetaHandler):
+        try:
+            payload = fields.get("payload")
+            if not payload:
+                await self.rds.xack(self.stream_key, self.group_name, entry_id)
+                return
+
+            meta = json.loads(payload)
+            event_id = meta.get("event_id", "")
+
+            if self._is_dup(event_id):
+                await self.rds.xack(self.stream_key, self.group_name, entry_id)
+                return
+
+            try:
+                await handler(meta)
+                await self.rds.xack(self.stream_key, self.group_name, entry_id)
+            except Exception as e:
+                logger.exception(f"handler 处理失败 event_id={event_id}: {e}")
+        except Exception as e:
+            logger.error(f"派发失败 entry={entry_id}: {e}")
+
+    def stop(self):
+        self._stop = True

consumer_sdk/snapshot.py

@@ -0,0 +1,352 @@
+# consumer_sdk/snapshot.py
+"""
+消费者侧：读 Redis 里的频道快照，格式化成可读文本或 embed。
+
+分页策略：
+- 用 code block (```...```) 包裹内容
+- 按"频道 + 子区"为整体块分页
+- 以"整行"为最小单位，绝不切字符
+- 每条 embed 都有 title（第 1 条也带序号 (1)，后续 (N)）
+- 取消零宽空格（code block 里 # 不触发 Markdown）
+
+过滤：
+- thread_max_age_days：只显示"最近 N 天有活动"的子区
+
+多 reader 支持：
+- fetch_snapshot(rds, reader_id="reader-01") 读 reader-01:* 前缀的 key
+"""
+import json
+import datetime
+import time
+
+
+EMBED_DESC_LIMIT = 4096
+CODE_BLOCK_OVERHEAD = 8
+
+
+def _utf16_len(s: str) -> int:
+    length = 0
+    for ch in s:
+        if ord(ch) > 0xFFFF:
+            length += 2
+        else:
+            length += 1
+    return length
+
+
+def _is_thread_active(t: dict, max_age_days: int) -> bool:
+    """
+    判断子区是否"最近 N 天有活动"。
+    - max_age_days <= 0 → 不过滤
+    - last_message_at 缺失 → 用 created_at 兜底
+    - 都没有 → 不过滤（保守）
+    """
+    if max_age_days <= 0:
+        return True
+
+    last_msg_at = t.get("last_message_at")
+    if last_msg_at is None:
+        last_msg_at = t.get("created_at") or t.get("thread_created_at")
+    if last_msg_at is None:
+        return True
+
+    try:
+        age_days = (time.time() - float(last_msg_at)) / 86400.0
+        return age_days <= max_age_days
+    except Exception:
+        return True
+
+
+async def fetch_snapshot(rds, reader_id: str = "reader-01") -> dict:
+    """
+    从 Redis 读频道快照。
+
+    参数：
+        rds: redis 客户端
+        reader_id: reader 标识，用于拼 key 前缀
+                   比如 "reader-01" → 读 "reader-01:channels" 等
+
+    返回：
+        {
+            "guilds": {...},
+            "categories": {...},
+            "channels": {...},
+            "updated_at": float 或 None,
+        }
+    """
+    guilds_raw = await rds.hgetall(f"{reader_id}:guilds")
+    cats_raw = await rds.hgetall(f"{reader_id}:categories")
+    chans_raw = await rds.hgetall(f"{reader_id}:channels")
+    ts = await rds.get(f"{reader_id}:snapshot:updated_at")
+
+    guilds = {}
+    for gid, j in guilds_raw.items():
+        try:
+            guilds[gid] = json.loads(j)
+        except Exception:
+            pass
+
+    cats = {}
+    for cid, j in cats_raw.items():
+        try:
+            cats[cid] = json.loads(j)
+        except Exception:
+            pass
+
+    chans = {}
+    for cid, j in chans_raw.items():
+        try:
+            chans[cid] = json.loads(j)
+        except Exception:
+            pass
+
+    return {
+        "guilds": guilds,
+        "categories": cats,
+        "channels": chans,
+        "updated_at": float(ts) if ts else None,
+    }
+
+
+def _channel_icon(channel_type: str) -> str:
+    if channel_type == "forum":
+        return "📋"
+    if channel_type == "voice":
+        return "🔊"
+    if channel_type == "announcement":
+        return "📢"
+    if channel_type == "stage":
+        return "🎙️"
+    return "#"
+
+
+def _make_channel_header(c: dict, indent: str = "") -> str:
+    cid = c["id"]
+    cname = c["name"]
+    ctype = c.get("channel_type", "")
+    icon = _channel_icon(ctype)
+    return f"{indent}{icon} {cname}（{cid}）"
+
+
+def _make_thread_line(t: dict, indent: str = "") -> str:
+    arch = " [已归档]" if t.get("archived") else ""
+    return f"{indent}    └─ {t['name']}（{t['id']}）{arch}"
+
+
+def _build_items(snapshot: dict, thread_max_age_days: int = 0) -> list:
+    """
+    生成所有"可独立分页的条目"。
+
+    thread_max_age_days:
+      0 或负数 → 不过滤
+      > 0 → 只显示"最近 N 天有活动"的子区
+    """
+    items = []
+
+    for gid, g in snapshot["guilds"].items():
+        gname = g.get("name", "?")
+        items.append({
+            "type": "text",
+            "text": f"🏠 {gname}（{gid}）{g.get('member_count','?')} 成员",
+        })
+
+        all_chans = [
+            c for c in snapshot["channels"].values()
+            if c.get("guild_id") == gid
+        ]
+        mains = [c for c in all_chans if not c.get("is_thread")]
+        threads = [c for c in all_chans if c.get("is_thread")]
+
+        threads_by_parent = {}
+        skipped_threads = 0
+        for t in threads:
+            # ★ 过滤"不活跃"的子区
+            if not _is_thread_active(t, thread_max_age_days):
+                skipped_threads += 1
+                continue
+            pid = t.get("parent_id") or "__orphan__"
+            threads_by_parent.setdefault(pid, []).append(t)
+
+        if skipped_threads > 0:
+            print(f"[快照格式] {gname}: 跳过 {skipped_threads} 个不活跃子区")
+
+        no_cat = [c for c in mains if not c.get("category_id")]
+        with_cat = [c for c in mains if c.get("category_id")]
+
+        no_cat_sorted = sorted(
+            no_cat,
+            key=lambda x: x.get("position") if x.get("position") is not None else 999999
+        )
+        for c in no_cat_sorted:
+            items.append(_make_channel_item(c, threads_by_parent, indent=""))
+
+        by_cat = {}
+        for c in with_cat:
+            by_cat.setdefault(c["category_id"], []).append(c)
+
+        cat_list = []
+        for cat_id, cat_items in by_cat.items():
+            cat = snapshot["categories"].get(cat_id, {})
+            cat_pos = cat.get("position")
+            if cat_pos is None:
+                cat_pos = 999998
+            cat_name = cat.get("name")
+            if not cat_name:
+                for c in cat_items:
+                    if c.get("category_name"):
+                        cat_name = c["category_name"]
+                        break
+            if not cat_name:
+                cat_name = f"分类 {cat_id}"
+            cat_list.append((cat_pos, cat_name, cat_items))
+        cat_list.sort(key=lambda x: x[0])
+
+        for cat_pos, cat_name, cat_items in cat_list:
+            items.append({"type": "text", "text": f"▾ {cat_name}"})
+            cat_items_sorted = sorted(
+                cat_items,
+                key=lambda x: x.get("position") if x.get("position") is not None else 999999
+            )
+            for c in cat_items_sorted:
+                items.append(_make_channel_item(c, threads_by_parent, indent="  "))
+
+        orphans = threads_by_parent.get("__orphan__", [])
+        if orphans:
+            items.append({"type": "text", "text": "▾ 孤儿子区"})
+            for t in sorted(orphans, key=lambda x: x.get("thread_created_at") or 0, reverse=True):
+                items.append({"type": "text", "text": _make_thread_line(t, "  ")})
+
+        items.append({"type": "text", "text": ""})
+
+    return items
+
+
+def _make_channel_item(c: dict, threads_by_parent: dict, indent: str = "") -> dict:
+    cid = c["id"]
+    header = _make_channel_header(c, indent)
+    children = []
+    child_threads = threads_by_parent.get(cid, [])
+    child_threads_sorted = sorted(
+        child_threads,
+        key=lambda x: x.get("thread_created_at") or 0,
+        reverse=True,
+    )
+    for t in child_threads_sorted:
+        children.append(_make_thread_line(t, indent))
+    return {
+        "type": "channel",
+        "header": header,
+        "indent": indent,
+        "children": children,
+    }
+
+
+def format_snapshot(snapshot: dict, max_chars: int = 1900,
+                    thread_max_age_days: int = 0) -> list:
+    lines = ["**📋 服务器频道清单**"]
+    if snapshot["updated_at"]:
+        ts = datetime.datetime.fromtimestamp(snapshot["updated_at"])
+        lines.append(f"更新时间：{ts}")
+    lines.append("")
+
+    for item in _build_items(snapshot, thread_max_age_days=thread_max_age_days):
+        if item["type"] == "text":
+            lines.append(item["text"])
+        else:
+            lines.append(item["header"])
+            lines.extend(item["children"])
+
+    chunks = []
+    cur = ""
+    for line in lines:
+        if len(cur) + len(line) + 1 > max_chars:
+            chunks.append(cur)
+            cur = line + "\n"
+        else:
+            cur += line + "\n"
+    if cur:
+        chunks.append(cur)
+    return chunks
+
+
+def format_snapshot_as_embeds(snapshot: dict, title: str = "📋 服务器频道清单",
+                              thread_max_age_days: int = 0) -> list:
+    header_lines = []
+    if snapshot["updated_at"]:
+        ts = datetime.datetime.fromtimestamp(snapshot["updated_at"])
+        header_lines.append(f"更新时间：{ts}")
+        header_lines.append("")
+
+    items = _build_items(snapshot, thread_max_age_days=thread_max_age_days)
+
+    MAX_CONTENT = EMBED_DESC_LIMIT - CODE_BLOCK_OVERHEAD
+
+    embeds = []
+    current_lines = list(header_lines)
+    page_num = 1
+
+    def _content_utf16(lines: list) -> int:
+        if not lines:
+            return 0
+        return _utf16_len("\n".join(lines))
+
+    def flush_page():
+        nonlocal page_num, current_lines
+        if not current_lines:
+            return
+        content = "\n".join(current_lines)
+        desc = "```\n" + content + "\n```"
+        # ★ 所有页都带序号（包括第 1 页）
+        title_text = f"{title}（{page_num}）"
+        embeds.append({
+            "title": title_text[:256],
+            "description": desc,
+            "color": 0x5865F2,
+        })
+        page_num += 1
+        current_lines = []
+
+    def try_append(line: str):
+        nonlocal current_lines
+        candidate_lines = current_lines + [line]
+        if _content_utf16(candidate_lines) > MAX_CONTENT:
+            flush_page()
+            current_lines = [line]
+        else:
+            current_lines.append(line)
+
+    for item in items:
+        if item["type"] == "text":
+            try_append(item["text"])
+        else:
+            header = item["header"]
+            children = item["children"]
+
+            block = [header] + children
+            candidate_lines = current_lines + block
+            if _content_utf16(candidate_lines) <= MAX_CONTENT:
+                current_lines.extend(block)
+            else:
+                try_append(header)
+                for c in children:
+                    try_append(c)
+
+    if current_lines:
+        flush_page()
+
+    return embeds
+
+
+def format_snapshot_summary(snapshot: dict) -> str:
+    lines = ["**📋 服务器频道快照**"]
+    for gid, g in snapshot["guilds"].items():
+        chans = [c for c in snapshot["channels"].values() if c.get("guild_id") == gid]
+        mains = [c for c in chans if not c.get("is_thread")]
+        threads = [c for c in chans if c.get("is_thread")]
+        lines.append(
+            f"- **{g['name']}**（`{gid}`）："
+            f"{len(mains)} 频道、{len(threads)} 子区"
+        )
+    lines.append("")
+    lines.append("完整清单请用 `tools/list_channels.py` 查看")
+    return "\n".join(lines)

consumer_sdk-1.0.0.dist-info/METADATA

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: consumer-sdk
+Version: 1.0.0
+Requires-Dist: redis>=5.0
+Requires-Dist: aiohttp>=3.9

consumer_sdk-1.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+consumer_sdk/__init__.py,sha256=YDoc7-PtxjiV9HlsSc1HSGN39qkI_g2fQhMIy9b54Hk,240
+consumer_sdk/consumer.py,sha256=FKu0L0WVbtPYAa1LEeEdRqEHTW7V9_lzrs2S18Vi7po,4984
+consumer_sdk/snapshot.py,sha256=DK8GKLMPd_D-Czi9b2qUK871WZW_kjOVfn6xtuxeejQ,10769
+consumer_sdk-1.0.0.dist-info/METADATA,sha256=7359igRfJBokwbRHwBRVirdcIEMwnX3ky-jb1QLrugk,110
+consumer_sdk-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+consumer_sdk-1.0.0.dist-info/top_level.txt,sha256=f-tgdcDvoE5nrFqYVG5PoRaSzb6reBSwvPswNKP7ogY,13
+consumer_sdk-1.0.0.dist-info/RECORD,,

consumer_sdk-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

consumer_sdk-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+consumer_sdk