scrollbreaker 0.1.0__py3-none-any.whl

scrollbreaker/__init__.py

@@ -0,0 +1,32 @@
+"""scrollbreaker —— 把滚动的屏幕，切成能读的页。
+
+核心问题：手机/电脑上读长内容时，一屏只能看到一小段。连续截屏会有三种噪声：
+  1. 屏幕没动，重复截到同一屏（浪费 token）
+  2. 内容滚动了，前后两屏有一段重叠（直接拼会重复）
+  3. **换页面了**，但上一屏和这一屏恰好有相同的一行（直接拼会把两件事糊成一件）
+
+前两个好解决，第三个才是真正会毁掉数据的问题。scrollbreaker 用四道闸门判定
+「这两屏到底是不是同一页」，只有全过才拼，否则另起一条。
+"""
+
+from .config import Config, DEFAULT_CONFIG
+from .pagetype import page_type, head_anchor, body_lines
+from .accumulate import Accumulator, Frame, Merge, Mode
+from .dedup import dhash, is_duplicate, is_blank
+
+__all__ = [
+    "Config",
+    "DEFAULT_CONFIG",
+    "page_type",
+    "head_anchor",
+    "body_lines",
+    "Accumulator",
+    "Frame",
+    "Merge",
+    "Mode",
+    "dhash",
+    "is_duplicate",
+    "is_blank",
+]
+
+__version__ = "0.1.0"

scrollbreaker/accumulate.py

@@ -0,0 +1,204 @@
+"""累积器：判定「这一屏和上一屏，到底是不是同一页」。
+
+这是整个项目最核心的一块，也是最容易做错的一块。
+
+反面教材（早期实现）：
+    只看「两屏有几行重合」就拼。实测中，公众号文章和聊天窗口之间恰好
+    重合了 6 行（都是「正常」「好的」这类短句），于是把一篇文章和一串
+    车间对话拼成了一条记录——**数据污染了，而且看不出来**。
+
+正确做法：四道闸门全部通过，才允许合并。
+
+    ① 时间窗   两屏间隔 ≤ window_s（默认 180s）。再久就是「过了一会儿又看」
+    ② 类型一致 两屏 page_type 相同，且都不是 mixed
+    ③ 页头一致 页头锚点相同（article 例外：滚动时首行会变，同类型即认同一篇）
+    ④ 有实质重合 完全相同行的数量 ≥ min_overlap（默认 2）
+
+任一条不过 = 换页了 = 另起一条，绝不拼。
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Sequence
+
+from .config import CompiledPatterns, Config, DEFAULT_CONFIG
+from .pagetype import (
+    ACCUMULABLE,
+    ARTICLE,
+    CHAT,
+    MIXED,
+    body_lines,
+    chat_anchor,
+    head_anchor,
+    normalize,
+    page_type,
+)
+
+
+class Mode(str, Enum):
+    """这一条记录是怎么来的。"""
+
+    ROLL = "roll"   # 与上一屏同页，合并而成
+    NEW = "new"     # 换页了，独立成条
+
+
+@dataclass
+class Frame:
+    """一屏采样。"""
+
+    lines: Sequence[str]
+    ts: float                      # 采样时间（秒，epoch）
+
+    def __post_init__(self):
+        self.lines = list(self.lines)
+
+
+@dataclass
+class Merge:
+    """一次 push 的结果。"""
+
+    mode: Mode
+    lines: list[str]
+    page: str                 # 页面类型
+    anchor: str               # 页头锚点
+    overlap: int              # 与上一屏的重合行数
+    scene: str                # "<page>|<anchor>"，同一 scene 才是同一次阅读
+    text: str                 # 最终文本（已按 max_text_chars 截断）
+    gate_log: dict = field(default_factory=dict)   # 每道闸门的判定结果，便于排查
+
+    @property
+    def line_count(self) -> int:
+        return len(self.lines)
+
+
+def _same_page(
+    prev: dict, ptype: str, phead: str, overlap: int, cur_ts: float, cfg: Config
+) -> tuple[bool, dict]:
+    """四道闸门。返回 (是否同页, 逐闸trace)。"""
+    prev_lines = prev.get("lines") or []
+    prev_type = prev.get("type") or ""
+    prev_head = prev.get("head") or ""
+    prev_ts = float(prev.get("ts") or 0.0)
+
+    g1_time = bool(prev_lines) and (cur_ts - prev_ts) <= cfg.window_s
+    g2_type = prev_type == ptype and prev_type != MIXED and ptype != MIXED
+    g3_head = prev_head == phead
+    g4_overlap = overlap >= cfg.min_overlap
+
+    # 文章页滚动时首行会变（标题滚出屏幕），靠「类型=article」认同一篇
+    g3_pass = True if ptype == ARTICLE else g3_head
+
+    ok = g1_time and g2_type and g3_pass and g4_overlap
+    trace = {
+        "time_window": g1_time,
+        "same_type": g2_type,
+        "same_anchor": g3_head,
+        "anchor_relaxed": ptype == ARTICLE,
+        "overlap_enough": g4_overlap,
+    }
+    return ok, trace
+
+
+class Accumulator:
+    """有状态的累积器：喂进一屏一屏的文本，吐出「该不该合并」。
+
+    用法：
+        acc = Accumulator()
+        for snap in snapshots:            # 每 N 秒一屏
+            m = acc.push(Frame(lines=snap.lines, ts=snap.ts))
+            if m.mode is Mode.ROLL:       # 同一页在滚动，用 m.text（完整累积）
+                ...
+            else:                          # 换页了，m.text 是这一页的第一屏
+                ...
+    """
+
+    def __init__(self, cfg: Config = DEFAULT_CONFIG):
+        self.cfg = cfg
+        self.pats: CompiledPatterns = cfg.compiled()
+        self._prev: dict = {}
+        self._last_norm: str | None = None
+
+    # ── 状态 ────────────────────────────────────────────────
+    @property
+    def current_lines(self) -> list[str]:
+        return list(self._prev.get("lines") or [])
+
+    @property
+    def current_page(self) -> str:
+        return str(self._prev.get("type") or "")
+
+    def reset(self) -> None:
+        self._prev = {}
+        self._last_norm = None
+
+    # ── 主流程 ──────────────────────────────────────────────
+    def push(self, frame: Frame) -> Merge:
+        keep = [l for l in frame.lines if l.strip()]
+        ptype = page_type(keep, self.cfg, self.pats)
+        # 聊天页用发言人集合做锚点（页头可能已滚出屏幕），其余用首行
+        if ptype == CHAT:
+            phead = chat_anchor(keep, self.cfg, self.pats)
+        else:
+            phead = head_anchor(keep, self.cfg, self.pats)
+
+        prev = self._prev
+        # 重合判定必须基于「上一帧真实观测到的行」（raw），不能用已剥横幅的输出版本，
+        # 否则文章页剥掉顶部几行后，下一帧就再也算不出重合，会误判成换页。
+        prev_observed = [str(x) for x in (prev.get("raw") or prev.get("lines") or [])]
+        prev_lines = [str(x) for x in (prev.get("lines") or [])]
+        overlap = len(set(keep) & set(prev_observed))
+
+        same, trace = _same_page(prev, ptype, phead, overlap, frame.ts, self.cfg)
+
+        if same:
+            # 文章页要剥掉顶部的消息横幅，否则正文里夹一句聊天
+            incoming = body_lines(keep, self.cfg) if ptype == ARTICLE else keep
+            merged = list(prev_lines)
+            seen = set(merged)
+            for line in incoming:
+                if line not in seen:      # 保序去重
+                    merged.append(line)
+                    seen.add(line)
+            if len(merged) > self.cfg.max_lines:
+                merged = merged[-self.cfg.max_lines:]
+            mode = Mode.ROLL
+        else:
+            # 新起一条时，文章页同样要剥掉顶部横幅——否则「在吗」就混进正文了
+            base = body_lines(keep, self.cfg) if ptype == ARTICLE else keep
+            merged = list(base)[-self.cfg.max_lines:]
+            mode = Mode.NEW
+
+        self._prev = {
+            "lines": merged,
+            "raw": list(keep),          # 本帧原始观测行，用于下一帧的重合判定
+            "ts": frame.ts,
+            "type": ptype,
+            "head": phead,
+        }
+
+        text = "\n".join(merged)[: self.cfg.max_text_chars]
+        return Merge(
+            mode=mode,
+            lines=merged,
+            page=ptype,
+            anchor=phead,
+            overlap=overlap,
+            scene=f"{ptype}|{phead}",
+            text=text,
+            gate_log=trace,
+        )
+
+    def push_text(self, raw_text: str, ts: float, splitter=None) -> Merge:
+        """便捷入口：直接喂 OCR 后的整段文本。"""
+        lines = splitter(raw_text) if splitter else [l.strip() for l in raw_text.splitlines() if l.strip()]
+        return self.push(Frame(lines=lines, ts=ts))
+
+    # ── 文本层去重（配合 dedup.Deduper） ──────────────────
+    def is_text_unchanged(self, merge: Merge) -> bool:
+        """整段归一化后与上次完全一致 → 内容没变，不必重复上报。"""
+        n = normalize(merge.text, self.cfg)
+        unchanged = (n == self._last_norm) or (n == "")
+        self._last_norm = n
+        return unchanged

scrollbreaker/cli.py

@@ -0,0 +1,180 @@
+"""命令行入口。
+
+两个常用子命令：
+
+  replay   拿一段「多帧文本」的 JSONL 离线跑分页逻辑，验证你的参数
+           （不用真截屏，最适合调阈值 / 写回归）
+
+  watch    真的去截屏 + OCR + 累积，把结果追加到 jsonl
+
+watch 在 Windows/macOS 上都能用（依赖 mss），但你要自己保证被采集的窗口
+在前台 —— 本项目只负责「看图说话」，不负责把窗口调出来。
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from .accumulate import Accumulator, Frame, Mode
+from .config import Config
+from .dedup import Deduper, is_blank
+from .ocr import CmdOCR, NullOCR
+
+
+def _cfg_from_args(a) -> Config:
+    cfg = Config()
+    if a.window_s is not None:
+        cfg.window_s = a.window_s
+    if a.max_lines is not None:
+        cfg.max_lines = a.max_lines
+    if a.min_overlap is not None:
+        cfg.min_overlap = a.min_overlap
+    return cfg
+
+
+def cmd_replay(a) -> int:
+    """离线回放：每行一个 JSON，形如 {"ts": 1000.0, "lines": ["...", "..."]}。"""
+    cfg = _cfg_from_args(a)
+    acc = Accumulator(cfg)
+    n_roll = n_new = 0
+    with open(a.input, encoding="utf-8") as f:
+        for lineno, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError as e:
+                print(f"[跳过] 第 {lineno} 行不是合法 JSON: {e}", file=sys.stderr)
+                continue
+            frame = Frame(lines=rec.get("lines") or [], ts=float(rec.get("ts") or lineno))
+            m = acc.push(frame)
+            n_roll += m.mode is Mode.ROLL
+            n_new += m.mode is Mode.NEW
+            print(
+                json.dumps(
+                    {
+                        "frame": lineno,
+                        "mode": m.mode.value,
+                        "page": m.page,
+                        "anchor": m.anchor,
+                        "overlap": m.overlap,
+                        "lines": m.line_count,
+                        "gate": m.gate_log,
+                    },
+                    ensure_ascii=False,
+                )
+            )
+    print(f"\n合计 {n_new} 条新页 / {n_roll} 次合并", file=sys.stderr)
+    return 0
+
+
+def cmd_watch(a) -> int:
+    """真截屏循环。"""
+    try:
+        import mss  # noqa: F401
+        from PIL import Image
+    except ImportError:
+        print("watch 需要 mss 和 Pillow：pip install mss pillow", file=sys.stderr)
+        return 2
+
+    import io
+    import mss
+
+    cfg = _cfg_from_args(a)
+    ocr = CmdOCR(a.ocr_cmd) if a.ocr_cmd else NullOCR()
+    acc = Accumulator(cfg)
+    ded = Deduper(cfg)
+    out = open(a.output, "a", encoding="utf-8")
+    sct = mss.mss()
+    mon = sct.monitors[a.monitor]
+    print(f"采集区域 {mon}，间隔 {a.interval}s，输出 {a.output}", file=sys.stderr)
+
+    try:
+        while True:
+            shot = sct.grab(mon)
+            buf = io.BytesIO()
+            Image.frombytes("RGB", shot.size, shot.rgb).save(buf, format="PNG")
+            png = buf.getvalue()
+
+            blank, mean, std = is_blank(png, cfg)
+            if blank:
+                time.sleep(a.interval)
+                continue
+
+            dup, _h = ded.seen_pixels(png)
+            if dup:
+                time.sleep(a.interval)
+                continue
+
+            text = ocr.read(png)
+            lines = [l.strip() for l in text.splitlines() if l.strip()]
+            if not lines:
+                time.sleep(a.interval)
+                continue
+
+            m = acc.push(Frame(lines=lines, ts=time.time()))
+            if not acc.is_text_unchanged(m):
+                rec = {
+                    "ts": time.time(),
+                    "mode": m.mode.value,
+                    "page": m.page,
+                    "anchor": m.anchor,
+                    "scene": m.scene,
+                    "overlap": m.overlap,
+                    "text": m.text,
+                }
+                out.write(json.dumps(rec, ensure_ascii=False) + "\n")
+                out.flush()
+                print(f"[{m.mode.value}] {m.page}/{m.anchor} {m.line_count} 行", file=sys.stderr)
+            time.sleep(a.interval)
+    except KeyboardInterrupt:
+        print("\n停止。", file=sys.stderr)
+    finally:
+        out.close()
+        sct.close()
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="scrollbreaker", description="把滚动的屏幕切成能读的页")
+    p.add_argument("--version", action="version", version="scrollbreaker 0.1.0")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    for name, help_text in (("replay", "离线回放多帧文本"), ("watch", "实时截屏采集")):
+        s = sub.add_parser(name, help=help_text)
+        s.add_argument("--window-s", type=float, default=None, help="同一次阅读的时间窗（秒）")
+        s.add_argument("--max-lines", type=int, default=None, help="单次累积行数上限")
+        s.add_argument("--min-overlap", type=int, default=None, help="认定同页所需最小重合行数")
+        if name == "replay":
+            s.add_argument("input", help="JSONL：每行 {\"ts\":..., \"lines\":[...]}")
+            s.set_defaults(func=cmd_replay)
+        else:
+            s.add_argument("--interval", type=float, default=20.0, help="截屏间隔（秒）")
+            s.add_argument("--monitor", type=int, default=1, help="显示器编号（mss）")
+            s.add_argument("--output", default="scrollbreaker.jsonl", help="输出 JSONL")
+            s.add_argument("--ocr-cmd", nargs="+", default=None,
+                           help="OCR 命令行，{img} 占位。如: --ocr-cmd tesseract {img} stdout -l chi_sim+eng --psm 6")
+            s.set_defaults(func=cmd_watch)
+    return p
+
+
+def main(argv=None) -> int:
+    # Windows 控制台默认编码（cp936/cp1252）打印中文会直接抛 UnicodeEncodeError，
+    # 表现为 `scrollbreaker --version` 在 Windows 上崩掉、在 Linux/macOS 上正常。
+    # 在解析参数之前就把流改掉，因为 argparse 的 --version 是在 parse 时输出的。
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scrollbreaker/config.py

@@ -0,0 +1,88 @@
+"""所有可调参数集中在这里。
+
+这些默认值不是拍脑袋来的，是在真实的长文阅读 + 群聊滚动场景里调出来的：
+
+ROLL_WINDOW_S = 180
+    同一次阅读最多算 3 分钟。再久就是「过了一会儿又看」，不该拼。
+
+ROLL_MAX_LINES = 80
+    一次累积最多 80 行。再多上下文就膨胀，模型也读不动。
+
+ROLL_MIN_OVERLAP = 2
+    至少 2 行完全一样，才认作「同一页在滚动」。1 行太容易误判——
+    两个不同页面里都可能出现「好的」「。  
+
+TIME_LIKE / STATUS_BAR
+    状态栏（网速、电量、时间）每屏都在变，既不能当页头，也不能当重合证据。
+"""
+
+from dataclasses import dataclass, field
+from typing import Pattern
+import re
+
+
+@dataclass
+class Config:
+    # ── 累积闸门 ──────────────────────────────────────────────
+    window_s: float = 180.0        # 同一次阅读的时间窗（秒）
+    max_lines: int = 80            # 单次累积行数上限
+    min_overlap: int = 2           # 认定为「同一页」所需的最小重合行数
+
+    # ── 静屏去重 ──────────────────────────────────────────────
+    hash_bits: int = 64            # dHash 位数（64 = 8x8 差分）
+    same_bits: int = 6             # 汉明距离 ≤ 此值视为同一屏（不重复上报）
+
+    # ── 空白屏 ────────────────────────────────────────────────
+    blank_mean: float = 12.0       # 平均亮度低于此值 → 黑屏
+    blank_std: float = 3.0         # 亮度标准差低于此值 → 纯色屏
+
+    # ── 页面类型判定 ──────────────────────────────────────────
+    # 命中任一 → 认为这是「文章/长文」页（阅读类页面底部常驻这些按钮）
+    article_patterns: list[str] = field(default_factory=lambda: [
+        r"\+ ?关注", r"写留言", r"阅读原文", r"在看", r"点赞", r"分享到",
+    ])
+    # 命中任一 → 认为这是「列表/会话列表」页
+    list_patterns: list[str] = field(default_factory=lambda: [
+        r"\[图片\]", r"\[语音\]", r"\[视频\]", r"\[文件\]",
+        r"折叠(的)?置顶", r"\[\d+条\]",
+    ])
+    # 命中 → 这是一行「发言人: 内容」的聊天
+    chat_pattern: str = r"^\S{1,24}[:：]\s*\S"
+    # 状态栏噪声行（网速/电量/时间），既不能当页头也不能当重合证据
+    status_pattern: str = (
+        r"^[\d\.\s]*(KB/S|M/s|MB/s|KB/s|G|%)?[\d\.\s]*$"
+        r"|^\d{1,2}[:：]\d{2}$"
+    )
+    # 页头锚点保留长度：不同会话/不同文章前 N 个字通常就够区分
+    anchor_len: int = 14
+    # 文章页拼接时，前几行里找「第一行真正的正文」用的最小字数
+    body_min_chars: int = 12
+    body_scan_lines: int = 5
+    # 横幅行上限字数：短于这个且不含文章控件特征的行，判为「顶部通知横幅」（"张三" / "在吗"）
+    body_banner_max_chars: int = 8
+
+    # ── 归一化 ────────────────────────────────────────────────
+    # 整段文本归一化后与上一段相同 → 判为重复（只变了时间/电量）
+    collapse_ws: bool = True
+    max_text_chars: int = 4000
+
+    def compiled(self) -> "CompiledPatterns":
+        return CompiledPatterns(self)
+
+
+@dataclass
+class CompiledPatterns:
+    article: "Pattern[str]"
+    list_sig: "Pattern[str]"
+    chat: "Pattern[str]"
+    status: "Pattern[str]"
+
+    def __init__(self, cfg: Config):
+        joiner = "|"
+        self.article = re.compile(joiner.join(cfg.article_patterns)) if cfg.article_patterns else None
+        self.list_sig = re.compile(joiner.join(cfg.list_patterns)) if cfg.list_patterns else None
+        self.chat = re.compile(cfg.chat_pattern)
+        self.status = re.compile(cfg.status_pattern)
+
+
+DEFAULT_CONFIG = Config()

scrollbreaker/dedup.py

@@ -0,0 +1,110 @@
+"""静屏去重：屏幕没动就别重复上报。
+
+两种「没动」：
+  1. 内容完全一样 → dHash 汉明距离极小
+  2. 状态栏时间跳了一分钟，但正文一个字没变 → 整段归一化后完全一致
+
+第一种在截图层挡掉（省 OCR），第二种在文本层挡掉（省 token）。
+"""
+
+from __future__ import annotations
+
+import io
+from typing import Optional, Sequence
+
+from .config import Config, DEFAULT_CONFIG
+
+
+def _to_gray(image_bytes: bytes):
+    """把图片字节转成 9x8 灰度二维列表（dHash 只要这个）。"""
+    from PIL import Image  # 延迟导入：只用文本链路的人不必装 Pillow
+
+    im = Image.open(io.BytesIO(image_bytes)).convert("L").resize((9, 8))
+    px = list(im.getdata())
+    return [px[i * 9:(i + 1) * 9] for i in range(8)]
+
+
+def dhash(image_bytes: bytes) -> int:
+    """差分哈希（dHash）。
+
+    做法：缩到 9x8，逐行比较相邻像素大小关系，得到 8x8=64 位。
+    相比 aHash，dHash 对整体亮度变化不敏感——状态栏变亮变暗不会误判成「新画面」。
+    """
+    grid = _to_gray(image_bytes)
+    bits = 0
+    for row in grid:
+        for x in range(8):
+            bits = (bits << 1) | (1 if row[x] > row[x + 1] else 0)
+    return bits
+
+
+def hamming(a: int, b: int) -> int:
+    """两个哈希的差异位数。"""
+    return bin(a ^ b).count("1")
+
+
+def is_duplicate(new_hash: int, prev_hash: Optional[int],
+                 cfg: Config = DEFAULT_CONFIG) -> bool:
+    """这一屏和上一屏是不是基本同一屏。"""
+    if prev_hash is None:
+        return False
+    return hamming(new_hash, prev_hash) <= cfg.same_bits
+
+
+def is_blank(image_bytes: bytes, cfg: Config = DEFAULT_CONFIG) -> tuple[bool, float, float]:
+    """空白屏判定（息屏 / 纯色 / 全白）。
+
+    返回 (is_blank, mean, std)。
+    黑屏时 OCR 输出 0 段是**正常现象**，不能判为采集失败——早期版本把它
+    当错误处理，日志里全是不存在的「OCR 失败」。
+    """
+    try:
+        from PIL import Image
+
+        im = Image.open(io.BytesIO(image_bytes)).convert("L").resize((48, 48))
+        px = list(im.getdata())
+        n = len(px)
+        mean = sum(px) / n
+        var = sum((p - mean) ** 2 for p in px) / n
+        std = var ** 0.5
+        return (mean < cfg.blank_mean or std < cfg.blank_std), mean, std
+    except Exception:
+        return False, -1.0, -1.0
+
+
+class Deduper:
+    """按场景记忆「最后一屏」，供循环里反复调用。"""
+
+    def __init__(self, cfg: Config = DEFAULT_CONFIG):
+        self.cfg = cfg
+        self._last_px_hash: Optional[int] = None
+        self._last_norm: Optional[str] = None
+
+    def seen_pixels(self, image_bytes: bytes) -> tuple[bool, int]:
+        """像素层去重：返回 (是否重复, 本次哈希)。"""
+        h = dhash(image_bytes)
+        dup = is_duplicate(h, self._last_px_hash, self.cfg)
+        self._last_px_hash = h
+        return dup, h
+
+    def seen_text(self, normalized: str) -> bool:
+        """文本层去重：归一化后与上次完全一致即视为重复。"""
+        dup = normalized == self._last_norm
+        self._last_norm = normalized
+        return dup
+
+    def reset(self) -> None:
+        self._last_px_hash = None
+        self._last_norm = None
+
+
+def dedup_lines(lines: Sequence[str]) -> list[str]:
+    """同一屏内的完全重复行只保留第一次出现（保序）。"""
+    seen: set[str] = set()
+    out: list[str] = []
+    for l in lines:
+        if l in seen:
+            continue
+        seen.add(l)
+        out.append(l)
+    return out

scrollbreaker/ocr.py

@@ -0,0 +1,91 @@
+"""OCR 适配层：本项目不绑定任何 OCR 引擎。
+
+设计原则：**OCR 是可替换的**。滚动分页逻辑才是这个项目的价值，
+所以这里只定义一个极小的协议，你想用哪个引擎都行。
+
+自带三种：
+  NullOCR      —— 不做 OCR，直接把传入的文本当结果（写测试/离线回放用）
+  CmdOCR       —— 调任意命令行 OCR（tesseract / paddleocr-cli / 自建服务）
+  CallableOCR  —— 传一个 Python 函数进来
+
+如果你的引擎返回带坐标的结果，按行分组后传进来即可 —— 本项目只关心「行」。
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Callable, Protocol, Sequence
+
+
+class OCR(Protocol):
+    def read(self, image_bytes: bytes) -> str:  # pragma: no cover - protocol
+        ...
+
+
+class NullOCR:
+    """不识别，返回空串（或一个固定文本，便于测试）。"""
+
+    def __init__(self, fixed: str = ""):
+        self.fixed = fixed
+
+    def read(self, image_bytes: bytes) -> str:
+        return self.fixed
+
+
+class CallableOCR:
+    """包装一个 (bytes) -> str 的函数。"""
+
+    def __init__(self, fn: Callable[[bytes], str]):
+        self.fn = fn
+
+    def read(self, image_bytes: bytes) -> str:
+        return self.fn(image_bytes)
+
+
+class CmdOCR:
+    """调命令行 OCR。
+
+    tesseract 例子：
+        CmdOCR(["tesseract", "{img}", "stdout", "-l", "chi_sim+eng", "--psm", "6"])
+
+    `{img}` 会被替换成临时图片路径。输出里的行顺序即屏幕顺序。
+    """
+
+    def __init__(self, argv: Sequence[str], timeout: float = 30.0, cleanup: bool = True):
+        self.argv = list(argv)
+        self.timeout = timeout
+        self.cleanup = cleanup
+
+    def available(self) -> bool:
+        if not self.argv:
+            return False
+        if shutil.which(self.argv[0]) is None:
+            return False
+        return True
+
+    def read(self, image_bytes: bytes) -> str:
+        if not self.available():
+            raise RuntimeError(f"OCR 可执行文件不存在: {self.argv[0] if self.argv else '(empty)'}")
+        suffix = ".png"
+        if image_bytes[:2] == b"\xff\xd8":
+            suffix = ".jpg"
+        fd, path = tempfile.mkstemp(suffix=suffix)
+        try:
+            with open(fd, "wb") as f:
+                f.write(image_bytes)
+            argv = [a.replace("{img}", path) for a in self.argv]
+            out = subprocess.run(argv, capture_output=True, timeout=self.timeout)
+            if out.returncode != 0:
+                raise RuntimeError(out.stderr.decode("utf-8", "ignore")[:300])
+            return out.stdout.decode("utf-8", "ignore")
+        finally:
+            if self.cleanup:
+                Path(path).unlink(missing_ok=True)
+
+
+def tesseract(lang: str = "chi_sim+eng") -> CmdOCR:
+    """tesseract 的常用配置（psm 6 = 假设一列文本，屏幕识别通常最稳）。"""
+    return CmdOCR(["tesseract", "{img}", "stdout", "-l", lang, "--psm", "6"])

scrollbreaker/pagetype.py

@@ -0,0 +1,165 @@
+"""页面类型判定：这一屏到底是什么？
+
+为什么必须先判类型：
+    只看「两屏有没有重合行」是不够的。实测中，公众号文章和聊天窗口之间
+    出现过 6 行完全相同的行（都是「正常」「好的」这类字），按重合拼就
+    把一篇文章和一段聊天糊成了一条记录——数据彻底废掉。
+
+所以先给每一屏定性，只有**同类型**的两屏才可能被拼到一起。
+"""
+
+import string
+from typing import Iterable, Sequence
+
+from .config import CompiledPatterns, Config, DEFAULT_CONFIG
+
+# 页面类型
+ARTICLE = "article"   # 文章/长文正文页
+CHAT = "chat"         # 单聊/群聊会话页
+LIST = "list"         # 会话列表 / 消息列表
+TEXT = "text"         # 其他（设置页、纯文本、表单…）
+MIXED = "mixed"       # 一屏里同时有文章特征和聊天/列表特征
+
+#: 可以参与累积的类型。mixed 是「切换过程中的混合画面」，既不能当起点也不能被吸收。
+ACCUMULABLE = (ARTICLE, CHAT, LIST, TEXT)
+
+
+def page_type(lines: Sequence[str], cfg: Config = DEFAULT_CONFIG,
+              pats: CompiledPatterns | None = None) -> str:
+    """给一屏文本定性。
+
+    Args:
+        lines: 已按屏幕顺序切好的文本行
+        cfg: 配置（阈值都在里面）
+        pats: 预编译的正则（循环里复用，别每帧重编译）
+
+    Returns:
+        ARTICLE / CHAT / LIST / TEXT / MIXED
+    """
+    p = pats or cfg.compiled()
+    chat = sum(1 for l in lines if p.chat.match(l))
+    art = sum(1 for l in lines if p.article and p.article.search(l))
+    lst = sum(1 for l in lines if p.list_sig and p.list_sig.search(l))
+
+    # 同时具备两类特征 → 这屏本身就是切页中间态，判 mixed 单独处理
+    if art and (chat >= 2 or lst >= 2):
+        return MIXED
+    if art:
+        return ARTICLE
+    if chat >= 2:
+        return CHAT
+    if lst:
+        return LIST
+    return TEXT
+
+
+def head_anchor(lines: Sequence[str], cfg: Config = DEFAULT_CONFIG,
+                pats: CompiledPatterns | None = None) -> str:
+    """页头锚点：当前页的「身份」。
+
+    聊天页的第一行通常是联系人名，文章页的第一行通常是标题，
+    正文页的第一行通常是文章名。取它做锚点，锚点变了 = 换页了。
+
+    尾部数字要剥掉：未读数是「张三 3」→「张三 12」，数字变了但人没变。
+    """
+    p = pats or cfg.compiled()
+    for line in lines[:2]:
+        s = line.strip()
+        if not s or p.status.match(s):
+            continue
+        # 去掉尾部数字（未读数 / 计数）
+        stripped = s.rstrip(string.digits + " ")
+        return stripped[: cfg.anchor_len]
+    return ""
+
+
+def body_lines(lines: Sequence[str], cfg: Config = DEFAULT_CONFIG) -> Sequence[str]:
+    """文章页取正文：跳过顶部那条「新消息横幅」。
+
+    很多应用在阅读文章时会在顶部浮一条消息通知（「某某: 在吗」）。
+    它会被 OCR 抓到正文前面，拼进去就成了「文章里夹着聊天」。
+    做法：在前几行里找第一行足够长的（≥ body_min_chars），从那行开始算正文。
+    找不到足够长的行时，退一步只剥掉那些「明显是横幅」的行（短行 + 聊天特征）。
+    """
+    head = lines[: cfg.body_scan_lines]
+    pats = cfg.compiled()
+    for i, line in enumerate(head):
+        s = line.strip()
+        if len(s) < cfg.body_min_chars:
+            continue
+        # 控件行（+关注 / 写留言 / 点赞…）不是正文，别拿它当正文起点
+        if pats.article and pats.article.search(s):
+            continue
+        return lines[i:]
+    # 全屏都是短行：剥掉确实像「通知横幅」的那些（2~8 字、非控件行），其余保留。
+    # 阈值故意卡在 8 字：真实文章正文行一般 15~30 字，不会这么短；
+    # 而横幅就是「张三」「在吗」这种 2~3 个字。
+    cut = 0
+    for i, line in enumerate(head):
+        s = line.strip()
+        is_banner_like = (
+            len(s) <= cfg.body_banner_max_chars
+            and not (pats.article and pats.article.search(s))
+        )
+        if is_banner_like:
+            cut = i + 1
+            continue
+        break
+    return lines[cut:] if cut else lines
+
+
+def chat_anchor(lines: Sequence[str], cfg: Config = DEFAULT_CONFIG,
+                pats: CompiledPatterns | None = None) -> str:
+    """聊天页的「同一会话」判据。
+
+    不能只看第一行：聊天页滚动时，第一行是**第一条消息**会变，页头（联系人名）
+    可能被滚出屏幕。实测里两种页面都存在，所以做一个鲁棒判据：
+
+      1. 若第一行不是「发言人: 内容」形态 → 它就是页头，直接用（最常见）
+      2. 否则 → 用本屏出现过的发言人集合（排序后拼接），同一会话的发言人集合稳定
+
+    第 2 种在「页头滚出屏幕」时特别有用：不同会话的发言人集合几乎不会重合。
+    """
+    p = pats or cfg.compiled()
+    for line in lines[:2]:
+        s = line.strip()
+        if not s or p.status.match(s):
+            continue
+        if not p.chat.match(s):
+            return s.rstrip(string.digits + " ")[: cfg.anchor_len]
+        break
+
+    speakers: list[str] = []
+    for line in lines:
+        m = p.chat.match(line)
+        if not m:
+            continue
+        name = line.split(":", 1)[0].split("：", 1)[0].strip()
+        if name and name not in speakers:
+            speakers.append(name)
+    return "|".join(sorted(speakers)[:4])[: cfg.anchor_len * 2]
+
+
+
+def normalize(text: str, cfg: Config = DEFAULT_CONFIG) -> str:
+    """整段归一化，用于判「内容一个字都没变」。"""
+    if not cfg.collapse_ws:
+        return text
+    return " ".join(text.split())
+
+
+def clean_lines(raw: str, min_len: int = 1) -> list[str]:
+    """把 OCR 出来的整段文本切成「有意义」的行：去空行、去纯符号、去重后再返回。
+
+    注意：这里**不做跨屏去重**，跨屏去重是 Accumulator 的活——
+    同一屏里重复出现的行（如列表里的分隔符）保留原位更利于比对。
+    """
+    out: list[str] = []
+    for line in raw.splitlines():
+        s = line.strip()
+        if len(s) < min_len:
+            continue
+        if all(not ch.isalnum() for ch in s):   # 纯标点/纯符号行
+            continue
+        out.append(s)
+    return out

scrollbreaker-0.1.0.dist-info/METADATA

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: scrollbreaker
+Version: 0.1.0
+Summary: Scroll-aware screen content collector for LLM agents — 把滚动的屏幕切成能读的页
+Author: scrollbreaker contributors
+License: MIT
+Keywords: ocr,screen,scroll,llm,agent,capture,information-collector,content-collector
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: image
+Requires-Dist: pillow>=9; extra == "image"
+Requires-Dist: mss>=9; extra == "image"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# ScrollBreaker
+
+**A scroll-aware screen content collector for LLM agents** — 一个会分辨页面的信息采集器。
+
+> **Your LLM is eating garbage. The screenshots are the problem.**
+>
+> Every scroll-capture tool on the planet splices frames together by line overlap.
+> That works — until it silently fuses an article and a chat log into one record
+> that looks perfectly normal. ScrollBreaker stops that with **four gates**.
+
+Most **information collectors** answer the question *"how do I grab the screen?"*
+ScrollBreaker answers the harder one: *"are these two grabs the same page at all?"*
+
+
+Screenshot → OCR → merge is the standard pipeline for feeding visual content to an
+agent. It has one failure mode that poisons your dataset and never warns you:
+**unrelated pages that happen to share lines.**
+
+```
+Article page : 正常 / 好的 / 收到 / 在看 / +关注 / body paragraph...
+Chat page    : 正常 / 好的 / 收到 / Alice: stop machine 4 / Bob: on it
+Shared lines : 3
+```
+
+Any tool that merges on "overlap ≥ 2" fuses these two. The result is one record
+containing an article with a factory conversation stapled into its middle. It parses
+fine. It embeds fine. It is **wrong**, and nothing downstream will tell you.
+
+ScrollBreaker refuses to merge unless four gates all pass.
+
+## The four gates
+
+| # | Gate | Rule | Default |
+|---|------|------|---------|
+| 1 | Time window | Frames further apart than N seconds are a different reading session | `180s` |
+| 2 | Same type | `article` / `chat` / `list` / `text` must match | — |
+| 3 | Same anchor | Same page header (relaxed for articles — titles scroll away) | — |
+| 4 | Real overlap | ≥ N **identical** lines, because 1 line is noise | `2` |
+
+Fail any one → **new record, no merge. Ever.**
+
+## Install
+
+No PyPI account needed — the wheel is attached to the release:
+
+```bash
+pip install https://github.com/xiangjianhuan007/scrollbreaker/releases/download/v0.1.0/scrollbreaker-0.1.0-py3-none-any.whl
+```
+
+Or from source, if you want to poke at it:
+
+```bash
+git clone https://github.com/xiangjianhuan007/scrollbreaker
+cd scrollbreaker
+pip install -e ".[dev]"
+pytest -q
+```
+
+Verify the install the same way CI does:
+
+```bash
+scrollbreaker --version
+scrollbreaker replay examples/frames.jsonl
+```
+
+Zero runtime dependencies. The `image` extra (`pillow`, `mss`) is only needed if you
+want to feed it real screenshots instead of OCR'd text.
+
+
+There is a fifth state, `mixed`: a frame containing both article and chat/list
+signatures. That frame *is* a page transition in progress, so it can neither start
+an accumulation nor be absorbed into one.
+
+## Quick start
+
+```bash
+pip install -e .
+# for live capture: pip install mss pillow
+```
+
+```python
+from scrollbreaker import Accumulator, Frame, Mode
+
+acc = Accumulator()                      # every threshold lives in Config
+
+for snap in screenshots:                 # one frame every N seconds
+    m = acc.push(Frame(lines=snap.lines, ts=snap.ts))
+
+    if m.mode is Mode.ROLL:
+        pass                             # same page scrolling; m.text is the full text
+    else:
+        handle_page(m.text)              # page changed; previous record is complete
+```
+
+Every decision is auditable — `m.gate_log` tells you which gate failed:
+
+```python
+{"time_window": True, "same_type": False, "same_anchor": False,
+ "anchor_relaxed": True, "overlap_enough": False}
+```
+
+### CLI
+
+```bash
+# offline replay — tune thresholds without capturing anything
+scrollbreaker replay examples/frames.jsonl
+
+# live capture
+scrollbreaker watch --interval 10 \
+  --ocr-cmd tesseract {img} stdout -l eng --psm 6 \
+  --output records.jsonl
+```
+
+Real output from `examples/frames.jsonl`:
+
+```
+frame 1  new   chat     anchor=Alice|Bob
+frame 2  roll  chat     anchor=Alice|Bob   overlap=2       <- same conversation, merged
+frame 3  new   article  same_type=false                    <- switched pages, hard break
+```
+
+That third line is the whole point: **wrong type, no merge — no matter the overlap.**
+
+## Android capture app (optional)
+
+`android/` contains the phone half of the pair: interval screen capture → dHash
+dedup → refuse password/payment screens → POST the frame to an endpoint you control.
+
+It is source, not a shipped app — you build it yourself, and **nothing leaves the
+device until you enter an endpoint**. Requires Android 11+ (API 30) because
+`AccessibilityService.takeScreenshot()` is the only unattended capture API.
+
+```
+android/          769 lines Kotlin, zero dependencies
+  CaptureService.kt   the loop + the four gates + the privacy gate
+  ImageSig.kt         dHash + blank detection
+  Uplink.kt           one POST, no SDK
+  CaptureConfig.kt    everything configurable, nothing hard-coded
+  MainActivity.kt     settings screen (built in code, no layouts)
+```
+
+See [android/README.md](android/README.md) for the permission trade-off, the payload
+shape and a 10-line receiver.
+
+
+## Output shape
+
+```json
+{"ts": 1000.0, "mode": "roll", "page": "chat", "anchor": "Alice|Bob",
+ "scene": "chat|Alice|Bob", "overlap": 2, "text": "..."}
+```
+
+`scene` is the field that matters downstream. Group by `scene` when you store,
+embed, or summarize, and unrelated captures can no longer contaminate each other.
+
+## Design trade-offs
+
+**OCR is a plug-in.** The whole contract is `read(bytes) -> str`. Use tesseract,
+PaddleOCR, a cloud API, a local VLM — ScrollBreaker never looks at pixels, only lines.
+
+**Screen only, no database spelunking.** No file parsing, no process injection,
+no root. You lose some fidelity versus reading the source directly; you gain
+application-agnostic capture that survives app updates.
+
+**Thresholds are empirical, not laws.** `180s / 80 lines / 2 overlap` came from real
+long-form reading and group-chat scrolling. Monitoring a dashboard or a log tail?
+Change `Config`.
+
+**Chat anchors degrade gracefully.** A chat header scrolls out of view, at which point
+the first line is a message, not a header. ScrollBreaker falls back to the set of
+speakers seen in the frame — stable within a conversation, rarely shared across two.
+
+## Test suite
+
+```bash
+pip install -e ".[dev]" && pytest -q
+```
+
+9 tests covering each gate. The one that matters is `test_article_to_chat_never_merged`:
+two frames sharing **6 lines** with different page types must still produce two records.
+It exists because that exact fusion happened and corrupted real data. Run it before
+touching the merge logic.
+
+## Compliance
+
+- Reads **only what is visible on screen**. No application data files, no databases,
+  no process memory.
+- **No network, no telemetry.** Everything stays local; output is a JSONL on your disk.
+- Capturing other people's screens, or redistributing captured content, may violate
+  local law and the target platform's terms. **Use it on your own device, your own
+  account, your own visible content.**
+- This project provides no authentication bypass, no database decryption, and no
+  process injection — and will not accept PRs that add them.
+
+## 中文说明
+
+**ScrollBreaker（破滚者）** —— 把滚动的屏幕切成能读的页。
+
+核心不是「截图 + OCR」，是**判断两屏到底是不是同一页**。绝大多数采集工具按「有
+两行重合」就拼，于是把一篇文章和一段聊天糊成一条记录——数据被污染了还看不出来。
+
+四道闸门（时间窗 / 类型一致 / 页头一致 / 实质重合）全部通过才允许合并，任一条不过
+就果断断开。`mixed`（一屏里同时出现文章和聊天特征）是切页中间态，既不累积也不被吸收。
+
+阈值都在 `Config` 里，改成你的场景就行。
+
+## License
+
+MIT — do whatever you want, just don't blame us when your OCR is worse than ours.

scrollbreaker-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+scrollbreaker/__init__.py,sha256=KNIbn7DHmbK66eCRTwKn8yAbY3R2NY2ByQD-SWnDhqg,1066
+scrollbreaker/accumulate.py,sha256=8UyCMQQtKzRWrPpMRqTZYEfvBt8ktU9q8jKRm1DBGBQ,7668
+scrollbreaker/cli.py,sha256=vhJCxQ8tgyK9piZO6YXoBmygimSLKM8MVIm5RVhgOHY,6525
+scrollbreaker/config.py,sha256=KQunXF7pNo0J-9_MxRxEGPWXzxLi5xbHAKKUEHLjy0Q,4211
+scrollbreaker/dedup.py,sha256=NxAb0B2K26ofDYcy3CqnBk0pn4Q8WIBTTZbDo_VG-FM,3610
+scrollbreaker/ocr.py,sha256=A_temluosAxBeZfNVvsH8E1rKEf9QZkoxn8lcHQcDzc,3023
+scrollbreaker/pagetype.py,sha256=wdCxIzO761TBnp0DQdif97_Hcs6kmQP_baORnrab2Zc,6472
+scrollbreaker-0.1.0.dist-info/licenses/LICENSE,sha256=qY3DTT14F8rlKSEgExvukmGV3JNUIE3DTJXDoCqOby8,1079
+scrollbreaker-0.1.0.dist-info/METADATA,sha256=BML79-TO6T61iTKByVVfkGl_aPrhjxmOIyGdU8sN4ik,8534
+scrollbreaker-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+scrollbreaker-0.1.0.dist-info/entry_points.txt,sha256=TagwEjxjvLdsw8_14MGBrnBQcNkal-D7bkxxlbs76Z0,57
+scrollbreaker-0.1.0.dist-info/top_level.txt,sha256=yUPq6epfsCMSlTaVpZhYT9OZjNMV5WKjFuWVrN-ViyY,14
+scrollbreaker-0.1.0.dist-info/RECORD,,

scrollbreaker-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrollbreaker-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+scrollbreaker = scrollbreaker.cli:main

scrollbreaker-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 scrollcap contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrollbreaker-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrollbreaker