dp-cli 0.7.0__tar.gz → 0.8.0__tar.gz

dp_cli-0.8.0/MANIFEST.in

@@ -0,0 +1 @@
+graft skills

{dp_cli-0.7.0 → dp_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dp-cli
-Version: 0.7.0
+Version: 0.8.0
 Summary: A powerful CLI for DrissionPage — browser automation, structured data extraction, network listening and more.
 License: BSD-3-Clause
 Project-URL: Homepage, https://github.com/mofanx/dp-cli
@@ -19,6 +19,11 @@ Requires-Dist: click>=8.0
 Requires-Dist: aiohttp>=3.9
 Requires-Dist: websockets>=12
 Requires-Dist: requests>=2.28
+Provides-Extra: tls
+Requires-Dist: curl_cffi>=0.7.0; extra == "tls"
+Provides-Extra: extract
+Requires-Dist: trafilatura>=1.6.0; extra == "extract"
+Requires-Dist: readability-lxml>=0.8.1; extra == "extract"
 Provides-Extra: test
 Requires-Dist: pytest>=7.4.0; extra == "test"
 Requires-Dist: pytest-cov>=4.1.0; extra == "test"
@@ -54,6 +59,18 @@ pip install dp-cli
 dp --help
 ```
 
+### Agent skill（可选，推荐）
+
+为 AI agent 安装完整使用指南（全部命令参考 + 工作流 playbook）：
+
+```bash
+dp install --skills                        # 装到 agent 的 skill 目录
+npx skills add mofanx/dp-cli               # 或走 skills CLI 渠道
+claude plugin marketplace add mofanx/dp-cli  # 或 Claude Code 插件市场
+```
+
+未安装也能用——skill 只是文档层；agent 可直接读 `dp --help` 或 `python -m dp_cli.catalog` 自举。
+
 ## Quick Start
 
 ```bash

{dp_cli-0.7.0 → dp_cli-0.8.0}/README.md

@@ -26,6 +26,18 @@ pip install dp-cli
 dp --help
 ```
 
+### Agent skill（可选，推荐）
+
+为 AI agent 安装完整使用指南（全部命令参考 + 工作流 playbook）：
+
+```bash
+dp install --skills                        # 装到 agent 的 skill 目录
+npx skills add mofanx/dp-cli               # 或走 skills CLI 渠道
+claude plugin marketplace add mofanx/dp-cli  # 或 Claude Code 插件市场
+```
+
+未安装也能用——skill 只是文档层；agent 可直接读 `dp --help` 或 `python -m dp_cli.catalog` 自举。
+
 ## Quick Start
 
 ```bash

dp_cli-0.8.0/dp_cli/catalog.py

@@ -0,0 +1,109 @@
+# -*- coding:utf-8 -*-
+"""从 click 命令树机械导出命令元数据。
+
+两类消费方：
+- ``python -m dp_cli.catalog --md``  → 生成 commands.md 风格的命令表
+- ``python -m dp_cli.catalog``       → 输出 MCP 风格 tool schema JSON
+  （未来 mcp-serve 薄壳的上游；schema 由代码生成，不手写，不腐化）
+
+文档腐化守卫见 tests/test_docs.py：commands.md 必须覆盖全部注册命令。
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+import click
+
+from dp_cli.main import cli
+
+# 模块名 → 命令表类别（按模块机械归类，新增模块时在此补一行）
+CATEGORY = {
+    'browser': '浏览器', 'misc': '导航/页面', 'snapshot_cmd': '快照',
+    'element': '提取/交互', 'keyboard': '键盘/滚动', 'network': '网络',
+    'assert_cmd': '断言', 'emulate': '模拟/下载/拦截', 'observe': '观测',
+    'tab': '标签页', 'storage': '存储', 'record': '录制',
+    'page': '页面', 'batch': '批量', 'rpc': '逆向',
+    'extract': '提取', 'hook': '逆向', 'serve': '批量', 'sh': '批量',
+    'meta': '元信息',
+}
+
+
+def _param_schema(p):
+    """click.Param → JSON Schema 片段。"""
+    if isinstance(p, click.Option) and p.is_flag:
+        t = {'type': 'boolean'}
+    elif isinstance(p.type, click.types.IntParamType):
+        t = {'type': 'integer'}
+    elif isinstance(p.type, click.types.FloatParamType):
+        t = {'type': 'number'}
+    elif isinstance(p.type, click.types.BoolParamType):
+        t = {'type': 'boolean'}
+    elif isinstance(p.type, click.types.Choice):
+        t = {'type': 'string', 'enum': list(p.type.choices)}
+    else:
+        t = {'type': 'string'}
+    if getattr(p, 'multiple', False):
+        t = {'type': 'array', 'items': t}
+    if getattr(p, 'help', None):
+        t['description'] = p.help
+    return t
+
+
+def iter_commands():
+    """yield (name, help, input_schema) —— MCP tool 风格。"""
+    for name in sorted(cli.commands):
+        cmd = cli.commands[name]
+        props, required = {}, []
+        for p in cmd.params:
+            props[p.name] = _param_schema(p)
+            if p.required:
+                required.append(p.name)
+        yield {
+            'name': name,
+            'description': (cmd.help or '').strip().splitlines()[0] if cmd.help else '',
+            'inputSchema': {'type': 'object', 'properties': props,
+                            'required': required},
+            '_category': CATEGORY.get(
+                getattr(cmd.callback, '__module__', '').rsplit('.', 1)[-1],
+                '其他'),
+        }
+
+
+def to_markdown(cmds):
+    """生成按类别分组的 markdown 表（含 options 摘要）。"""
+    by_cat = {}
+    for c in cmds:
+        by_cat.setdefault(c['_category'], []).append(c)
+    lines = ['<!-- GENERATED by `python -m dp_cli.catalog --md` —— 勿手改 -->',
+             '', '| 类别 | 命令 | 参数 |', '|------|------|------|']
+    for cat in CATEGORY.values():
+        for c in by_cat.pop(cat, []):
+            params = ', '.join(
+                (f'<{k}>' if k in c['inputSchema']['required'] else f'[{k}]')
+                for k in c['inputSchema']['properties'])
+            lines.append(f"| {cat} | `{c['name']}` | {params} |")
+    for cat, cs in sorted(by_cat.items()):
+        for c in cs:
+            params = ', '.join(c['inputSchema']['properties'])
+            lines.append(f"| {cat} | `{c['name']}` | {params} |")
+    return '\n'.join(lines) + '\n'
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog='dp-cli-catalog')
+    ap.add_argument('--md', action='store_true', help='输出 markdown 命令表')
+    args = ap.parse_args(argv)
+    cmds = list(iter_commands())
+    if args.md:
+        sys.stdout.write(to_markdown(cmds))
+    else:
+        tools = [{k: v for k, v in c.items() if not k.startswith('_')}
+                 for c in cmds]
+        json.dump({'tools': tools}, sys.stdout, ensure_ascii=False, indent=2)
+        sys.stdout.write('\n')
+
+
+if __name__ == '__main__':
+    main()

{dp_cli-0.7.0 → dp_cli-0.8.0}/dp_cli/commands/__init__.py

@@ -2,11 +2,13 @@
 from dp_cli.commands import (
     browser, snapshot_cmd, element, keyboard,
     page, tab, storage, network, record, misc,
-    assert_cmd, emulate, observe,
+    assert_cmd, emulate, observe, batch, rpc, extract, hook, serve,
+    meta,
 )
 
 _MODULES = [browser, snapshot_cmd, element, keyboard, page, tab, storage, network, record, misc,
-            assert_cmd, emulate, observe]
+            assert_cmd, emulate, observe, batch, rpc, extract, hook, serve,
+            meta]
 
 
 def register_all(cli):

{dp_cli-0.7.0 → dp_cli-0.8.0}/dp_cli/commands/_utils.py

@@ -3,6 +3,7 @@
 import io
 import csv
 import re
+import threading
 from time import perf_counter, sleep as _sleep
 
 import click
@@ -25,6 +26,34 @@ def session_option(f):
                         help='会话名称，默认 default', show_default=True)(f)
 
 
+def trim_options(f):
+    """--fields/--head 输出裁剪选项组（记录列表类命令共用）。"""
+    f = click.option('--head', type=int, default=None,
+                     help='只输出前 N 条记录（附 total 字段说明截断）')(f)
+    f = click.option('--fields', default=None,
+                     help='字段白名单，逗号分隔；非法字段名报错')(f)
+    return f
+
+
+def shape_records(records, fields, head, valid_fields):
+    """应用 --fields/--head 裁剪，返回 (shaped, total)；参数非法经 error() 退出。"""
+    total = len(records)
+    if fields is not None:
+        keys = [k.strip() for k in fields.split(',') if k.strip()]
+        bad = [k for k in keys if k not in valid_fields]
+        if not keys or bad:
+            error(f'--fields 非法字段: {", ".join(bad) or "（空）"}'
+                  f'（可用: {", ".join(sorted(valid_fields))}）',
+                  code='INVALID_ARGS')
+        records = [{k: (r.get(k) if isinstance(r, dict) else None)
+                    for k in keys} for r in records]
+    if head is not None:
+        if head < 0:
+            error('--head 需为非负整数', code='INVALID_ARGS')
+        records = records[:head]
+    return records, total
+
+
 def frame_option(f):
     return click.option('--frame', default=None,
                         help='限定在 iframe 内查找元素（值为 iframe 的定位器，'
@@ -121,10 +150,106 @@ def _get_page(session: str, raw: bool = False, inject_recording: bool = True):
             _apply_intercept(target, intercept_cfg)
         except Exception as e:
             debug('intercept 重放失败', e)
+    browser_cdp_cfg = sess.get('browser_cdp')
+    if browser_cdp_cfg:
+        try:
+            from dp_cli.commands.browser import _apply_browser_cdp
+            _apply_browser_cdp(target, browser_cdp_cfg)
+        except Exception as e:
+            debug('browser_cdp 重放失败', e)
+    _replay_persist_scripts(target, session, sess)
 
     return target
 
 
+def _replay_persist_scripts(target, session, sess, only_ids=None):
+    """重放会话持久化 init-js：rpc/hooks 的 N 个脚本合并为单个 bundle
+    一次 addScriptToEvaluateOnNewDocument 注册，省 N-1 个 CDP RTT。
+
+    每段 try/catch 包裹隔离运行时异常（原独立注册语义）；各段均为
+    生成器模板 + json.dumps 插值，语法安全由生成器保证。
+    js_id 共享 bundle id 并落盘：跨进程 unregister/clear 拿到当前
+    生效 id 才能移除整包（否则只见注册时单 id，bundle 残留无法清除）。
+
+    :param only_ids: 只重放 js_id 命中该集合的 spec——用于 unregister/
+                     clear 移除整包后精准补回其中剩余段；其它 js_id 的
+                     脚本仍在 target 上存活，不得重复注册。None 表示
+                     全量重放（_get_page 路径）。
+    """
+    def _wanted(spec):
+        return only_ids is None or spec.get('js_id') in only_ids
+
+    bundle_parts = []
+
+    rpc_cfg = sess.get('rpc')
+    if rpc_cfg:
+        try:
+            from dp_cli.commands.rpc import _register_js
+            for rpc_name, spec in rpc_cfg.items():
+                if not _wanted(spec):
+                    continue
+                try:
+                    bundle_parts.append(_register_js(rpc_name, spec['body']))
+                except Exception as e:
+                    debug(f'rpc 重放失败 name={rpc_name}', e)
+        except Exception as e:
+            debug('rpc 重放失败', e)
+    hooks_cfg = sess.get('hooks')
+    if hooks_cfg:
+        try:
+            from dp_cli.commands.hook import _hook_js
+            for hid, spec in hooks_cfg.items():
+                if not _wanted(spec):
+                    continue
+                try:
+                    bundle_parts.append(_hook_js(hid, spec['type'], spec['spec']))
+                except Exception as e:
+                    debug(f'hook 重放失败 id={hid}', e)
+        except Exception as e:
+            debug('hook 重放失败', e)
+
+    if not bundle_parts:
+        return
+    try:
+        bundle_js = ';'.join(
+            f'try{{{s}}}catch(e){{}}' for s in bundle_parts)
+        bundle_id = target.add_init_js(bundle_js)
+        for spec in (rpc_cfg or {}).values():
+            if _wanted(spec):
+                spec['js_id'] = bundle_id
+        for spec in (hooks_cfg or {}).values():
+            if _wanted(spec):
+                spec['js_id'] = bundle_id
+        try:
+            save_session(session, sess)
+        except Exception as e:
+            debug('bundle js_id 回写失败', e)
+    except Exception as e:
+        debug('persist 脚本打包注册失败', e)
+
+
+def _remove_init_js(page, js_id):
+    """移除 init-js，绕开 DrissionPage remove_init_js 的 _init_jss 门控。
+
+    DP 的 remove_init_js 只在 script_id in self._init_jss（本进程
+    实例属性，跨进程为空）时执行 CDP 调用——跨进程 unregister 静默
+    失效。直接调 Page.removeScriptToEvaluateOnNewDocument 绕过；
+    identifier 失效（session 已死/从未注册）吞异常返回 False。
+    """
+    if not js_id:
+        return False
+    try:
+        page.run_cdp('Page.removeScriptToEvaluateOnNewDocument',
+                     identifier=js_id)
+        # 同步清理 DP 内部注册表（若本进程注册过）
+        jss = getattr(page, '_init_jss', None)
+        if isinstance(jss, list) and js_id in jss:
+            jss.remove(js_id)
+        return True
+    except Exception:
+        return False
+
+
 _KNOWN_PREFIX = re.compile(
     r'^(css[:=]|xpath[:=]|text[:=^$]|tag[:=^$]|@@?[\w]|ref:|pw:)', re.IGNORECASE)
 _CSS_ID_CLASS = re.compile(r'^[#.][\w-]')           # #id  .class
@@ -296,6 +421,103 @@ def resolve_locator(locator: str, session: str = 'default', page=None) -> str:
     raise SystemExit(1)
 
 
+class CdpWaitGate:
+    """CDP 事件驱动的等待唤醒门：事件到达即唤醒重检，无事件按 poll 节拍回落。
+
+    事件源：
+    - Runtime.bindingCalled：页内 watcher（MutationObserver + 低频 interval）
+      检测到条件变真后经 __dp_wait binding 上报——JS 侧探测零 CDP 轮询流量
+    - Page.frameNavigated/domContentEventFired/loadEventFired：导航唤醒
+      重检，并在新文档重注入 watcher（导航后 JS 上下文销毁）
+
+    arm() 任一步失败（旧内核不支持 addBinding / driver 不可用）或调用方
+    强制轮询 → armed=False，调用方走纯 _sleep 路径，语义完全等价。
+    """
+    BIND_NAME = '__dp_wait'
+    _WATCHER = (
+        '(function(){if(window.__dp_wait_armed)return"armed";'
+        'window.__dp_wait_armed=1;var done=false;'
+        'function chk(){if(done)return;var v=false;'
+        'try{v=!!(__COND__)}catch(e){}'
+        'if(v){done=true;try{window.__dp_wait("hit")}catch(e){}}}'
+        'try{new MutationObserver(chk).observe('
+        'document.documentElement||document,'
+        '{subtree:true,childList:true,attributes:true,characterData:true})}'
+        'catch(e){}'
+        'window.__dp_wait_iv=setInterval(chk,250);chk();return "ok"})()')
+    _PAGE_EVENTS = ('Page.frameNavigated', 'Page.domContentEventFired',
+                    'Page.loadEventFired')
+    _EVENTS = ('Runtime.bindingCalled',) + _PAGE_EVENTS
+
+    def __init__(self, page):
+        self.page = page
+        self.wake = threading.Event()
+        self.nav = threading.Event()
+        self.armed = False
+        self._driver = None
+        self._cond = None
+
+    def arm(self, cond_js=None, force_poll=False) -> bool:
+        """注册 binding + Page 事件 + 注入页内 watcher。返回是否成功武装。"""
+        if force_poll:
+            return False
+        self._cond = cond_js
+        try:
+            self._driver = self.page.driver
+            self.page.run_cdp('Runtime.enable')
+            self.page.run_cdp('Runtime.addBinding', name=self.BIND_NAME)
+            self.page.run_cdp('Page.enable')
+            self._driver.set_callback('Runtime.bindingCalled', self._on_binding)
+            for ev in self._PAGE_EVENTS:
+                self._driver.set_callback(ev, self._on_nav)
+            self.armed = True
+            if cond_js:
+                self.reinject()
+        except Exception as e:
+            debug('事件驱动等待不可用，回落轮询', e)
+            self.disarm()
+        return self.armed
+
+    def reinject(self):
+        """（导航后的新文档）重新注入页内条件 watcher。"""
+        if not (self.armed and self._cond):
+            return
+        try:
+            self.page.run_js(
+                self._WATCHER.replace('__COND__', f'({self._cond})'))
+        except Exception as e:
+            # 导航瞬断期注入失败无碍：后续 Page 事件/轮询节拍会再触发重试
+            debug('watcher 注入失败', e)
+
+    def wait(self, seconds: float) -> bool:
+        """阻塞至事件唤醒或超时；返回 True=被事件唤醒。自动处理导航重注入。"""
+        fired = self.wake.wait(max(0.0, seconds))
+        self.wake.clear()
+        if self.nav.is_set():
+            self.nav.clear()
+            self.reinject()
+        return fired
+
+    def disarm(self):
+        """摘除全部回调（binding 留在会话内无副作用）。"""
+        if self._driver is not None:
+            for ev in self._EVENTS:
+                try:
+                    self._driver.set_callback(ev, None)
+                except Exception:
+                    pass
+            self._driver = None
+        self.armed = False
+
+    def _on_binding(self, **kw):
+        if kw.get('name') == self.BIND_NAME:
+            self.wake.set()
+
+    def _on_nav(self, **kw):
+        self.nav.set()
+        self.wake.set()
+
+
 def wait_network_idle(page, idle_time: float = 2.0, timeout: float = 30) -> bool:
     """等待网络空闲：通过 CDP 监听，直到无活跃请求持续 idle_time 秒。