dp-cli 0.1.0__py3-none-any.whl

dp_cli/snapshot/extract.py

@@ -0,0 +1,158 @@
+# -*- coding:utf-8 -*-
+"""
+extract_structured / query_elements — 数据提取函数
+"""
+from .utils import suggest_locator
+
+_JS_CSS_PATH = """
+var el = this;
+var parts = [];
+while (el && el !== document.body && el.nodeType === 1) {
+    var seg = el.tagName.toLowerCase();
+    if (el.id && /^[a-zA-Z][\\w-]*$/.test(el.id)) {
+        parts.unshift('#' + el.id);
+        break;
+    }
+    var classes = Array.from(el.classList)
+        .filter(function(c) { return c.length >= 3; });
+    if (classes.length > 0) {
+        seg = '.' + classes[0];
+        var siblings = el.parentElement
+            ? Array.from(el.parentElement.querySelectorAll(':scope > ' + seg))
+            : [];
+        if (siblings.length > 1) {
+            var idx = Array.from(el.parentElement.children).indexOf(el) + 1;
+            seg = seg + ':nth-child(' + idx + ')';
+        }
+    } else {
+        var allSiblings = el.parentElement
+            ? Array.from(el.parentElement.children).filter(function(c) { return c.tagName === el.tagName; })
+            : [];
+        if (allSiblings.length > 1) {
+            var idx2 = Array.from(el.parentElement.children).indexOf(el) + 1;
+            seg = seg + ':nth-child(' + idx2 + ')';
+        }
+    }
+    parts.unshift(seg);
+    el = el.parentElement;
+}
+return parts.join(' > ');
+"""
+
+_JS_XPATH = """
+var el = this;
+var parts = [];
+while (el && el.nodeType === 1) {
+    var seg = el.tagName.toLowerCase();
+    if (el.id && /^[a-zA-Z][\\w-]*$/.test(el.id)) {
+        parts.unshift('//' + seg + '[@id="' + el.id + '"]');
+        return parts.join('/');
+    }
+    var siblings = el.parentElement
+        ? Array.from(el.parentElement.children).filter(function(c) { return c.tagName === el.tagName; })
+        : [];
+    if (siblings.length > 1) {
+        var idx = siblings.indexOf(el) + 1;
+        seg = seg + '[' + idx + ']';
+    }
+    parts.unshift(seg);
+    el = el.parentElement;
+}
+return '/' + parts.join('/');
+"""
+
+
+def extract_structured(page, container: str, fields: dict,
+                       limit: int = 100) -> list:
+    """
+    结构化批量提取。
+
+    :param container: 容器定位器，如 'css:.job-card'
+    :param fields: 字段映射字典
+    :param limit: 最多提取多少条
+    """
+    try:
+        containers = page.eles(container, timeout=5)
+    except Exception:
+        containers = page.s_eles(container)
+    if not containers:
+        return []
+
+    results = []
+    for item in list(containers)[:limit]:
+        record = {}
+        for field_name, spec in fields.items():
+            if isinstance(spec, str):
+                spec = {'selector': spec}
+            sel = spec.get('selector', '')
+            multi = spec.get('multi', False)
+            attr = spec.get('attr', None)
+            default = spec.get('default', '')
+            try:
+                if multi:
+                    eles = item.eles(sel)
+                    record[field_name] = [
+                        (e.attr(attr) if attr else (e.raw_text or '').strip())
+                        for e in eles
+                    ]
+                else:
+                    ele = item.ele(sel)
+                    if ele and ele.__class__.__name__ != 'NoneElement':
+                        if attr:
+                            record[field_name] = ele.attr(attr) or default
+                        else:
+                            record[field_name] = (ele.raw_text or '').strip() or default
+                    else:
+                        record[field_name] = default
+            except Exception:
+                record[field_name] = default
+        results.append(record)
+    return results
+
+
+def query_elements(page, selector: str, fields: list,
+                   limit: int = 200) -> list:
+    """
+    query 模式：找到所有匹配 selector 的元素，批量提取指定属性/文本。
+    """
+    try:
+        eles = page.eles(selector, timeout=5)
+    except Exception:
+        eles = page.s_eles(selector)
+
+    results = []
+    for ele in list(eles)[:limit]:
+        record = {}
+        for f in fields:
+            try:
+                if f == 'text':
+                    record['text'] = (ele.raw_text or '').strip()
+                elif f == 'tag':
+                    record['tag'] = ele.tag
+                elif f == 'loc':
+                    record['loc'] = suggest_locator(
+                        ele.tag, ele.attrs, (ele.raw_text or '').strip()[:50]
+                    )
+                elif f in ('css', 'css_path'):
+                    try:
+                        path = ele.run_js(_JS_CSS_PATH)
+                        record['css'] = f'css:{path}' if path else ''
+                    except Exception:
+                        record['css'] = ''
+                elif f == 'xpath':
+                    try:
+                        path = ele.run_js(_JS_XPATH)
+                        record['xpath'] = f'xpath:{path}' if path else ''
+                    except Exception:
+                        record['xpath'] = ''
+                elif f == 'html':
+                    record['html'] = ele.inner_html or ''
+                elif f == 'outer_html':
+                    record['outer_html'] = ele.html or ''
+                else:
+                    val = ele.attrs.get(f, '') if hasattr(ele, 'attrs') else ''
+                    record[f] = val or ''
+            except Exception:
+                record[f] = ''
+        results.append(record)
+    return results

dp_cli/snapshot/js_scripts.py

@@ -0,0 +1,155 @@
+# -*- coding:utf-8 -*-
+"""
+浏览器注入 JS 脚本。
+
+_JS_A11Y_FALLBACK — 当 CDP Accessibility API 不可用时，通过 DOM 遍历模拟 a11y tree。
+"""
+
+# ── A11y Tree JS 降级脚本 ─────────────────────────────────────────────────────
+# 当 CDP Accessibility API 不可用时，通过 DOM 遍历模拟 a11y tree
+_JS_A11Y_FALLBACK = """
+(() => {
+    const ROLE_MAP = {
+        a: 'link', button: 'button', input: 'textbox', textarea: 'textbox',
+        select: 'combobox', option: 'option', img: 'img', nav: 'navigation',
+        main: 'main', header: 'banner', footer: 'contentinfo', aside: 'complementary',
+        form: 'form', table: 'table', tr: 'row', td: 'cell', th: 'columnheader',
+        ul: 'list', ol: 'list', li: 'listitem', h1: 'heading', h2: 'heading',
+        h3: 'heading', h4: 'heading', h5: 'heading', h6: 'heading',
+        article: 'article', section: 'region', dialog: 'dialog',
+        details: 'group', summary: 'button', progress: 'progressbar',
+        meter: 'meter', output: 'status',
+    };
+    const INPUT_ROLE = { checkbox: 'checkbox', radio: 'radio', range: 'slider',
+        search: 'searchbox', number: 'spinbutton', submit: 'button', reset: 'button',
+        button: 'button', file: 'button' };
+
+    let nodeCounter = 0;
+    let stats = { total: 0, ignored: 0, interactive: 0 };
+    const INTERACTIVE = new Set([
+        'button','link','textbox','combobox','checkbox','radio',
+        'slider','spinbutton','tab','menuitem','searchbox','switch',
+        'option','menuitemcheckbox','menuitemradio','treeitem',
+    ]);
+
+    function getRole(el) {
+        const explicit = el.getAttribute('role');
+        if (explicit) return explicit;
+        const tag = el.tagName.toLowerCase();
+        if (tag === 'input') return INPUT_ROLE[el.type] || 'textbox';
+        return ROLE_MAP[tag] || '';
+    }
+
+    function getName(el) {
+        const label = el.getAttribute('aria-label');
+        if (label) return label;
+        const labelledBy = el.getAttribute('aria-labelledby');
+        if (labelledBy) {
+            const ref = document.getElementById(labelledBy);
+            if (ref) return (ref.textContent || '').trim().slice(0, 100);
+        }
+        const alt = el.getAttribute('alt');
+        if (alt) return alt;
+        const title = el.getAttribute('title');
+        if (title) return title;
+        const placeholder = el.getAttribute('placeholder');
+        if (placeholder) return placeholder;
+        // 直接子文本
+        let text = '';
+        for (const child of el.childNodes) {
+            if (child.nodeType === 3) text += child.textContent;
+        }
+        text = text.trim().slice(0, 100);
+        return text;
+    }
+
+    function isHidden(el) {
+        if (el.hidden || el.getAttribute('aria-hidden') === 'true') return true;
+        const st = getComputedStyle(el);
+        return st.display === 'none' || st.visibility === 'hidden';
+    }
+
+    function getProps(el) {
+        const props = {};
+        const tag = el.tagName.toLowerCase();
+        if (el.hasAttribute('aria-expanded'))
+            props.expanded = el.getAttribute('aria-expanded') === 'true';
+        if (el.hasAttribute('aria-checked'))
+            props.checked = el.getAttribute('aria-checked') === 'true';
+        if (el.hasAttribute('aria-selected'))
+            props.selected = el.getAttribute('aria-selected') === 'true';
+        if (el.hasAttribute('aria-disabled') || el.disabled)
+            props.disabled = true;
+        if (el.hasAttribute('aria-required') || el.required)
+            props.required = true;
+        if (el.hasAttribute('aria-pressed'))
+            props.pressed = el.getAttribute('aria-pressed') === 'true';
+        if (/^h[1-6]$/.test(tag))
+            props.level = parseInt(tag[1]);
+        return props;
+    }
+
+    function buildNode(el, depth) {
+        if (depth > 20) return null;
+        if (el.nodeType !== 1) return null;
+        if (isHidden(el)) { stats.ignored++; return null; }
+
+        const role = getRole(el);
+        const name = getName(el);
+        const id = String(++nodeCounter);
+        stats.total++;
+        if (INTERACTIVE.has(role)) stats.interactive++;
+
+        const children = [];
+        for (const child of el.children) {
+            const cn = buildNode(child, depth + 1);
+            if (cn) children.push(cn);
+        }
+
+        // 跳过无意义容器（无 role、无 name、只有一个子节点）
+        if (!role && !name && children.length === 1) {
+            return children[0];
+        }
+        // 跳过完全空的无 role 节点
+        if (!role && !name && children.length === 0) {
+            return null;
+        }
+
+        const node = { nodeId: id, role: role || 'generic', name: name };
+        const props = getProps(el);
+        if (Object.keys(props).length) node.properties = props;
+
+        const value = el.value;
+        if (value !== undefined && value !== '' && role &&
+            ['textbox','combobox','slider','spinbutton','searchbox'].includes(role)) {
+            node.value = String(value).slice(0, 200);
+        }
+
+        // 定位器
+        const loc = suggestLoc(el);
+        if (loc) node.locator = loc;
+
+        if (children.length) node.children = children;
+        return node;
+    }
+
+    function suggestLoc(el) {
+        const id = el.id;
+        if (id) return '#' + id;
+        for (const attr of ['data-testid','data-qa','aria-label','name','placeholder']) {
+            const v = el.getAttribute(attr);
+            if (v) return '@' + attr + '=' + v;
+        }
+        const cls = el.className;
+        if (typeof cls === 'string' && cls.trim()) {
+            return '.' + cls.trim().split(/\\s+/)[0];
+        }
+        const text = (el.textContent || '').trim();
+        if (text && text.length <= 30) return 'text:' + text;
+        return 't:' + el.tagName.toLowerCase();
+    }
+
+    const tree = buildNode(document.body, 0) || { nodeId: '0', role: 'WebArea', name: document.title };
+    return { tree: tree, stats: stats };
+})()
+"""

dp_cli/snapshot/utils.py

@@ -0,0 +1,43 @@
+# -*- coding:utf-8 -*-
+"""共享工具函数"""
+import re
+
+
+def _is_meaningful_class(cls: str) -> bool:
+    """判断 CSS 类名是否有语义（过滤混淆/哈希类名）"""
+    if not cls or len(cls) < 2:
+        return False
+    # CSS module 风格：prefix-hash，后缀含数字（如 btn-abc1234、css-1d2e3f）
+    if re.match(r'^[a-z]+-(?=\w*\d)\w{4,}$', cls):
+        return False
+    # 纯随机字符串：6+ 字符且无分隔符（-_），大小写混杂或全小写无元音
+    if len(cls) >= 6 and not re.search(r'[-_]', cls):
+        # 大小写混杂无分隔符（如 hkJMPzDNh、BAyykwGBSi）
+        if re.search(r'[a-z]', cls) and re.search(r'[A-Z]', cls):
+            return False
+        # 全小写但无元音（如 bcdfgh）→ 大概率是哈希
+        if cls.islower() and not re.search(r'[aeiou]', cls):
+            return False
+    return True
+
+
+def suggest_locator(tag: str, attrs: dict, text: str) -> str:
+    """为静态元素生成最优 DrissionPage 定位字符串"""
+    if attrs.get('id'):
+        return f'#{attrs["id"]}'
+
+    for semantic in ('data-testid', 'data-qa', 'data-cy', 'aria-label', 'name', 'placeholder'):
+        if attrs.get(semantic):
+            val = attrs[semantic]
+            return f'@{semantic}={val}'
+
+    cls = attrs.get('class', '')
+    if cls:
+        classes = [c for c in cls.strip().split() if _is_meaningful_class(c)]
+        if classes:
+            return f'.{classes[0]}'
+
+    if text and len(text) <= 30:
+        return f'text:{text}'
+
+    return f't:{tag}'

dp_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: dp-cli
+Version: 0.1.0
+Summary: A powerful CLI for DrissionPage — browser automation, structured data extraction, network listening and more.
+License: BSD-3-Clause
+Project-URL: Homepage, https://github.com/mofanx/dp-cli
+Project-URL: Repository, https://github.com/mofanx/dp-cli
+Keywords: drissionpage,browser,automation,cli,web-scraping
+Classifier: Programming Language :: Python :: 3
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Topic :: Utilities
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: DrissionPage>=4.0
+Requires-Dist: click>=8.0
+
+# dp-cli
+
+A powerful CLI for [DrissionPage](https://github.com/g1879/DrissionPage) — browser automation, structured data extraction, network listening and more.
+
+## Features
+
+- **Anti-detection by default** — not based on webdriver, `navigator.webdriver` is `false`
+- **Reuse your own browser** — connect to a running Chrome via `--port`, keeping login state and cookies
+- **Powerful locator syntax** — descriptive strings stable across navigation (no ephemeral refs)
+- **Structured data extraction** — `extract` + `query` + `snapshot --mode content` for scraping list pages
+- **Network listening** — capture XHR/Fetch requests and response bodies
+- **Dual mode** — browser control + pure HTTP requests
+- **Shadow-root / iframe** — traverse directly without switching context
+- **JSON output** — all commands output JSON, AI-friendly
+
+## Installation
+
+```bash
+pip install dp-cli
+dp --help
+```
+
+## Quick Start
+
+```bash
+# Auto-managed browser
+dp open https://example.com
+dp snapshot
+dp click "text:Login"
+dp fill "@name=username" admin
+dp press Enter
+dp close
+
+# Connect to your own logged-in browser
+google-chrome --remote-debugging-port=9222
+dp open https://example.com --port 9222
+dp snapshot
+```
+
+## Data Extraction (3-step workflow)
+
+```bash
+# 1. Discover CSS class names via noise-filtered content tree
+dp snapshot --mode content --max-text 40
+
+# 2. Verify field selectors
+dp query "css:.item-title" --fields "text,loc"
+
+# 3. Batch extract to CSV
+dp extract "css:.item-card" \
+  '{"title":"css:.item-title",
+    "price":"css:.item-price",
+    "tags":{"selector":"css:.tag","multi":true},
+    "url":{"selector":"css:a","attr":"href"}}' \
+  --limit 100 --output csv --filename result.csv
+```
+
+## Project Structure
+
+```
+dp_cli/
+├── main.py              # CLI entry point (~47 lines)
+├── session.py           # Browser session management
+├── snapshot.py          # Page snapshot & data extraction engine
+├── output.py            # JSON output helpers
+└── commands/
+    ├── _utils.py        # Shared decorators & helpers
+    ├── browser.py       # open / goto / reload / close / list
+    ├── snapshot_cmd.py  # snapshot / extract / query / find / inspect
+    ├── element.py       # click / fill / select / hover / drag / check / upload
+    ├── keyboard.py      # press / type / scroll / scroll-to
+    ├── page.py          # screenshot / pdf / eval / wait / dialog
+    ├── tab.py           # tab-list / tab-new / tab-select / tab-close
+    ├── storage.py       # cookie-* / localstorage-* / sessionstorage-*
+    ├── network.py       # listen / listen-stop / http-get / http-post
+    └── misc.py          # resize / maximize / state-save / state-load / config-set
+```
+
+## Documentation
+
+See [`skills/SKILL.md`](skills/SKILL.md) for full workflow guide and [`skills/references/commands.md`](skills/references/commands.md) for complete command reference.
+
+## License
+
+BSD-3-Clause

dp_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+dp_cli/__init__.py,sha256=-yYBDkgMDSHYf_F_uX23wZsDSjrudhppJvm34su8_I4,23
+dp_cli/main.py,sha256=kzQlpsBt42vi43KlDJbnrmCEaX_K0PFoifrK95YoqVc,1110
+dp_cli/output.py,sha256=MDs3U64iiDfP7ONWEY3HDK02LHmWhHE6ojuyfuTv4R0,2356
+dp_cli/session.py,sha256=2qHuaSiYn3-6BAn4FifyUDrjdBwMT8CzUAm9aF5VEkI,6023
+dp_cli/commands/__init__.py,sha256=fU5LGWUmhYYrZXYBELfUnmUgzhSEuzsIWqQl7Jn1Afk,306
+dp_cli/commands/_utils.py,sha256=KAkUOj_teoJmfH6LKmXWZNGLE9LeJnrguDHy6xqou7I,3466
+dp_cli/commands/browser.py,sha256=FOY0nT0pugztNkRNNj4_PfcX0IVqkaixCrb3ZEtmV4c,6193
+dp_cli/commands/element.py,sha256=AkkQIlmMfmakXAwQ1i57Td0T6kntGMhVgcJvxYBb7a8,11159
+dp_cli/commands/keyboard.py,sha256=Ojr8-twK3tYAumO__H_Y0nBeBUqxVJ3zV5UhpsJoT1w,4328
+dp_cli/commands/misc.py,sha256=QdnO9mWtve6K59nBGXOQEPkB3SryQ9j2Lh_vysfBtP8,4994
+dp_cli/commands/network.py,sha256=aWJqj0iZXPhvVHeKeDXHaC75bvBgBIH3637I_iFPc3w,6735
+dp_cli/commands/page.py,sha256=lswoaO-r3tBx25YRQ22iDgvAeImbj0Fp6-8xEEoRmn8,8268
+dp_cli/commands/snapshot_cmd.py,sha256=YJT8DzuUJJCFXdSCRlX1IzXIu8D0Ooc7LuqrZpkPi10,16888
+dp_cli/commands/storage.py,sha256=J0HN1YBDGm6JwwoPTs5-RN0Sr6oidN1vLA08VBh0DJw,8556
+dp_cli/commands/tab.py,sha256=WOuF2YP_1eQSjcnAmA_aVOza3Bd5NgbkbI2ZTRPVjvs,7277
+dp_cli/snapshot/__init__.py,sha256=08i54F9n7p_1jZVhOPQLzXn3IFsvFXMiqgYJgLtamqg,786
+dp_cli/snapshot/a11y.py,sha256=tvvOkQz3AHfTZ6cj-vf5UqOLsPFF2UDPnFuEAegBreY,23991
+dp_cli/snapshot/extract.py,sha256=rHzdGCEp292El3DTe1EhT6xx4a91NxMv1seJIB5Ho2E,5413
+dp_cli/snapshot/js_scripts.py,sha256=Zybph9IbVlUCARfLQxGeD6BpjLjSKG6zqjIMEoyt_9k,6104
+dp_cli/snapshot/utils.py,sha256=_nYaqhvZjvbWzZQW1ZtM3tzmMoEHqmh9YWZax3WRzvU,1537
+dp_cli-0.1.0.dist-info/METADATA,sha256=9JWR6XS2p2l6-iZG3WecSADGkOQ_uKgTml6NFCINX6w,3685
+dp_cli-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+dp_cli-0.1.0.dist-info/entry_points.txt,sha256=EgN4pBb_UDeB8bfysCeZuhdCH6EiRz91cJMH9Q1PNBo,40
+dp_cli-0.1.0.dist-info/top_level.txt,sha256=6o7k3gaYATX5gnc7N5WbROP6kfynSgVUX2gjx87_E_o,7
+dp_cli-0.1.0.dist-info/RECORD,,

dp_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dp_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dp = dp_cli.main:main

dp_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dp_cli