cmdop 0.1.17__py3-none-any.whl → 0.1.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmdop/__init__.py +1 -1
- cmdop/services/browser/aio/session.py +115 -1
- cmdop/services/browser/base/session.py +47 -0
- cmdop/services/browser/js/__init__.py +2 -0
- cmdop/services/browser/js/interaction.py +28 -0
- cmdop/services/browser/js/scroll.py +114 -23
- cmdop/services/browser/sync/session.py +58 -23
- {cmdop-0.1.17.dist-info → cmdop-0.1.18.dist-info}/METADATA +30 -23
- {cmdop-0.1.17.dist-info → cmdop-0.1.18.dist-info}/RECORD +11 -11
- {cmdop-0.1.17.dist-info → cmdop-0.1.18.dist-info}/WHEEL +0 -0
- {cmdop-0.1.17.dist-info → cmdop-0.1.18.dist-info}/licenses/LICENSE +0 -0
cmdop/__init__.py
CHANGED
|
@@ -4,9 +4,17 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any
|
|
6
6
|
|
|
7
|
+
import asyncio
|
|
8
|
+
|
|
7
9
|
from cmdop.services.browser.base.session import BaseSession
|
|
8
|
-
from cmdop.services.browser.models import
|
|
10
|
+
from cmdop.services.browser.models import (
|
|
11
|
+
BrowserCookie,
|
|
12
|
+
BrowserState,
|
|
13
|
+
ScrollInfo,
|
|
14
|
+
ScrollResult,
|
|
15
|
+
)
|
|
9
16
|
from cmdop.services.browser.js import parse_json_result
|
|
17
|
+
from cmdop.services.browser.parsing import parse_html as _parse_html, SoupWrapper
|
|
10
18
|
|
|
11
19
|
if TYPE_CHECKING:
|
|
12
20
|
from cmdop.services.browser.aio.service import AsyncBrowserService
|
|
@@ -158,6 +166,112 @@ class AsyncBrowserSession(BaseSession):
|
|
|
158
166
|
self._session_id, item, fields_json, limit
|
|
159
167
|
)
|
|
160
168
|
|
|
169
|
+
# === HTML Parsing ===
|
|
170
|
+
|
|
171
|
+
async def parse_html(self, html: str | None = None, selector: str | None = None) -> "BeautifulSoup":
|
|
172
|
+
"""Parse HTML with BeautifulSoup."""
|
|
173
|
+
if html is None:
|
|
174
|
+
html = await self.get_html(selector)
|
|
175
|
+
return _parse_html(html)
|
|
176
|
+
|
|
177
|
+
async def soup(self, selector: str | None = None) -> SoupWrapper:
|
|
178
|
+
"""Get page HTML as SoupWrapper."""
|
|
179
|
+
html = await self.get_html(selector)
|
|
180
|
+
return SoupWrapper(html=html)
|
|
181
|
+
|
|
182
|
+
# === Scroll & Navigation ===
|
|
183
|
+
|
|
184
|
+
async def scroll(
|
|
185
|
+
self,
|
|
186
|
+
direction: str = "down",
|
|
187
|
+
amount: int = 500,
|
|
188
|
+
selector: str | None = None,
|
|
189
|
+
smooth: bool = True,
|
|
190
|
+
human_like: bool = False,
|
|
191
|
+
container: str | None = None,
|
|
192
|
+
) -> ScrollResult:
|
|
193
|
+
"""Scroll the page or container."""
|
|
194
|
+
js = self._build_scroll(direction, amount, selector, smooth, human_like, container)
|
|
195
|
+
result = await self.execute_script(js)
|
|
196
|
+
data = parse_json_result(result) or {}
|
|
197
|
+
return ScrollResult(
|
|
198
|
+
success=data.get("success", False),
|
|
199
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
200
|
+
scrolled_by=int(data.get("scrolledBy", 0)),
|
|
201
|
+
at_bottom=data.get("atBottom", False),
|
|
202
|
+
error=data.get("error"),
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
async def scroll_to(self, selector: str) -> ScrollResult:
|
|
206
|
+
"""Scroll element into view."""
|
|
207
|
+
return await self.scroll(selector=selector)
|
|
208
|
+
|
|
209
|
+
async def scroll_to_bottom(self) -> ScrollResult:
|
|
210
|
+
"""Scroll to page bottom."""
|
|
211
|
+
js = self._build_scroll_to_bottom()
|
|
212
|
+
result = await self.execute_script(js)
|
|
213
|
+
data = parse_json_result(result) or {}
|
|
214
|
+
return ScrollResult(
|
|
215
|
+
success=data.get("success", False),
|
|
216
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
217
|
+
scrolled_by=int(data.get("scrolledBy", 0)),
|
|
218
|
+
at_bottom=True,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
async def get_scroll_info(self) -> ScrollInfo:
|
|
222
|
+
"""Get current scroll position and page dimensions."""
|
|
223
|
+
js = self._build_get_scroll_info()
|
|
224
|
+
result = await self.execute_script(js)
|
|
225
|
+
data = parse_json_result(result) or {}
|
|
226
|
+
return ScrollInfo(
|
|
227
|
+
scroll_x=int(data.get("scrollX", 0)),
|
|
228
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
229
|
+
page_height=int(data.get("pageHeight", 0)),
|
|
230
|
+
page_width=int(data.get("pageWidth", 0)),
|
|
231
|
+
viewport_height=int(data.get("viewportHeight", 0)),
|
|
232
|
+
viewport_width=int(data.get("viewportWidth", 0)),
|
|
233
|
+
at_bottom=data.get("atBottom", False),
|
|
234
|
+
at_top=data.get("atTop", True),
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
# === UI Interaction Helpers ===
|
|
238
|
+
|
|
239
|
+
async def hover(self, selector: str) -> bool:
|
|
240
|
+
"""Hover over element."""
|
|
241
|
+
js = self._build_hover(selector)
|
|
242
|
+
result = await self.execute_script(js)
|
|
243
|
+
data = parse_json_result(result) or {}
|
|
244
|
+
return data.get("success", False)
|
|
245
|
+
|
|
246
|
+
async def select(self, selector: str, value: str | None = None, text: str | None = None) -> dict:
|
|
247
|
+
"""Select option from dropdown."""
|
|
248
|
+
js = self._build_select(selector, value, text)
|
|
249
|
+
result = await self.execute_script(js)
|
|
250
|
+
return parse_json_result(result) or {}
|
|
251
|
+
|
|
252
|
+
async def close_modal(self, selectors: list[str] | None = None) -> bool:
|
|
253
|
+
"""Try to close modal/dialog."""
|
|
254
|
+
js = self._build_close_modal(selectors)
|
|
255
|
+
result = await self.execute_script(js)
|
|
256
|
+
data = parse_json_result(result) or {}
|
|
257
|
+
return data.get("success", False)
|
|
258
|
+
|
|
259
|
+
async def wait_seconds(self, seconds: float) -> None:
|
|
260
|
+
"""Wait for specified seconds."""
|
|
261
|
+
await asyncio.sleep(seconds)
|
|
262
|
+
|
|
263
|
+
async def wait_random(self, min_sec: float = 0.5, max_sec: float = 1.5) -> None:
|
|
264
|
+
"""Wait for random time between min and max seconds."""
|
|
265
|
+
import random
|
|
266
|
+
await asyncio.sleep(min_sec + random.random() * (max_sec - min_sec))
|
|
267
|
+
|
|
268
|
+
async def click_all_by_text(self, text: str, role: str = "button") -> int:
|
|
269
|
+
"""Click all elements containing specific text."""
|
|
270
|
+
js = self._build_click_all_by_text(text, role)
|
|
271
|
+
result = await self.execute_script(js)
|
|
272
|
+
data = parse_json_result(result) or {}
|
|
273
|
+
return data.get("clicked", 0)
|
|
274
|
+
|
|
161
275
|
# === Context Manager ===
|
|
162
276
|
|
|
163
277
|
async def close(self) -> None:
|
|
@@ -10,6 +10,13 @@ from cmdop.services.browser.js import (
|
|
|
10
10
|
build_async_js,
|
|
11
11
|
build_fetch_js,
|
|
12
12
|
build_fetch_all_js,
|
|
13
|
+
build_scroll_js,
|
|
14
|
+
build_scroll_to_bottom_js,
|
|
15
|
+
build_get_scroll_info_js,
|
|
16
|
+
build_hover_js,
|
|
17
|
+
build_select_js,
|
|
18
|
+
build_close_modal_js,
|
|
19
|
+
build_click_all_by_text_js,
|
|
13
20
|
parse_json_result,
|
|
14
21
|
)
|
|
15
22
|
|
|
@@ -70,3 +77,43 @@ class BaseSession(ABC):
|
|
|
70
77
|
def _parse_fetch_all(self, result: Any) -> dict[str, Any]:
|
|
71
78
|
"""Ensure fetch_all returns dict."""
|
|
72
79
|
return result if isinstance(result, dict) else {}
|
|
80
|
+
|
|
81
|
+
# === Scroll helpers ===
|
|
82
|
+
|
|
83
|
+
def _build_scroll(
|
|
84
|
+
self,
|
|
85
|
+
direction: str,
|
|
86
|
+
amount: int,
|
|
87
|
+
selector: str | None,
|
|
88
|
+
smooth: bool,
|
|
89
|
+
human_like: bool,
|
|
90
|
+
container: str | None = None,
|
|
91
|
+
) -> str:
|
|
92
|
+
"""Build scroll JS."""
|
|
93
|
+
return build_scroll_js(direction, amount, selector, smooth, human_like, container)
|
|
94
|
+
|
|
95
|
+
def _build_scroll_to_bottom(self) -> str:
|
|
96
|
+
"""Build scroll to bottom JS."""
|
|
97
|
+
return build_scroll_to_bottom_js()
|
|
98
|
+
|
|
99
|
+
def _build_get_scroll_info(self) -> str:
|
|
100
|
+
"""Build get scroll info JS."""
|
|
101
|
+
return build_get_scroll_info_js()
|
|
102
|
+
|
|
103
|
+
# === Interaction helpers ===
|
|
104
|
+
|
|
105
|
+
def _build_hover(self, selector: str) -> str:
|
|
106
|
+
"""Build hover JS."""
|
|
107
|
+
return build_hover_js(selector)
|
|
108
|
+
|
|
109
|
+
def _build_select(self, selector: str, value: str | None, text: str | None) -> str:
|
|
110
|
+
"""Build select JS."""
|
|
111
|
+
return build_select_js(selector, value, text)
|
|
112
|
+
|
|
113
|
+
def _build_close_modal(self, selectors: list[str] | None) -> str:
|
|
114
|
+
"""Build close modal JS."""
|
|
115
|
+
return build_close_modal_js(selectors)
|
|
116
|
+
|
|
117
|
+
def _build_click_all_by_text(self, text: str, role: str) -> str:
|
|
118
|
+
"""Build click all by text JS."""
|
|
119
|
+
return build_click_all_by_text_js(text, role)
|
|
@@ -28,6 +28,7 @@ from cmdop.services.browser.js.interaction import (
|
|
|
28
28
|
build_hover_js,
|
|
29
29
|
build_select_js,
|
|
30
30
|
build_close_modal_js,
|
|
31
|
+
build_click_all_by_text_js,
|
|
31
32
|
)
|
|
32
33
|
|
|
33
34
|
__all__ = [
|
|
@@ -46,4 +47,5 @@ __all__ = [
|
|
|
46
47
|
"build_hover_js",
|
|
47
48
|
"build_select_js",
|
|
48
49
|
"build_close_modal_js",
|
|
50
|
+
"build_click_all_by_text_js",
|
|
49
51
|
]
|
|
@@ -107,3 +107,31 @@ def build_close_modal_js(
|
|
|
107
107
|
return JSON.stringify({{ success: false, error: 'No modal close button found' }});
|
|
108
108
|
}})()
|
|
109
109
|
"""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_click_all_by_text_js(text: str, role: str = "button") -> str:
|
|
113
|
+
"""
|
|
114
|
+
Build JS to click all elements containing specific text.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
text: Text to match (case-insensitive)
|
|
118
|
+
role: Element role to filter (default: "button")
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
JS code that returns { clicked: number }
|
|
122
|
+
"""
|
|
123
|
+
return f"""
|
|
124
|
+
(function() {{
|
|
125
|
+
const elements = document.querySelectorAll('[role="{role}"]');
|
|
126
|
+
let clicked = 0;
|
|
127
|
+
const targetText = "{text}".toLowerCase();
|
|
128
|
+
elements.forEach(el => {{
|
|
129
|
+
const elText = el.textContent.trim().toLowerCase();
|
|
130
|
+
if (elText === targetText || elText.includes(targetText)) {{
|
|
131
|
+
el.click();
|
|
132
|
+
clicked++;
|
|
133
|
+
}}
|
|
134
|
+
}});
|
|
135
|
+
return JSON.stringify({{ clicked: clicked }});
|
|
136
|
+
}})()
|
|
137
|
+
"""
|
|
@@ -9,6 +9,9 @@ def build_scroll_js(
|
|
|
9
9
|
direction: str = "down",
|
|
10
10
|
amount: int = 500,
|
|
11
11
|
selector: str | None = None,
|
|
12
|
+
smooth: bool = True,
|
|
13
|
+
human_like: bool = False,
|
|
14
|
+
container: str | None = None,
|
|
12
15
|
) -> str:
|
|
13
16
|
"""
|
|
14
17
|
Build JS for scrolling.
|
|
@@ -16,40 +19,128 @@ def build_scroll_js(
|
|
|
16
19
|
Args:
|
|
17
20
|
direction: "up", "down", "left", "right"
|
|
18
21
|
amount: Pixels to scroll (ignored if selector provided)
|
|
19
|
-
selector: CSS selector to scroll into view
|
|
22
|
+
selector: CSS selector to scroll element into view
|
|
23
|
+
smooth: Use smooth scroll animation (default True)
|
|
24
|
+
human_like: Add random variations for natural scrolling
|
|
25
|
+
container: CSS selector for scroll container (default: window)
|
|
20
26
|
"""
|
|
21
27
|
if selector:
|
|
28
|
+
behavior = "smooth" if smooth else "instant"
|
|
22
29
|
return f"""
|
|
23
|
-
(function() {{
|
|
30
|
+
(async function() {{
|
|
24
31
|
const el = document.querySelector("{selector}");
|
|
25
32
|
if (el) {{
|
|
26
|
-
el.scrollIntoView({{ behavior: '
|
|
27
|
-
|
|
33
|
+
el.scrollIntoView({{ behavior: '{behavior}', block: 'center' }});
|
|
34
|
+
await new Promise(r => setTimeout(r, {300 if smooth else 50}));
|
|
35
|
+
return JSON.stringify({{ success: true, scrollY: window.scrollY }});
|
|
28
36
|
}}
|
|
29
37
|
return JSON.stringify({{ success: false, error: 'Element not found' }});
|
|
30
38
|
}})()
|
|
31
39
|
"""
|
|
32
40
|
|
|
33
|
-
|
|
34
|
-
"down": f"window.scrollBy(0, {amount})",
|
|
35
|
-
"up": f"window.scrollBy(0, -{amount})",
|
|
36
|
-
"right": f"window.scrollBy({amount}, 0)",
|
|
37
|
-
"left": f"window.scrollBy(-{amount}, 0)",
|
|
38
|
-
}
|
|
39
|
-
scroll_code = scroll_map.get(direction, scroll_map["down"])
|
|
41
|
+
behavior = "smooth" if smooth else "instant"
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
43
|
+
# Container scroll target
|
|
44
|
+
container_js = f'document.querySelector("{container}")' if container else "null"
|
|
45
|
+
|
|
46
|
+
if human_like:
|
|
47
|
+
# Human-like scroll with random variation and micro-scrolls
|
|
48
|
+
return f"""
|
|
49
|
+
(async function() {{
|
|
50
|
+
const container = {container_js};
|
|
51
|
+
const scrollTarget = container || window;
|
|
52
|
+
const getScrollY = () => container ? container.scrollTop : window.scrollY;
|
|
53
|
+
const getMaxScroll = () => container
|
|
54
|
+
? container.scrollHeight - container.clientHeight
|
|
55
|
+
: document.body.scrollHeight - window.innerHeight;
|
|
56
|
+
|
|
57
|
+
const before = getScrollY();
|
|
58
|
+
const baseAmount = {amount};
|
|
59
|
+
const direction = "{direction}";
|
|
60
|
+
|
|
61
|
+
// Random variation ±15%
|
|
62
|
+
const variation = 0.85 + Math.random() * 0.3;
|
|
63
|
+
const actualAmount = Math.round(baseAmount * variation);
|
|
64
|
+
|
|
65
|
+
// Split into 2-4 micro-scrolls
|
|
66
|
+
const steps = 2 + Math.floor(Math.random() * 3);
|
|
67
|
+
const stepAmount = actualAmount / steps;
|
|
68
|
+
|
|
69
|
+
for (let i = 0; i < steps; i++) {{
|
|
70
|
+
const stepVariation = 0.8 + Math.random() * 0.4;
|
|
71
|
+
const thisStep = Math.round(stepAmount * stepVariation);
|
|
72
|
+
|
|
73
|
+
let x = 0, y = 0;
|
|
74
|
+
if (direction === "down") y = thisStep;
|
|
75
|
+
else if (direction === "up") y = -thisStep;
|
|
76
|
+
else if (direction === "right") x = thisStep;
|
|
77
|
+
else if (direction === "left") x = -thisStep;
|
|
78
|
+
|
|
79
|
+
if (container) {{
|
|
80
|
+
container.scrollBy({{ left: x, top: y, behavior: '{behavior}' }});
|
|
81
|
+
}} else {{
|
|
82
|
+
window.scrollBy({{ left: x, top: y, behavior: '{behavior}' }});
|
|
83
|
+
}}
|
|
84
|
+
|
|
85
|
+
// Random micro-pause 50-150ms between steps
|
|
86
|
+
if (i < steps - 1) {{
|
|
87
|
+
await new Promise(r => setTimeout(r, 50 + Math.random() * 100));
|
|
88
|
+
}}
|
|
89
|
+
}}
|
|
90
|
+
|
|
91
|
+
// Wait for smooth scroll to finish
|
|
92
|
+
await new Promise(r => setTimeout(r, {250 if smooth else 50}));
|
|
93
|
+
|
|
94
|
+
const after = getScrollY();
|
|
95
|
+
return JSON.stringify({{
|
|
96
|
+
success: true,
|
|
97
|
+
scrollY: after,
|
|
98
|
+
scrolledBy: after - before,
|
|
99
|
+
atBottom: after >= getMaxScroll() - 50
|
|
100
|
+
}});
|
|
101
|
+
}})()
|
|
102
|
+
"""
|
|
103
|
+
else:
|
|
104
|
+
# Standard scroll
|
|
105
|
+
scroll_x = 0
|
|
106
|
+
scroll_y = 0
|
|
107
|
+
if direction == "down":
|
|
108
|
+
scroll_y = amount
|
|
109
|
+
elif direction == "up":
|
|
110
|
+
scroll_y = -amount
|
|
111
|
+
elif direction == "right":
|
|
112
|
+
scroll_x = amount
|
|
113
|
+
elif direction == "left":
|
|
114
|
+
scroll_x = -amount
|
|
115
|
+
|
|
116
|
+
return f"""
|
|
117
|
+
(async function() {{
|
|
118
|
+
const container = {container_js};
|
|
119
|
+
const getScrollY = () => container ? container.scrollTop : window.scrollY;
|
|
120
|
+
const getMaxScroll = () => container
|
|
121
|
+
? container.scrollHeight - container.clientHeight
|
|
122
|
+
: document.body.scrollHeight - window.innerHeight;
|
|
123
|
+
|
|
124
|
+
const before = getScrollY();
|
|
125
|
+
|
|
126
|
+
if (container) {{
|
|
127
|
+
container.scrollBy({{ left: {scroll_x}, top: {scroll_y}, behavior: '{behavior}' }});
|
|
128
|
+
}} else {{
|
|
129
|
+
window.scrollBy({{ left: {scroll_x}, top: {scroll_y}, behavior: '{behavior}' }});
|
|
130
|
+
}}
|
|
131
|
+
|
|
132
|
+
// Wait for scroll to complete
|
|
133
|
+
await new Promise(r => setTimeout(r, {200 if smooth else 30}));
|
|
134
|
+
|
|
135
|
+
const after = getScrollY();
|
|
136
|
+
return JSON.stringify({{
|
|
137
|
+
success: true,
|
|
138
|
+
scrollY: after,
|
|
139
|
+
scrolledBy: after - before,
|
|
140
|
+
atBottom: after >= getMaxScroll() - 50
|
|
141
|
+
}});
|
|
142
|
+
}})()
|
|
143
|
+
"""
|
|
53
144
|
|
|
54
145
|
|
|
55
146
|
def build_scroll_to_bottom_js() -> str:
|
|
@@ -15,13 +15,7 @@ from cmdop.services.browser.models import (
|
|
|
15
15
|
)
|
|
16
16
|
from cmdop.services.browser.js import (
|
|
17
17
|
parse_json_result,
|
|
18
|
-
build_scroll_js,
|
|
19
|
-
build_scroll_to_bottom_js,
|
|
20
18
|
build_infinite_scroll_js,
|
|
21
|
-
build_hover_js,
|
|
22
|
-
build_select_js,
|
|
23
|
-
build_close_modal_js,
|
|
24
|
-
build_get_scroll_info_js,
|
|
25
19
|
)
|
|
26
20
|
from cmdop.services.browser.parsing import (
|
|
27
21
|
parse_html as _parse_html,
|
|
@@ -236,25 +230,41 @@ class BrowserSession(BaseSession):
|
|
|
236
230
|
direction: str = "down",
|
|
237
231
|
amount: int = 500,
|
|
238
232
|
selector: str | None = None,
|
|
233
|
+
smooth: bool = True,
|
|
234
|
+
human_like: bool = False,
|
|
235
|
+
container: str | None = None,
|
|
239
236
|
) -> ScrollResult:
|
|
240
237
|
"""
|
|
241
|
-
Scroll the page.
|
|
238
|
+
Scroll the page or container.
|
|
242
239
|
|
|
243
240
|
Args:
|
|
244
241
|
direction: "up", "down", "left", "right"
|
|
245
242
|
amount: Pixels to scroll
|
|
246
243
|
selector: If provided, scroll element into view instead
|
|
244
|
+
smooth: Use smooth scroll animation (default True)
|
|
245
|
+
human_like: Add random variations for natural scrolling
|
|
246
|
+
container: CSS selector for scroll container (default: window)
|
|
247
247
|
|
|
248
248
|
Returns:
|
|
249
249
|
ScrollResult with position info
|
|
250
|
+
|
|
251
|
+
Example:
|
|
252
|
+
# Fast scroll
|
|
253
|
+
browser.scroll("down", 800)
|
|
254
|
+
|
|
255
|
+
# Natural human-like scroll
|
|
256
|
+
browser.scroll("down", 600, human_like=True)
|
|
257
|
+
|
|
258
|
+
# Scroll inside a container (e.g., Facebook feed)
|
|
259
|
+
browser.scroll("down", 800, container="[role='feed']")
|
|
250
260
|
"""
|
|
251
|
-
js =
|
|
261
|
+
js = self._build_scroll(direction, amount, selector, smooth, human_like, container)
|
|
252
262
|
result = self.execute_script(js)
|
|
253
263
|
data = parse_json_result(result) or {}
|
|
254
264
|
return ScrollResult(
|
|
255
265
|
success=data.get("success", False),
|
|
256
|
-
scroll_y=data.get("scrollY", 0),
|
|
257
|
-
scrolled_by=data.get("scrolledBy", 0),
|
|
266
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
267
|
+
scrolled_by=int(data.get("scrolledBy", 0)),
|
|
258
268
|
at_bottom=data.get("atBottom", False),
|
|
259
269
|
error=data.get("error"),
|
|
260
270
|
)
|
|
@@ -265,28 +275,28 @@ class BrowserSession(BaseSession):
|
|
|
265
275
|
|
|
266
276
|
def scroll_to_bottom(self) -> ScrollResult:
|
|
267
277
|
"""Scroll to page bottom."""
|
|
268
|
-
js =
|
|
278
|
+
js = self._build_scroll_to_bottom()
|
|
269
279
|
result = self.execute_script(js)
|
|
270
280
|
data = parse_json_result(result) or {}
|
|
271
281
|
return ScrollResult(
|
|
272
282
|
success=data.get("success", False),
|
|
273
|
-
scroll_y=data.get("scrollY", 0),
|
|
274
|
-
scrolled_by=data.get("scrolledBy", 0),
|
|
283
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
284
|
+
scrolled_by=int(data.get("scrolledBy", 0)),
|
|
275
285
|
at_bottom=True,
|
|
276
286
|
)
|
|
277
287
|
|
|
278
288
|
def get_scroll_info(self) -> ScrollInfo:
|
|
279
289
|
"""Get current scroll position and page dimensions."""
|
|
280
|
-
js =
|
|
290
|
+
js = self._build_get_scroll_info()
|
|
281
291
|
result = self.execute_script(js)
|
|
282
292
|
data = parse_json_result(result) or {}
|
|
283
293
|
return ScrollInfo(
|
|
284
|
-
scroll_x=data.get("scrollX", 0),
|
|
285
|
-
scroll_y=data.get("scrollY", 0),
|
|
286
|
-
page_height=data.get("pageHeight", 0),
|
|
287
|
-
page_width=data.get("pageWidth", 0),
|
|
288
|
-
viewport_height=data.get("viewportHeight", 0),
|
|
289
|
-
viewport_width=data.get("viewportWidth", 0),
|
|
294
|
+
scroll_x=int(data.get("scrollX", 0)),
|
|
295
|
+
scroll_y=int(data.get("scrollY", 0)),
|
|
296
|
+
page_height=int(data.get("pageHeight", 0)),
|
|
297
|
+
page_width=int(data.get("pageWidth", 0)),
|
|
298
|
+
viewport_height=int(data.get("viewportHeight", 0)),
|
|
299
|
+
viewport_width=int(data.get("viewportWidth", 0)),
|
|
290
300
|
at_bottom=data.get("atBottom", False),
|
|
291
301
|
at_top=data.get("atTop", True),
|
|
292
302
|
)
|
|
@@ -374,7 +384,7 @@ class BrowserSession(BaseSession):
|
|
|
374
384
|
|
|
375
385
|
def hover(self, selector: str) -> bool:
|
|
376
386
|
"""Hover over element."""
|
|
377
|
-
js =
|
|
387
|
+
js = self._build_hover(selector)
|
|
378
388
|
result = self.execute_script(js)
|
|
379
389
|
data = parse_json_result(result) or {}
|
|
380
390
|
return data.get("success", False)
|
|
@@ -396,7 +406,7 @@ class BrowserSession(BaseSession):
|
|
|
396
406
|
Returns:
|
|
397
407
|
Dict with selected_value and selected_text
|
|
398
408
|
"""
|
|
399
|
-
js =
|
|
409
|
+
js = self._build_select(selector, value, text)
|
|
400
410
|
result = self.execute_script(js)
|
|
401
411
|
return parse_json_result(result) or {}
|
|
402
412
|
|
|
@@ -410,7 +420,7 @@ class BrowserSession(BaseSession):
|
|
|
410
420
|
Returns:
|
|
411
421
|
True if modal was closed
|
|
412
422
|
"""
|
|
413
|
-
js =
|
|
423
|
+
js = self._build_close_modal(selectors)
|
|
414
424
|
result = self.execute_script(js)
|
|
415
425
|
data = parse_json_result(result) or {}
|
|
416
426
|
return data.get("success", False)
|
|
@@ -419,6 +429,31 @@ class BrowserSession(BaseSession):
|
|
|
419
429
|
"""Wait for specified seconds."""
|
|
420
430
|
time.sleep(seconds)
|
|
421
431
|
|
|
432
|
+
def wait_random(self, min_sec: float = 0.5, max_sec: float = 1.5) -> None:
|
|
433
|
+
"""Wait for random time between min and max seconds."""
|
|
434
|
+
import random
|
|
435
|
+
time.sleep(min_sec + random.random() * (max_sec - min_sec))
|
|
436
|
+
|
|
437
|
+
def click_all_by_text(self, text: str, role: str = "button") -> int:
|
|
438
|
+
"""
|
|
439
|
+
Click all elements containing specific text.
|
|
440
|
+
|
|
441
|
+
Args:
|
|
442
|
+
text: Text to match (case-insensitive)
|
|
443
|
+
role: Element role to filter (default: "button")
|
|
444
|
+
|
|
445
|
+
Returns:
|
|
446
|
+
Number of elements clicked
|
|
447
|
+
|
|
448
|
+
Example:
|
|
449
|
+
browser.click_all_by_text("See more") # Expand all posts
|
|
450
|
+
browser.click_all_by_text("Load more", role="link")
|
|
451
|
+
"""
|
|
452
|
+
js = self._build_click_all_by_text(text, role)
|
|
453
|
+
result = self.execute_script(js)
|
|
454
|
+
data = parse_json_result(result) or {}
|
|
455
|
+
return data.get("clicked", 0)
|
|
456
|
+
|
|
422
457
|
# === Context Manager ===
|
|
423
458
|
|
|
424
459
|
def close(self) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cmdop
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.18
|
|
4
4
|
Summary: Python SDK for CMDOP agent interaction
|
|
5
5
|
Project-URL: Homepage, https://cmdop.com
|
|
6
6
|
Project-URL: Documentation, https://cmdop.com
|
|
@@ -148,56 +148,63 @@ health: Health = result.output # Typed!
|
|
|
148
148
|
```python
|
|
149
149
|
with client.browser.create_session() as b:
|
|
150
150
|
b.navigate("https://shop.com/products")
|
|
151
|
+
b.close_modal() # Close popups
|
|
151
152
|
|
|
152
|
-
#
|
|
153
|
-
|
|
153
|
+
# BeautifulSoup parsing
|
|
154
|
+
soup = b.soup() # SoupWrapper with chainable API
|
|
155
|
+
for item in soup.select(".product"):
|
|
156
|
+
title = item.select_one("h2").text()
|
|
157
|
+
price = item.attr("data-price")
|
|
154
158
|
|
|
155
|
-
#
|
|
156
|
-
html = b.get_html("[role='feed']")
|
|
157
|
-
soup = b.parse_html(html) # Returns BeautifulSoup object
|
|
158
|
-
|
|
159
|
-
# Scroll & extract pattern
|
|
159
|
+
# Human-like scrolling with random delays
|
|
160
160
|
for _ in range(10):
|
|
161
|
-
|
|
162
|
-
# ... parse
|
|
163
|
-
b.scroll("down",
|
|
164
|
-
b.
|
|
161
|
+
soup = b.soup(".listings")
|
|
162
|
+
# ... parse ...
|
|
163
|
+
b.scroll("down", 700, human_like=True) # Natural micro-scrolls
|
|
164
|
+
b.wait_random(0.8, 1.5) # Random delay
|
|
165
|
+
|
|
166
|
+
# Scroll inside container (Facebook, Twitter feeds)
|
|
167
|
+
b.scroll("down", 800, container="[role='feed']")
|
|
168
|
+
|
|
169
|
+
# Click all "See more" buttons
|
|
170
|
+
b.click_all_by_text("See more")
|
|
165
171
|
|
|
166
172
|
# JS fetch (bypass CORS, inherit cookies)
|
|
167
173
|
data = b.fetch_json("https://api.site.com/v1/items")
|
|
168
|
-
|
|
169
|
-
# Parallel fetch
|
|
170
|
-
results = b.fetch_all({
|
|
171
|
-
"users": "https://api.site.com/v1/users",
|
|
172
|
-
"orders": "https://api.site.com/v1/orders",
|
|
173
|
-
}, credentials=True)
|
|
174
174
|
```
|
|
175
175
|
|
|
176
176
|
| Method | Description |
|
|
177
177
|
|--------|-------------|
|
|
178
178
|
| `navigate(url)` | Go to URL |
|
|
179
179
|
| `click(selector)` | Click element |
|
|
180
|
+
| `click_all_by_text(text, role)` | Click all matching elements |
|
|
180
181
|
| `type(selector, text)` | Type text |
|
|
181
182
|
| `wait_for(selector, ms)` | Wait for element |
|
|
182
183
|
| `wait_seconds(n)` | Sleep |
|
|
184
|
+
| `wait_random(min, max)` | Random sleep |
|
|
183
185
|
| `extract(selector, attr)` | Get text/attr |
|
|
184
186
|
| `get_html(selector)` | Get HTML |
|
|
185
|
-
| `
|
|
187
|
+
| `soup(selector)` | → SoupWrapper |
|
|
186
188
|
| `parse_html(html)` | → BeautifulSoup |
|
|
187
|
-
| `extract_data(item, fields, limit)` | Bulk extract |
|
|
188
189
|
| `fetch_json(url)` | JS fetch → dict |
|
|
189
|
-
| `fetch_all(urls
|
|
190
|
+
| `fetch_all(urls)` | Parallel fetch |
|
|
190
191
|
| `execute_js(code)` | Run async JS |
|
|
191
192
|
| `screenshot()` | PNG bytes |
|
|
192
|
-
| `scroll(dir, amount)` | Scroll page |
|
|
193
|
+
| `scroll(dir, amount, ...)` | Scroll page/container |
|
|
193
194
|
| `scroll_to(selector)` | Scroll to element |
|
|
194
195
|
| `get_scroll_info()` | Position + page size |
|
|
195
|
-
| `infinite_scroll(fn, limit)` | Smart scroll loop |
|
|
196
196
|
| `hover(selector)` | Hover |
|
|
197
197
|
| `select(selector, value)` | Dropdown select |
|
|
198
198
|
| `close_modal()` | Close dialogs |
|
|
199
199
|
| `get/set_cookies()` | Cookie management |
|
|
200
200
|
|
|
201
|
+
**scroll() parameters:**
|
|
202
|
+
- `direction`: "up", "down", "left", "right"
|
|
203
|
+
- `amount`: pixels to scroll
|
|
204
|
+
- `smooth`: animate scroll (default True)
|
|
205
|
+
- `human_like`: random micro-scrolls + variation
|
|
206
|
+
- `container`: CSS selector for scroll container
|
|
207
|
+
|
|
201
208
|
## SDKBaseModel
|
|
202
209
|
|
|
203
210
|
Auto-cleaning Pydantic model for scraped data:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
cmdop/__init__.py,sha256=
|
|
1
|
+
cmdop/__init__.py,sha256=tqdxrpPGr61930IR-Igh10FiIwvyNHquP6Vt7ujlv88,5200
|
|
2
2
|
cmdop/client.py,sha256=NzCBMIAj2vnVzwU1OUttW_kpgJh5aRtmixOwa9HUUUg,14924
|
|
3
3
|
cmdop/config.py,sha256=vpw1aGCyS4NKlZyzVur81Lt06QmN3FnscZji0bypUi0,4398
|
|
4
4
|
cmdop/discovery.py,sha256=HNxSOa5tSuG7ppfFs21XdviW5ucjpRswVPguhX5j8Dg,7479
|
|
@@ -171,18 +171,18 @@ cmdop/services/browser/models.py,sha256=2xvbydS2SHLb0BtATBktaZxAn5pRMYVoqTn91eZI
|
|
|
171
171
|
cmdop/services/browser/parsing.py,sha256=0hQAy-0ZwJqtmhEqHO3EEdVB3iYmyhXRdouN_dCbig8,3820
|
|
172
172
|
cmdop/services/browser/aio/__init__.py,sha256=7DI4Q0LZaAdu3fl8D1SaZ8E3LCGpTw97lqca1pOi5Kk,241
|
|
173
173
|
cmdop/services/browser/aio/service.py,sha256=dSiT6hS1wKRinoon1zfO34xR_irGrNYIDU1gt_6OD54,11494
|
|
174
|
-
cmdop/services/browser/aio/session.py,sha256=
|
|
174
|
+
cmdop/services/browser/aio/session.py,sha256=It9XjLxe73Kqm0vYbLv3rYXyZG0QaH1tfDsXNJHtzCU,10518
|
|
175
175
|
cmdop/services/browser/base/__init__.py,sha256=mgG4y324zSDy-13uQiTg6rlFpaQFLc67ynbykRZuMms,227
|
|
176
176
|
cmdop/services/browser/base/service.py,sha256=w8foDUGZcD4HyF5eyLZUFxbx_fctAFsYRovvsksi3l4,1584
|
|
177
|
-
cmdop/services/browser/base/session.py,sha256=
|
|
178
|
-
cmdop/services/browser/js/__init__.py,sha256=
|
|
177
|
+
cmdop/services/browser/base/session.py,sha256=YymfV_G-b9HGae26hMoCNAlNzwb_pGEWgE4i6dgH4_s,3594
|
|
178
|
+
cmdop/services/browser/js/__init__.py,sha256=u4mkKwUeCZSe8EshwRIdcfqbnGqkUcuF8h2qUkXY_xs,1185
|
|
179
179
|
cmdop/services/browser/js/core.py,sha256=QXCCX_al5tMgz7aCwMqhIs1aRe_IdG8teOJniaumA5Q,995
|
|
180
180
|
cmdop/services/browser/js/fetch.py,sha256=WPy_H4LLkneSx06wpfnx4Sx_0Okf2ENXi6bveCd9ZCg,2188
|
|
181
|
-
cmdop/services/browser/js/interaction.py,sha256=
|
|
182
|
-
cmdop/services/browser/js/scroll.py,sha256=
|
|
181
|
+
cmdop/services/browser/js/interaction.py,sha256=CkuUjTtwgBWWtO48L1-YGh6NkKZbplp8-GUtlA2ODPE,4049
|
|
182
|
+
cmdop/services/browser/js/scroll.py,sha256=2nGH-zqbxJkmC5uuAJxv4MY43y0vJJoGzGja3gX0DNQ,7681
|
|
183
183
|
cmdop/services/browser/sync/__init__.py,sha256=sFePvIsWqhaeCU9fw_ZyOITeeEHvZx4Gtcp8W3KBKG0,222
|
|
184
184
|
cmdop/services/browser/sync/service.py,sha256=TVw8yMuGWXc0NetUnxTGU39uv4UH_nPBBk27nPWs7eI,11136
|
|
185
|
-
cmdop/services/browser/sync/session.py,sha256=
|
|
185
|
+
cmdop/services/browser/sync/session.py,sha256=rj8xIoocpuIMGn_xMwBbcwWGmd-QtFklo8TDoUoAbFc,15478
|
|
186
186
|
cmdop/streaming/__init__.py,sha256=kG9UlJRqv8ndcwKMzWUddPlZT61pFO_Uf_c08A_8TxA,877
|
|
187
187
|
cmdop/streaming/base.py,sha256=r7Q2QlRxgULzs9vlSGcOC_fwAQ_cF3Z3M7WsPQtxG5I,2990
|
|
188
188
|
cmdop/streaming/handlers.py,sha256=FDEhADmCEFRbifvr9dU1X3C-K_96noz89Bl3tuDa_rQ,2616
|
|
@@ -193,7 +193,7 @@ cmdop/transport/base.py,sha256=2pkV8i9epgp_21dyReCfX47abRUrnALm0W5BXb-Fuz0,5571
|
|
|
193
193
|
cmdop/transport/discovery.py,sha256=rcGAuVrR1l6jwcP0dqZxVhX1NsFK7sRHygFMCLmmUbA,10673
|
|
194
194
|
cmdop/transport/local.py,sha256=ob6tWVxSdKwblHSMK8CkgjyuSdQoAeWgy5OAUd5ZNuE,7411
|
|
195
195
|
cmdop/transport/remote.py,sha256=FNVqus9wOv7LlxKarXjLmSyvJiHwhvPbNDOPv1IQkmE,4329
|
|
196
|
-
cmdop-0.1.
|
|
197
|
-
cmdop-0.1.
|
|
198
|
-
cmdop-0.1.
|
|
199
|
-
cmdop-0.1.
|
|
196
|
+
cmdop-0.1.18.dist-info/METADATA,sha256=hyz91IkVkE_zZy1RfQpn7TCSXj30jjQgetxXpUU5DhI,6721
|
|
197
|
+
cmdop-0.1.18.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
198
|
+
cmdop-0.1.18.dist-info/licenses/LICENSE,sha256=6hyzbI1QVXW6B-XT7PaQ6UG9lns11Y_nnap8uUKGUqo,1062
|
|
199
|
+
cmdop-0.1.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|