grip-browser 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grip/__init__.py +20 -0
- grip/adapters/__init__.py +0 -0
- grip/adapters/anthropic.py +33 -0
- grip/adapters/base.py +24 -0
- grip/adapters/openai.py +36 -0
- grip/browser.py +98 -0
- grip/cdp/__init__.py +0 -0
- grip/cdp/engine.py +78 -0
- grip/cdp/launcher.py +83 -0
- grip/cdp/shadow.py +142 -0
- grip/compression/__init__.py +0 -0
- grip/compression/cache.py +26 -0
- grip/compression/diff.py +21 -0
- grip/compression/summarizer.py +103 -0
- grip/errors/__init__.py +3 -0
- grip/errors/classifier.py +116 -0
- grip/errors/types.py +43 -0
- grip/page.py +252 -0
- grip/py.typed +0 -0
- grip/runner.py +146 -0
- grip/security/__init__.py +0 -0
- grip/security/injection.py +65 -0
- grip/security/sanitizer.py +40 -0
- grip/trace.py +53 -0
- grip_browser-0.1.0.dist-info/METADATA +249 -0
- grip_browser-0.1.0.dist-info/RECORD +27 -0
- grip_browser-0.1.0.dist-info/WHEEL +4 -0
grip/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from grip.browser import Browser
|
|
2
|
+
from grip.compression.summarizer import Element, PageSnapshot
|
|
3
|
+
from grip.errors.types import BrowserError, ErrorType, GripError, RecoveryAction
|
|
4
|
+
from grip.page import Screenshot
|
|
5
|
+
from grip.trace import Trace, TraceEntry
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Browser",
|
|
9
|
+
"PageSnapshot",
|
|
10
|
+
"Element",
|
|
11
|
+
"BrowserError",
|
|
12
|
+
"ErrorType",
|
|
13
|
+
"GripError",
|
|
14
|
+
"RecoveryAction",
|
|
15
|
+
"Screenshot",
|
|
16
|
+
"Trace",
|
|
17
|
+
"TraceEntry",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
File without changes
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
import anthropic as anthropic
|
|
5
|
+
except ImportError:
|
|
6
|
+
anthropic = None
|
|
7
|
+
|
|
8
|
+
from grip.adapters.base import LLMResponse, ToolCall
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AnthropicAdapter:
|
|
12
|
+
def __init__(
|
|
13
|
+
self, api_key: str | None = None, model: str = "claude-opus-4-7"
|
|
14
|
+
) -> None:
|
|
15
|
+
if anthropic is None:
|
|
16
|
+
raise ImportError("pip install grip-browser[anthropic]")
|
|
17
|
+
self._client = anthropic.AsyncAnthropic(api_key=api_key)
|
|
18
|
+
self._model = model
|
|
19
|
+
|
|
20
|
+
async def complete(self, messages: list[dict], tools: list[dict]) -> LLMResponse:
|
|
21
|
+
kwargs = {"model": self._model, "max_tokens": 4096, "messages": messages}
|
|
22
|
+
if tools:
|
|
23
|
+
kwargs["tools"] = tools
|
|
24
|
+
response = await self._client.messages.create(**kwargs)
|
|
25
|
+
if response.stop_reason == "tool_use":
|
|
26
|
+
for block in response.content:
|
|
27
|
+
if hasattr(block, "type") and block.type == "tool_use":
|
|
28
|
+
return LLMResponse(
|
|
29
|
+
content=None,
|
|
30
|
+
tool_call=ToolCall(name=block.name, arguments=block.input),
|
|
31
|
+
)
|
|
32
|
+
text = "".join(block.text for block in response.content if hasattr(block, "text"))
|
|
33
|
+
return LLMResponse(content=text, tool_call=None)
|
grip/adapters/base.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any, Protocol, runtime_checkable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class ToolCall:
|
|
8
|
+
name: str
|
|
9
|
+
arguments: dict[str, Any]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class LLMResponse:
|
|
14
|
+
content: str | None
|
|
15
|
+
tool_call: ToolCall | None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@runtime_checkable
|
|
19
|
+
class LLMAdapter(Protocol):
|
|
20
|
+
async def complete(
|
|
21
|
+
self,
|
|
22
|
+
messages: list[dict],
|
|
23
|
+
tools: list[dict],
|
|
24
|
+
) -> LLMResponse: ...
|
grip/adapters/openai.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
import openai as openai
|
|
7
|
+
except ImportError:
|
|
8
|
+
openai = None
|
|
9
|
+
|
|
10
|
+
from grip.adapters.base import LLMResponse, ToolCall
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class OpenAIAdapter:
|
|
14
|
+
def __init__(self, api_key: str | None = None, model: str = "gpt-4o") -> None:
|
|
15
|
+
if openai is None:
|
|
16
|
+
raise ImportError("pip install grip-browser[openai]")
|
|
17
|
+
self._client = openai.AsyncOpenAI(api_key=api_key)
|
|
18
|
+
self._model = model
|
|
19
|
+
|
|
20
|
+
async def complete(self, messages: list[dict], tools: list[dict]) -> LLMResponse:
|
|
21
|
+
kwargs: dict[str, Any] = {"model": self._model, "messages": messages}
|
|
22
|
+
if tools:
|
|
23
|
+
kwargs["tools"] = tools
|
|
24
|
+
response = await self._client.chat.completions.create(**kwargs)
|
|
25
|
+
choice = response.choices[0]
|
|
26
|
+
msg = choice.message
|
|
27
|
+
if msg.tool_calls:
|
|
28
|
+
tc = msg.tool_calls[0]
|
|
29
|
+
return LLMResponse(
|
|
30
|
+
content=None,
|
|
31
|
+
tool_call=ToolCall(
|
|
32
|
+
name=tc.function.name,
|
|
33
|
+
arguments=json.loads(tc.function.arguments),
|
|
34
|
+
),
|
|
35
|
+
)
|
|
36
|
+
return LLMResponse(content=msg.content, tool_call=None)
|
grip/browser.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from grip.cdp.engine import CDPEngine
|
|
7
|
+
from grip.cdp.launcher import ChromeLauncher
|
|
8
|
+
from grip.page import Page
|
|
9
|
+
from grip.trace import Trace
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from grip.adapters.base import LLMAdapter
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
async def fetch_tab_ws_url(port: int) -> str:
|
|
16
|
+
import urllib.request
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
def _do_fetch() -> list:
|
|
20
|
+
with urllib.request.urlopen(
|
|
21
|
+
f"http://localhost:{port}/json", timeout=2
|
|
22
|
+
) as resp:
|
|
23
|
+
return json.loads(resp.read())
|
|
24
|
+
|
|
25
|
+
deadline = time.monotonic() + 10.0
|
|
26
|
+
while time.monotonic() < deadline:
|
|
27
|
+
try:
|
|
28
|
+
tabs = await asyncio.to_thread(_do_fetch)
|
|
29
|
+
for tab in tabs:
|
|
30
|
+
if tab.get("type") == "page":
|
|
31
|
+
return tab["webSocketDebuggerUrl"]
|
|
32
|
+
except Exception:
|
|
33
|
+
pass
|
|
34
|
+
await asyncio.sleep(0.2)
|
|
35
|
+
raise RuntimeError(f"No Chrome tab found on port {port}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Browser:
|
|
39
|
+
def __init__(self, llm: "LLMAdapter | None" = None, headless: bool = True) -> None:
|
|
40
|
+
self._llm = llm
|
|
41
|
+
self._headless = headless
|
|
42
|
+
self._launcher: ChromeLauncher | None = None
|
|
43
|
+
self._engine: CDPEngine | None = None
|
|
44
|
+
self.trace = Trace()
|
|
45
|
+
|
|
46
|
+
async def __aenter__(self) -> "Browser":
|
|
47
|
+
self._launcher = ChromeLauncher()
|
|
48
|
+
port = self._launcher.launch(headless=self._headless)
|
|
49
|
+
ws_url = await fetch_tab_ws_url(port)
|
|
50
|
+
self._engine = CDPEngine()
|
|
51
|
+
await self._engine.connect(ws_url)
|
|
52
|
+
return self
|
|
53
|
+
|
|
54
|
+
async def __aexit__(self, *args) -> None:
|
|
55
|
+
await self.close()
|
|
56
|
+
|
|
57
|
+
async def open(self, url: str) -> Page:
|
|
58
|
+
if not self._engine:
|
|
59
|
+
self._launcher = ChromeLauncher()
|
|
60
|
+
port = self._launcher.launch(headless=self._headless)
|
|
61
|
+
ws_url = await fetch_tab_ws_url(port)
|
|
62
|
+
self._engine = CDPEngine()
|
|
63
|
+
await self._engine.connect(ws_url)
|
|
64
|
+
|
|
65
|
+
if not url.startswith(("http", "about:", "data:", "file:", "blob:")):
|
|
66
|
+
url = "https://" + url
|
|
67
|
+
|
|
68
|
+
await self._engine.send("Page.navigate", {"url": url})
|
|
69
|
+
await self._engine.send("Page.enable")
|
|
70
|
+
|
|
71
|
+
load_event = asyncio.Event()
|
|
72
|
+
|
|
73
|
+
def on_load(params):
|
|
74
|
+
load_event.set()
|
|
75
|
+
|
|
76
|
+
self._engine.on("Page.loadEventFired", on_load)
|
|
77
|
+
try:
|
|
78
|
+
await asyncio.wait_for(load_event.wait(), timeout=30.0)
|
|
79
|
+
except asyncio.TimeoutError:
|
|
80
|
+
pass
|
|
81
|
+
finally:
|
|
82
|
+
self._engine.off("Page.loadEventFired", on_load)
|
|
83
|
+
|
|
84
|
+
return Page(engine=self._engine, trace=self.trace)
|
|
85
|
+
|
|
86
|
+
async def run(self, goal: str, url: str) -> "RunResult":
|
|
87
|
+
from grip.runner import Runner
|
|
88
|
+
page = await self.open(url)
|
|
89
|
+
runner = Runner(llm=self._llm, page=page, trace=self.trace)
|
|
90
|
+
return await runner.run(goal)
|
|
91
|
+
|
|
92
|
+
async def close(self) -> None:
|
|
93
|
+
if self._engine:
|
|
94
|
+
await self._engine.disconnect()
|
|
95
|
+
self._engine = None
|
|
96
|
+
if self._launcher:
|
|
97
|
+
self._launcher.terminate()
|
|
98
|
+
self._launcher = None
|
grip/cdp/__init__.py
ADDED
|
File without changes
|
grip/cdp/engine.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import websockets
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CDPEngine:
|
|
14
|
+
def __init__(self) -> None:
|
|
15
|
+
self._ws: websockets.WebSocketClientProtocol | None = None
|
|
16
|
+
self._id = 0
|
|
17
|
+
self._pending: dict[int, asyncio.Future] = {}
|
|
18
|
+
self._listeners: dict[str, list[Callable]] = {}
|
|
19
|
+
self._receive_task: asyncio.Task | None = None
|
|
20
|
+
|
|
21
|
+
def _next_id(self) -> int:
|
|
22
|
+
self._id += 1
|
|
23
|
+
return self._id
|
|
24
|
+
|
|
25
|
+
async def connect(self, url: str) -> None:
|
|
26
|
+
self._ws = await websockets.connect(url, max_size=50 * 1024 * 1024)
|
|
27
|
+
self._receive_task = asyncio.create_task(self._receive_loop())
|
|
28
|
+
|
|
29
|
+
async def disconnect(self) -> None:
|
|
30
|
+
if self._receive_task:
|
|
31
|
+
self._receive_task.cancel()
|
|
32
|
+
try:
|
|
33
|
+
await self._receive_task
|
|
34
|
+
except asyncio.CancelledError:
|
|
35
|
+
pass
|
|
36
|
+
if self._ws:
|
|
37
|
+
await self._ws.close()
|
|
38
|
+
self._ws = None
|
|
39
|
+
|
|
40
|
+
async def send(self, method: str, params: dict[str, Any] | None = None) -> Any:
|
|
41
|
+
msg_id = self._next_id()
|
|
42
|
+
fut: asyncio.Future = asyncio.get_running_loop().create_future()
|
|
43
|
+
self._pending[msg_id] = fut
|
|
44
|
+
payload = json.dumps({"id": msg_id, "method": method, "params": params or {}})
|
|
45
|
+
await self._ws.send(payload)
|
|
46
|
+
try:
|
|
47
|
+
return await asyncio.wait_for(fut, timeout=30.0)
|
|
48
|
+
except asyncio.TimeoutError:
|
|
49
|
+
self._pending.pop(msg_id, None)
|
|
50
|
+
raise TimeoutError(f"CDP command {method} timed out")
|
|
51
|
+
|
|
52
|
+
def on(self, event: str, callback: Callable) -> None:
|
|
53
|
+
self._listeners.setdefault(event, []).append(callback)
|
|
54
|
+
|
|
55
|
+
def off(self, event: str, callback: Callable) -> None:
|
|
56
|
+
listeners = self._listeners.get(event, [])
|
|
57
|
+
if callback in listeners:
|
|
58
|
+
listeners.remove(callback)
|
|
59
|
+
|
|
60
|
+
async def _receive_loop(self) -> None:
|
|
61
|
+
try:
|
|
62
|
+
async for raw in self._ws:
|
|
63
|
+
msg = json.loads(raw)
|
|
64
|
+
if "id" in msg:
|
|
65
|
+
fut = self._pending.pop(msg["id"], None)
|
|
66
|
+
if fut and not fut.done():
|
|
67
|
+
if "error" in msg:
|
|
68
|
+
fut.set_exception(RuntimeError(msg["error"]["message"]))
|
|
69
|
+
else:
|
|
70
|
+
fut.set_result(msg.get("result", {}))
|
|
71
|
+
elif "method" in msg:
|
|
72
|
+
for cb in self._listeners.get(msg["method"], []):
|
|
73
|
+
try:
|
|
74
|
+
cb(msg.get("params", {}))
|
|
75
|
+
except Exception:
|
|
76
|
+
logger.exception("CDP listener error for %s", msg["method"])
|
|
77
|
+
except Exception:
|
|
78
|
+
logger.debug("CDP receive loop ended")
|
grip/cdp/launcher.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
import tempfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_CHROME_CANDIDATES = [
|
|
9
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
10
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
11
|
+
"/usr/bin/google-chrome",
|
|
12
|
+
"/usr/bin/chromium-browser",
|
|
13
|
+
"/usr/bin/chromium",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def find_chrome() -> str | None:
|
|
18
|
+
if exe := os.environ.get("CHROME_EXECUTABLE"):
|
|
19
|
+
return exe
|
|
20
|
+
for candidate in _CHROME_CANDIDATES:
|
|
21
|
+
if Path(candidate).exists():
|
|
22
|
+
return candidate
|
|
23
|
+
for name in ("google-chrome", "chromium", "chromium-browser"):
|
|
24
|
+
try:
|
|
25
|
+
result = subprocess.run(["which", name], capture_output=True, text=True)
|
|
26
|
+
if result.returncode == 0 and result.stdout.strip():
|
|
27
|
+
return result.stdout.strip()
|
|
28
|
+
except FileNotFoundError:
|
|
29
|
+
pass
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ChromeLauncher:
|
|
34
|
+
def __init__(self) -> None:
|
|
35
|
+
exe = find_chrome()
|
|
36
|
+
if not exe:
|
|
37
|
+
raise RuntimeError(
|
|
38
|
+
"Chrome/Chromium not found. Install Chrome or set CHROME_EXECUTABLE."
|
|
39
|
+
)
|
|
40
|
+
self.executable = exe
|
|
41
|
+
self._process: subprocess.Popen | None = None
|
|
42
|
+
self._user_data_dir: str | None = None
|
|
43
|
+
|
|
44
|
+
def launch(self, headless: bool = True) -> int:
|
|
45
|
+
self._user_data_dir = tempfile.mkdtemp(prefix="grip_chrome_")
|
|
46
|
+
args = [
|
|
47
|
+
self.executable,
|
|
48
|
+
"--remote-debugging-port=0",
|
|
49
|
+
f"--user-data-dir={self._user_data_dir}",
|
|
50
|
+
"--no-first-run",
|
|
51
|
+
"--no-default-browser-check",
|
|
52
|
+
"--disable-extensions",
|
|
53
|
+
]
|
|
54
|
+
if headless:
|
|
55
|
+
args.append("--headless=new")
|
|
56
|
+
self._process = subprocess.Popen(
|
|
57
|
+
args,
|
|
58
|
+
stdout=subprocess.DEVNULL,
|
|
59
|
+
stderr=subprocess.DEVNULL,
|
|
60
|
+
)
|
|
61
|
+
port = self._read_port()
|
|
62
|
+
return port
|
|
63
|
+
|
|
64
|
+
def _read_port(self) -> int:
|
|
65
|
+
import time
|
|
66
|
+
port_file = Path(self._user_data_dir) / "DevToolsActivePort"
|
|
67
|
+
deadline = time.monotonic() + 10.0
|
|
68
|
+
while time.monotonic() < deadline:
|
|
69
|
+
if port_file.exists():
|
|
70
|
+
text = port_file.read_text().strip()
|
|
71
|
+
return int(text.split("\n")[0])
|
|
72
|
+
time.sleep(0.05)
|
|
73
|
+
raise RuntimeError("Timed out waiting for Chrome DevTools port")
|
|
74
|
+
|
|
75
|
+
def terminate(self) -> None:
|
|
76
|
+
if self._process:
|
|
77
|
+
self._process.terminate()
|
|
78
|
+
self._process.wait(timeout=5)
|
|
79
|
+
self._process = None
|
|
80
|
+
if self._user_data_dir:
|
|
81
|
+
import shutil
|
|
82
|
+
shutil.rmtree(self._user_data_dir, ignore_errors=True)
|
|
83
|
+
self._user_data_dir = None
|
grip/cdp/shadow.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
DISCOVER_ELEMENTS_JS = """
|
|
2
|
+
(function() {
|
|
3
|
+
const results = [];
|
|
4
|
+
let idx = 0;
|
|
5
|
+
|
|
6
|
+
const INTERACTIVE_TAGS = new Set([
|
|
7
|
+
'a','button','input','select','textarea','details','summary'
|
|
8
|
+
]);
|
|
9
|
+
const INTERACTIVE_ROLES = new Set([
|
|
10
|
+
'button','link','checkbox','radio','menuitem','tab','textbox',
|
|
11
|
+
'combobox','listbox','option','switch','treeitem','slider'
|
|
12
|
+
]);
|
|
13
|
+
|
|
14
|
+
function collectElements(root, inShadow) {
|
|
15
|
+
const walker = document.createTreeWalker(
|
|
16
|
+
root,
|
|
17
|
+
NodeFilter.SHOW_ELEMENT,
|
|
18
|
+
null
|
|
19
|
+
);
|
|
20
|
+
let node = walker.currentNode;
|
|
21
|
+
while (node) {
|
|
22
|
+
const el = node;
|
|
23
|
+
if (!el.tagName) { node = walker.nextNode(); continue; }
|
|
24
|
+
const tag = el.tagName.toLowerCase();
|
|
25
|
+
const role = el.getAttribute('role') || el.getAttribute('aria-role') || '';
|
|
26
|
+
const ariaHidden = el.getAttribute('aria-hidden') === 'true';
|
|
27
|
+
const style = window.getComputedStyle(el);
|
|
28
|
+
const hidden = (
|
|
29
|
+
style.display === 'none' ||
|
|
30
|
+
style.visibility === 'hidden' ||
|
|
31
|
+
parseFloat(style.opacity) === 0 ||
|
|
32
|
+
ariaHidden ||
|
|
33
|
+
el.offsetWidth === 0 ||
|
|
34
|
+
el.offsetHeight === 0
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
if (!hidden && (INTERACTIVE_TAGS.has(tag) || INTERACTIVE_ROLES.has(role))) {
|
|
38
|
+
const rect = el.getBoundingClientRect();
|
|
39
|
+
results.push({
|
|
40
|
+
index: idx++,
|
|
41
|
+
tag: tag,
|
|
42
|
+
role: role || tag,
|
|
43
|
+
text: (el.innerText || el.value || el.getAttribute('aria-label') || '').trim().slice(0, 120),
|
|
44
|
+
placeholder: el.getAttribute('placeholder') || null,
|
|
45
|
+
inShadowDom: inShadow,
|
|
46
|
+
cx: Math.round(rect.left + rect.width / 2),
|
|
47
|
+
cy: Math.round(rect.top + rect.height / 2),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (el.shadowRoot) {
|
|
52
|
+
collectElements(el.shadowRoot, true);
|
|
53
|
+
}
|
|
54
|
+
node = walker.nextNode();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
collectElements(document.body, false);
|
|
59
|
+
return results;
|
|
60
|
+
})();
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
CLICK_ELEMENT_JS = """
|
|
64
|
+
function(index) {
|
|
65
|
+
const elements = [];
|
|
66
|
+
const INTERACTIVE_TAGS = new Set([
|
|
67
|
+
'a','button','input','select','textarea','details','summary'
|
|
68
|
+
]);
|
|
69
|
+
const INTERACTIVE_ROLES = new Set([
|
|
70
|
+
'button','link','checkbox','radio','menuitem','tab','textbox',
|
|
71
|
+
'combobox','listbox','option','switch','treeitem','slider'
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
function collect(root) {
|
|
75
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null);
|
|
76
|
+
let node = walker.currentNode;
|
|
77
|
+
while (node) {
|
|
78
|
+
const el = node;
|
|
79
|
+
if (!el.tagName) { node = walker.nextNode(); continue; }
|
|
80
|
+
const tag = el.tagName.toLowerCase();
|
|
81
|
+
const role = el.getAttribute('role') || '';
|
|
82
|
+
if (INTERACTIVE_TAGS.has(tag) || INTERACTIVE_ROLES.has(role)) {
|
|
83
|
+
const style = window.getComputedStyle(el);
|
|
84
|
+
const hidden = (style.display === 'none' || style.visibility === 'hidden');
|
|
85
|
+
if (!hidden) elements.push(el);
|
|
86
|
+
}
|
|
87
|
+
if (el.shadowRoot) collect(el.shadowRoot);
|
|
88
|
+
node = walker.nextNode();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
collect(document.body);
|
|
93
|
+
if (index < elements.length) {
|
|
94
|
+
elements[index].click();
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
TYPE_ELEMENT_JS = """
|
|
102
|
+
function(index, text) {
|
|
103
|
+
const elements = [];
|
|
104
|
+
const INPUT_TAGS = new Set(['input','textarea']);
|
|
105
|
+
|
|
106
|
+
function collect(root) {
|
|
107
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null);
|
|
108
|
+
let node = walker.currentNode;
|
|
109
|
+
while (node) {
|
|
110
|
+
const el = node;
|
|
111
|
+
if (!el.tagName) { node = walker.nextNode(); continue; }
|
|
112
|
+
const tag = el.tagName.toLowerCase();
|
|
113
|
+
if (INPUT_TAGS.has(tag) || el.isContentEditable) {
|
|
114
|
+
const style = window.getComputedStyle(el);
|
|
115
|
+
if (style.display !== 'none') elements.push(el);
|
|
116
|
+
}
|
|
117
|
+
if (el.shadowRoot) collect(el.shadowRoot);
|
|
118
|
+
node = walker.nextNode();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
collect(document.body);
|
|
123
|
+
if (index < elements.length) {
|
|
124
|
+
const el = elements[index];
|
|
125
|
+
el.focus();
|
|
126
|
+
el.value = '';
|
|
127
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
128
|
+
el.value = text;
|
|
129
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
130
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
131
|
+
return true;
|
|
132
|
+
}
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
PAGE_TEXT_JS = """
|
|
138
|
+
(function() {
|
|
139
|
+
const main = document.querySelector('main, [role="main"]') || document.body;
|
|
140
|
+
return (main.innerText || main.textContent || '').replace(/\\s+/g, ' ').trim().slice(0, 8000);
|
|
141
|
+
})();
|
|
142
|
+
"""
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import hashlib
|
|
3
|
+
from grip.compression.summarizer import Element
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _cache_key(tag: str, text: str) -> str:
|
|
7
|
+
return hashlib.md5(f"{tag}:{text}".encode()).hexdigest()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ElementCache:
|
|
11
|
+
def __init__(self) -> None:
|
|
12
|
+
self._store: dict[str, Element] = {}
|
|
13
|
+
|
|
14
|
+
def store(self, element: Element) -> None:
|
|
15
|
+
key = _cache_key(element.tag, element.text)
|
|
16
|
+
self._store[key] = element
|
|
17
|
+
|
|
18
|
+
def get(self, tag: str, text: str) -> Element | None:
|
|
19
|
+
return self._store.get(_cache_key(tag, text))
|
|
20
|
+
|
|
21
|
+
def invalidate(self) -> None:
|
|
22
|
+
self._store.clear()
|
|
23
|
+
|
|
24
|
+
def store_many(self, elements: list[Element]) -> None:
|
|
25
|
+
for el in elements:
|
|
26
|
+
self.store(el)
|
grip/compression/diff.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import hashlib
|
|
3
|
+
from grip.compression.summarizer import PageSnapshot
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _snapshot_fingerprint(snap: PageSnapshot) -> str:
|
|
7
|
+
element_sig = "|".join(f"{el.tag}:{el.text}" for el in snap.elements)
|
|
8
|
+
raw = f"{snap.url}||{snap.text_content[:500]}||{element_sig}"
|
|
9
|
+
return hashlib.md5(raw.encode()).hexdigest()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SnapshotDiff:
|
|
13
|
+
def __init__(self) -> None:
|
|
14
|
+
self._last_fingerprint: str | None = None
|
|
15
|
+
|
|
16
|
+
def has_changed(self, snapshot: PageSnapshot) -> bool:
|
|
17
|
+
fp = _snapshot_fingerprint(snapshot)
|
|
18
|
+
return fp != self._last_fingerprint
|
|
19
|
+
|
|
20
|
+
def record(self, snapshot: PageSnapshot) -> None:
|
|
21
|
+
self._last_fingerprint = _snapshot_fingerprint(snapshot)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
|
|
4
|
+
from grip.security.sanitizer import RawElement
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import tiktoken
|
|
8
|
+
_ENC = tiktoken.get_encoding("cl100k_base")
|
|
9
|
+
|
|
10
|
+
def _count_tokens(text: str) -> int:
|
|
11
|
+
return len(_ENC.encode(text))
|
|
12
|
+
except Exception:
|
|
13
|
+
def _count_tokens(text: str) -> int:
|
|
14
|
+
return len(text) // 4
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_TAG_ABBREV = {
|
|
18
|
+
"button": "btn",
|
|
19
|
+
"input": "inp",
|
|
20
|
+
"a": "lnk",
|
|
21
|
+
"select": "sel",
|
|
22
|
+
"textarea": "inp",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Element:
|
|
28
|
+
index: int
|
|
29
|
+
snapshot_version: int
|
|
30
|
+
tag: str
|
|
31
|
+
role: str
|
|
32
|
+
text: str
|
|
33
|
+
placeholder: str | None
|
|
34
|
+
in_shadow_dom: bool
|
|
35
|
+
cx: int
|
|
36
|
+
cy: int
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class PageSnapshot:
|
|
41
|
+
version: int
|
|
42
|
+
url: str
|
|
43
|
+
title: str
|
|
44
|
+
elements: list[Element]
|
|
45
|
+
text_content: str
|
|
46
|
+
tokens_estimated: int
|
|
47
|
+
changed_from_previous: bool = True
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Summarizer:
|
|
51
|
+
def build(
|
|
52
|
+
self,
|
|
53
|
+
version: int,
|
|
54
|
+
url: str,
|
|
55
|
+
title: str,
|
|
56
|
+
raw_elements: list[RawElement],
|
|
57
|
+
page_text: str,
|
|
58
|
+
) -> PageSnapshot:
|
|
59
|
+
elements = [
|
|
60
|
+
Element(
|
|
61
|
+
index=i,
|
|
62
|
+
snapshot_version=version,
|
|
63
|
+
tag=el.tag,
|
|
64
|
+
role=el.role,
|
|
65
|
+
text=el.text,
|
|
66
|
+
placeholder=el.placeholder,
|
|
67
|
+
in_shadow_dom=el.in_shadow_dom,
|
|
68
|
+
cx=el.cx,
|
|
69
|
+
cy=el.cy,
|
|
70
|
+
)
|
|
71
|
+
for i, el in enumerate(raw_elements)
|
|
72
|
+
]
|
|
73
|
+
text_content = page_text.strip()
|
|
74
|
+
formatted = self._build_format_str(url, title, elements, text_content)
|
|
75
|
+
tokens = _count_tokens(formatted)
|
|
76
|
+
return PageSnapshot(
|
|
77
|
+
version=version,
|
|
78
|
+
url=url,
|
|
79
|
+
title=title,
|
|
80
|
+
elements=elements,
|
|
81
|
+
text_content=text_content,
|
|
82
|
+
tokens_estimated=tokens,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def format(self, snapshot: PageSnapshot) -> str:
|
|
86
|
+
return self._build_format_str(
|
|
87
|
+
snapshot.url, snapshot.title, snapshot.elements, snapshot.text_content
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
def _build_format_str(
|
|
91
|
+
self, url: str, title: str, elements: list[Element], text: str
|
|
92
|
+
) -> str:
|
|
93
|
+
lines = [f"PAGE: {title}", f"URL: {url}"]
|
|
94
|
+
if elements:
|
|
95
|
+
lines.append("INTERACTIVE:")
|
|
96
|
+
for el in elements:
|
|
97
|
+
abbrev = _TAG_ABBREV.get(el.tag, el.tag[:3])
|
|
98
|
+
desc = el.text or el.placeholder or el.role
|
|
99
|
+
lines.append(f" [{abbrev}:{el.index}] {desc!r}")
|
|
100
|
+
if text:
|
|
101
|
+
lines.append("CONTENT:")
|
|
102
|
+
lines.append(f" {text[:2000]}")
|
|
103
|
+
return "\n".join(lines)
|
grip/errors/__init__.py
ADDED