cookieguard-browser-use 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cookieguard_browser_use/__init__.py +22 -0
- cookieguard_browser_use/_audit.py +289 -0
- cookieguard_browser_use/_collector.py +388 -0
- cookieguard_browser_use/_engine.py +256 -0
- cookieguard_browser_use/_errors.py +44 -0
- cookieguard_browser_use/_node/bridge.mjs +179 -0
- cookieguard_browser_use/_node/manifest.json +40 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/LICENSE +21 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/README.md +95 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/package.json +17 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/index.d.ts +80 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/index.js +4 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/normalize.js +169 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/onnx.d.ts +6 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/onnx.js +37 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/policy.js +182 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/report.js +28 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/src/safety.js +54 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-core/vendor/engine.cjs +799 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-playwright/LICENSE +21 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-playwright/README.md +62 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-playwright/package.json +15 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-playwright/src/index.d.ts +19 -0
- cookieguard_browser_use/_node/node_modules/@aiscihub/cookieguard-playwright/src/index.js +145 -0
- cookieguard_browser_use/_origins.py +85 -0
- cookieguard_browser_use/_result.py +211 -0
- cookieguard_browser_use/demo/__init__.py +0 -0
- cookieguard_browser_use/demo/__main__.py +58 -0
- cookieguard_browser_use/demo/app.py +176 -0
- cookieguard_browser_use/demo/scripted.py +141 -0
- cookieguard_browser_use/py.typed +0 -0
- cookieguard_browser_use-0.1.0.dist-info/METADATA +104 -0
- cookieguard_browser_use-0.1.0.dist-info/RECORD +35 -0
- cookieguard_browser_use-0.1.0.dist-info/WHEEL +4 -0
- cookieguard_browser_use-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""CookieGuard authentication-cookie policy checks for Browser Use agents.
|
|
2
|
+
|
|
3
|
+
The agent performs the authorized workflow; CookieGuard observes the same
|
|
4
|
+
browser over its own CDP connection and the local CookieGuard core engine
|
|
5
|
+
decides PASS, FAIL, or INCOMPLETE. The agent cannot override that decision.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from ._audit import ACTION_DESCRIPTION, DEFAULT_ACTION_NAME, CookieGuardAudit
|
|
9
|
+
from ._errors import CookieGuardError, CookiePolicyError
|
|
10
|
+
from ._result import AuditResult, Finding
|
|
11
|
+
|
|
12
|
+
__version__ = '0.1.0'
|
|
13
|
+
__all__ = [
|
|
14
|
+
'ACTION_DESCRIPTION',
|
|
15
|
+
'AuditResult',
|
|
16
|
+
'CookieGuardAudit',
|
|
17
|
+
'CookieGuardError',
|
|
18
|
+
'CookiePolicyError',
|
|
19
|
+
'DEFAULT_ACTION_NAME',
|
|
20
|
+
'Finding',
|
|
21
|
+
'__version__',
|
|
22
|
+
]
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Public CookieGuard audit for a Browser Use BrowserSession."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from collections.abc import Iterable, Mapping
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ._collector import CdpEvidenceCollector
|
|
11
|
+
from ._engine import LocalEngine
|
|
12
|
+
from ._errors import CookieGuardError
|
|
13
|
+
from ._origins import validate_origins
|
|
14
|
+
from ._result import AuditResult, guard_values
|
|
15
|
+
|
|
16
|
+
_logger = logging.getLogger('cookieguard_browser_use')
|
|
17
|
+
_SEVERITIES = ('info', 'low', 'medium', 'high', 'critical')
|
|
18
|
+
DEFAULT_ACTION_NAME = 'cookieguard_audit'
|
|
19
|
+
ACTION_DESCRIPTION = (
|
|
20
|
+
'Check authentication-cookie security for the authorized test application with CookieGuard. '
|
|
21
|
+
'Read-only: returns sanitized cookie metadata findings, never cookie values. Use after logging in. '
|
|
22
|
+
'This is a provisional check; the final pass/fail result is enforced outside the agent.'
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def _sanitized(awaitable, code: str):
|
|
27
|
+
"""Await, re-raising only fixed-message errors. No exception text or context escapes."""
|
|
28
|
+
error: CookieGuardError | None = None
|
|
29
|
+
try:
|
|
30
|
+
return await awaitable
|
|
31
|
+
except CookieGuardError as caught:
|
|
32
|
+
error = CookieGuardError(caught.code)
|
|
33
|
+
except asyncio.CancelledError:
|
|
34
|
+
raise
|
|
35
|
+
except Exception:
|
|
36
|
+
error = CookieGuardError(code)
|
|
37
|
+
raise error
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _names(values: Iterable[str] | None) -> list[str]:
|
|
41
|
+
if values is None:
|
|
42
|
+
return []
|
|
43
|
+
if isinstance(values, str):
|
|
44
|
+
raise CookieGuardError('CONFIG')
|
|
45
|
+
names = list(values)
|
|
46
|
+
if len(names) > 100 or any(not isinstance(name, str) or not name for name in names):
|
|
47
|
+
raise CookieGuardError('CONFIG')
|
|
48
|
+
return names
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class CookieGuardAudit:
|
|
52
|
+
"""Audit authentication cookies created during a Browser Use workflow.
|
|
53
|
+
|
|
54
|
+
Collection starts with :meth:`start` and ends with :meth:`finish`, both called
|
|
55
|
+
by developer code. The final status is decided by the CookieGuard core engine,
|
|
56
|
+
never by the agent. One instance audits one workflow; create a new instance for
|
|
57
|
+
another workflow or session.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
*,
|
|
63
|
+
allowed_origins: Iterable[str],
|
|
64
|
+
auth_cookies: Iterable[str] | None = None,
|
|
65
|
+
fail_on: str = 'high',
|
|
66
|
+
allowed_same_site: Iterable[str] = ('lax', 'strict'),
|
|
67
|
+
require_explicit_same_site: bool = False,
|
|
68
|
+
require_host_only: bool = False,
|
|
69
|
+
max_lifetime_days: float = 30,
|
|
70
|
+
auth_evidence: Iterable[Mapping[str, str]] | None = None,
|
|
71
|
+
capture_headers: bool = True,
|
|
72
|
+
require_header_evidence: bool | None = None,
|
|
73
|
+
allow_nonlocal_origins: bool = False,
|
|
74
|
+
timeout_seconds: float = 600,
|
|
75
|
+
node_path: str | None = None,
|
|
76
|
+
) -> None:
|
|
77
|
+
if isinstance(allowed_origins, str):
|
|
78
|
+
raise CookieGuardError('CONFIG')
|
|
79
|
+
self.origins = validate_origins(list(allowed_origins), allow_nonlocal_origins)
|
|
80
|
+
if fail_on not in _SEVERITIES or isinstance(allowed_same_site, str):
|
|
81
|
+
raise CookieGuardError('CONFIG')
|
|
82
|
+
for flag in (require_explicit_same_site, require_host_only, capture_headers):
|
|
83
|
+
if not isinstance(flag, bool):
|
|
84
|
+
raise CookieGuardError('CONFIG')
|
|
85
|
+
if require_header_evidence is None:
|
|
86
|
+
require_header_evidence = capture_headers
|
|
87
|
+
if not isinstance(require_header_evidence, bool) or (require_header_evidence and not capture_headers):
|
|
88
|
+
raise CookieGuardError('CONFIG')
|
|
89
|
+
if isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, (int, float)) or not 0.1 <= timeout_seconds <= 600:
|
|
90
|
+
raise CookieGuardError('CONFIG')
|
|
91
|
+
if isinstance(max_lifetime_days, bool) or not isinstance(max_lifetime_days, (int, float)):
|
|
92
|
+
raise CookieGuardError('CONFIG')
|
|
93
|
+
evidence = []
|
|
94
|
+
for entry in auth_evidence or ():
|
|
95
|
+
if not isinstance(entry, Mapping):
|
|
96
|
+
raise CookieGuardError('CONFIG')
|
|
97
|
+
evidence.append(dict(entry))
|
|
98
|
+
# Semantic validation (names, SameSite values, limits, evidence) is done by
|
|
99
|
+
# the CookieGuard core when the audit starts; unknown keys are rejected.
|
|
100
|
+
self.policy: dict[str, Any] = {
|
|
101
|
+
'authCookies': _names(auth_cookies), 'failOn': fail_on, 'allowedSameSite': list(allowed_same_site),
|
|
102
|
+
'requireExplicitSameSite': require_explicit_same_site, 'requireHostOnly': require_host_only,
|
|
103
|
+
'maxLifetimeDays': max_lifetime_days, 'authEvidence': evidence,
|
|
104
|
+
}
|
|
105
|
+
self.capture_headers = capture_headers
|
|
106
|
+
self.require_header_evidence = require_header_evidence
|
|
107
|
+
self._timeout_ms = int(timeout_seconds * 1000)
|
|
108
|
+
self._node_path = node_path
|
|
109
|
+
self._state = 'new'
|
|
110
|
+
self._session: Any = None
|
|
111
|
+
self._engine: LocalEngine | None = None
|
|
112
|
+
self._collector: CdpEvidenceCollector | None = None
|
|
113
|
+
self._lock = asyncio.Lock()
|
|
114
|
+
self._agent_checks = 0
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def state(self) -> str:
|
|
118
|
+
"""``new``, ``running``, ``finished``, or ``closed``."""
|
|
119
|
+
return self._state
|
|
120
|
+
|
|
121
|
+
def browser_allowed_domains(self) -> list[str]:
|
|
122
|
+
"""Patterns for ``BrowserProfile(allowed_domains=...)`` that keep the agent on audited hosts.
|
|
123
|
+
|
|
124
|
+
Browser Use matches scheme and host; it does not distinguish ports.
|
|
125
|
+
"""
|
|
126
|
+
result: list[str] = []
|
|
127
|
+
for origin in self.origins:
|
|
128
|
+
scheme, rest = origin.split('://', 1)
|
|
129
|
+
host = rest.rsplit(':', 1)[0] if not rest.startswith('[') else rest.split(']', 1)[0] + ']'
|
|
130
|
+
pattern = f'{scheme}://{host}'
|
|
131
|
+
if pattern not in result:
|
|
132
|
+
result.append(pattern)
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
# ----- Browser Use tool ------------------------------------------------
|
|
136
|
+
|
|
137
|
+
def register_tools(self, tools: Any, *, action_name: str = DEFAULT_ACTION_NAME) -> None:
|
|
138
|
+
"""Register a read-only ``cookieguard_audit`` action on a Browser Use ``Tools`` instance."""
|
|
139
|
+
from browser_use import ActionResult
|
|
140
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
141
|
+
|
|
142
|
+
if not isinstance(action_name, str) or not action_name.isidentifier():
|
|
143
|
+
raise CookieGuardError('CONFIG')
|
|
144
|
+
audit = self
|
|
145
|
+
|
|
146
|
+
class CookieGuardAuditParams(BaseModel):
|
|
147
|
+
# Same shape as Browser Use's NoParamsAction; any note is ignored.
|
|
148
|
+
model_config = ConfigDict(extra='ignore')
|
|
149
|
+
note: str | None = Field(None, description='Optional; ignored by CookieGuard')
|
|
150
|
+
|
|
151
|
+
# Deliberately unannotated: Browser Use type-checks injected parameters
|
|
152
|
+
# against real classes, and this module uses postponed annotations.
|
|
153
|
+
async def action(params, browser_session):
|
|
154
|
+
text = await audit._agent_check(browser_session)
|
|
155
|
+
return ActionResult(extracted_content=text, long_term_memory=text.split('\n', 1)[0])
|
|
156
|
+
|
|
157
|
+
action.__name__ = action_name
|
|
158
|
+
action.__qualname__ = action_name
|
|
159
|
+
tools.registry.action(ACTION_DESCRIPTION, param_model=CookieGuardAuditParams)(action)
|
|
160
|
+
|
|
161
|
+
async def _agent_check(self, browser_session: Any) -> str:
|
|
162
|
+
# Agent-visible path: fixed text only, never exceptions or values.
|
|
163
|
+
if self._state != 'running':
|
|
164
|
+
return 'CookieGuard audit is not active. The developer must start it before the agent runs.'
|
|
165
|
+
if browser_session is not self._session:
|
|
166
|
+
return 'CookieGuard is attached to a different browser session; no audit was performed.'
|
|
167
|
+
self._agent_checks += 1
|
|
168
|
+
try:
|
|
169
|
+
result = await self.check()
|
|
170
|
+
except CookieGuardError as error:
|
|
171
|
+
return f'CookieGuard provisional check could not complete ({error.code}). The final audit is unaffected.'
|
|
172
|
+
except Exception:
|
|
173
|
+
return 'CookieGuard provisional check could not complete. The final audit is unaffected.'
|
|
174
|
+
return result.agent_summary()
|
|
175
|
+
|
|
176
|
+
# ----- lifecycle -------------------------------------------------------
|
|
177
|
+
|
|
178
|
+
async def start(self, browser_session: Any) -> None:
|
|
179
|
+
"""Begin passive evidence collection on a started BrowserSession. Call before login."""
|
|
180
|
+
await _sanitized(self._start(browser_session), 'SESSION')
|
|
181
|
+
|
|
182
|
+
async def _start(self, browser_session: Any) -> None:
|
|
183
|
+
async with self._lock:
|
|
184
|
+
if self._state != 'new':
|
|
185
|
+
raise CookieGuardError('STATE')
|
|
186
|
+
profile = getattr(browser_session, 'browser_profile', None)
|
|
187
|
+
if profile is None or getattr(profile, 'keep_alive', None) is not True:
|
|
188
|
+
raise CookieGuardError('KEEP_ALIVE')
|
|
189
|
+
try:
|
|
190
|
+
root = browser_session.cdp_client
|
|
191
|
+
except Exception:
|
|
192
|
+
root = None
|
|
193
|
+
if root is None or not getattr(root, 'url', None):
|
|
194
|
+
raise CookieGuardError('SESSION')
|
|
195
|
+
engine = LocalEngine(self._node_path)
|
|
196
|
+
collector = CdpEvidenceCollector(self.origins, engine, capture_headers=self.capture_headers)
|
|
197
|
+
error: CookieGuardError | None = None
|
|
198
|
+
try:
|
|
199
|
+
await engine.start({'origins': list(self.origins), 'policy': self.policy, 'timeoutMs': self._timeout_ms,
|
|
200
|
+
'requireHeaderEvidence': self.require_header_evidence})
|
|
201
|
+
except CookieGuardError as caught:
|
|
202
|
+
error = CookieGuardError(caught.code)
|
|
203
|
+
except Exception:
|
|
204
|
+
error = CookieGuardError('ENGINE')
|
|
205
|
+
if error is None:
|
|
206
|
+
try:
|
|
207
|
+
await collector.start(root)
|
|
208
|
+
except Exception:
|
|
209
|
+
error = CookieGuardError('SESSION')
|
|
210
|
+
if error is not None:
|
|
211
|
+
await collector.stop()
|
|
212
|
+
await engine.close()
|
|
213
|
+
self._state = 'closed'
|
|
214
|
+
raise error
|
|
215
|
+
self._session, self._engine, self._collector = browser_session, engine, collector
|
|
216
|
+
self._state = 'running'
|
|
217
|
+
|
|
218
|
+
async def check(self) -> AuditResult:
|
|
219
|
+
"""Provisional, store-only evaluation while collection continues. Never satisfies policy."""
|
|
220
|
+
return await _sanitized(self._check(), 'COLLECT')
|
|
221
|
+
|
|
222
|
+
async def _check(self) -> AuditResult:
|
|
223
|
+
async with self._lock:
|
|
224
|
+
if self._state != 'running' or not self._collector or not self._engine:
|
|
225
|
+
raise CookieGuardError('STATE')
|
|
226
|
+
cookies, values = await self._collector.store_cookies()
|
|
227
|
+
reply = await self._engine.snapshot(cookies)
|
|
228
|
+
del cookies
|
|
229
|
+
result = AuditResult.from_engine(reply, provisional=True,
|
|
230
|
+
collection={**self._collector.stats(), 'evidence': 'browser-store-only'})
|
|
231
|
+
return guard_values(result, values)
|
|
232
|
+
|
|
233
|
+
async def finish(self, *, baseline: Mapping[str, Any] | None = None) -> AuditResult:
|
|
234
|
+
"""Stop collection and return the final CookieGuard result. Enforce it with ``assert_policy()``."""
|
|
235
|
+
return await _sanitized(self._finish(baseline), 'COLLECT')
|
|
236
|
+
|
|
237
|
+
async def _finish(self, baseline: Mapping[str, Any] | None) -> AuditResult:
|
|
238
|
+
async with self._lock:
|
|
239
|
+
if self._state != 'running' or not self._collector or not self._engine:
|
|
240
|
+
raise CookieGuardError('STATE')
|
|
241
|
+
self._state = 'finished'
|
|
242
|
+
collector, engine = self._collector, self._engine
|
|
243
|
+
try:
|
|
244
|
+
healthy = await collector.healthy()
|
|
245
|
+
collector.settle()
|
|
246
|
+
cookies, values = await collector.store_cookies() if healthy else (None, [])
|
|
247
|
+
reasons = set(collector.reasons)
|
|
248
|
+
if not healthy:
|
|
249
|
+
reasons.add('observer-disconnected')
|
|
250
|
+
if cookies is None:
|
|
251
|
+
reasons.add('cookie-store-unavailable')
|
|
252
|
+
stats = collector.stats()
|
|
253
|
+
await collector.stop()
|
|
254
|
+
if reasons:
|
|
255
|
+
await engine.mark_incomplete()
|
|
256
|
+
reply = await engine.finish(cookies, None if baseline is None else dict(baseline))
|
|
257
|
+
del cookies
|
|
258
|
+
missing = reply.get('headerEvidenceMissing') or []
|
|
259
|
+
if missing:
|
|
260
|
+
reasons.add('auth-cookie-declaration-not-observed')
|
|
261
|
+
result = AuditResult.from_engine(reply, provisional=False, collection={
|
|
262
|
+
**stats, 'evidence': 'server-declarations-and-browser-store' if self.capture_headers else 'browser-store-only',
|
|
263
|
+
'requireHeaderEvidence': self.require_header_evidence, 'incompleteReasons': sorted(reasons),
|
|
264
|
+
'agentChecks': self._agent_checks})
|
|
265
|
+
return guard_values(result, values)
|
|
266
|
+
finally:
|
|
267
|
+
await self._release()
|
|
268
|
+
|
|
269
|
+
async def close(self) -> None:
|
|
270
|
+
"""Stop collection without a result. Safe to call repeatedly."""
|
|
271
|
+
async with self._lock:
|
|
272
|
+
await self._release()
|
|
273
|
+
if self._state != 'finished':
|
|
274
|
+
self._state = 'closed'
|
|
275
|
+
|
|
276
|
+
async def _release(self) -> None:
|
|
277
|
+
collector, engine = self._collector, self._engine
|
|
278
|
+
self._collector = self._engine = None
|
|
279
|
+
self._session = None
|
|
280
|
+
if collector is not None:
|
|
281
|
+
await collector.stop()
|
|
282
|
+
if engine is not None:
|
|
283
|
+
await engine.close()
|
|
284
|
+
|
|
285
|
+
async def __aenter__(self) -> CookieGuardAudit:
|
|
286
|
+
return self
|
|
287
|
+
|
|
288
|
+
async def __aexit__(self, *exc_info: object) -> None:
|
|
289
|
+
await self.close()
|
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"""Passive CDP evidence collection on a dedicated observer connection.
|
|
2
|
+
|
|
3
|
+
cdp-use keeps one handler per CDP event per client, so registering on Browser
|
|
4
|
+
Use's own client would replace (or be replaced by) its watchdog handlers.
|
|
5
|
+
Like Browser Use's own toolset integrations, CookieGuard opens a second CDP
|
|
6
|
+
client to the same browser and attaches its own sessions to page targets.
|
|
7
|
+
|
|
8
|
+
Server declarations come from ``Network.responseReceivedExtraInfo`` raw
|
|
9
|
+
headers, which carry no URL. Each one is attributed to its redirect hop by
|
|
10
|
+
order and verified by status code; unattributable headers are discarded and
|
|
11
|
+
never forwarded. Browser state comes from ``Storage.getCookies``.
|
|
12
|
+
|
|
13
|
+
Raw cookie values exist only transiently in this process: Set-Cookie lines
|
|
14
|
+
for authorized origins are forwarded to the local engine immediately, other
|
|
15
|
+
origins' lines are dropped, and store values are sent once and released.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import logging
|
|
22
|
+
from collections import OrderedDict
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from ._engine import LocalEngine
|
|
27
|
+
from ._origins import cookie_matches_origins, origin_of
|
|
28
|
+
|
|
29
|
+
_logger = logging.getLogger('cookieguard_browser_use')
|
|
30
|
+
|
|
31
|
+
MAX_TRACKED_REQUESTS = 4096
|
|
32
|
+
MAX_FORWARDED_RESPONSES = 1000
|
|
33
|
+
MAX_LINES_PER_RESPONSE = 200
|
|
34
|
+
MAX_HEADER_LENGTH = 16384
|
|
35
|
+
MAX_STORE_COOKIES = 1000
|
|
36
|
+
OBSERVED_TARGET_TYPES = ('page', 'iframe')
|
|
37
|
+
CDP_TIMEOUT = 8.0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class _Hop:
|
|
42
|
+
url: str
|
|
43
|
+
status: int | None = None
|
|
44
|
+
cached: bool = False
|
|
45
|
+
extra_status: int | None = None # Status of the ExtraInfo attributed to this hop.
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class _Request:
|
|
50
|
+
hops: list[_Hop] = field(default_factory=list)
|
|
51
|
+
attributed: int = 0
|
|
52
|
+
pending: list[tuple[int | None, list[str]]] = field(default_factory=list)
|
|
53
|
+
had_lines: bool = False
|
|
54
|
+
mismatch: bool = False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _from_cache(response: dict[str, Any]) -> bool:
|
|
58
|
+
return bool(response.get('fromDiskCache') or response.get('fromPrefetchCache') or response.get('fromServiceWorker')
|
|
59
|
+
or response.get('fromEarlyHints'))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _set_cookie_lines(headers: Any) -> list[str]:
|
|
63
|
+
lines: list[str] = []
|
|
64
|
+
if isinstance(headers, dict):
|
|
65
|
+
for name, value in headers.items():
|
|
66
|
+
if isinstance(name, str) and name.lower() == 'set-cookie' and isinstance(value, str):
|
|
67
|
+
lines.extend(line for line in value.split('\n') if line.strip())
|
|
68
|
+
return lines
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class CdpEvidenceCollector:
|
|
72
|
+
def __init__(self, origins: tuple[str, ...], engine: LocalEngine, *, capture_headers: bool) -> None:
|
|
73
|
+
self._origins = origins
|
|
74
|
+
self._engine = engine
|
|
75
|
+
self._capture_headers = capture_headers
|
|
76
|
+
self._client: Any = None
|
|
77
|
+
self._targets: dict[str, str] = {}
|
|
78
|
+
self._attach_tasks: dict[str, asyncio.Task] = {}
|
|
79
|
+
self._sessions: set[str] = set()
|
|
80
|
+
self._contexts: set[str] = set()
|
|
81
|
+
self._requests: OrderedDict[tuple[str | None, str], _Request] = OrderedDict()
|
|
82
|
+
self._tasks: set[asyncio.Task] = set()
|
|
83
|
+
self._reasons: set[str] = set()
|
|
84
|
+
self._header_names: set[str] = set()
|
|
85
|
+
self._forwarded = 0
|
|
86
|
+
self._unattributed = 0
|
|
87
|
+
self._attach_failures = 0
|
|
88
|
+
self._attached_total = 0
|
|
89
|
+
self._stopping = False
|
|
90
|
+
|
|
91
|
+
# ----- lifecycle -------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
async def start(self, root_client: Any) -> None:
|
|
94
|
+
from cdp_use import CDPClient
|
|
95
|
+
|
|
96
|
+
self._client = CDPClient(root_client.url, additional_headers=getattr(root_client, 'additional_headers', None))
|
|
97
|
+
await asyncio.wait_for(self._client.start(), CDP_TIMEOUT)
|
|
98
|
+
register = self._client.register
|
|
99
|
+
register.Target.targetCreated(self._guard(self._on_target_created))
|
|
100
|
+
register.Target.detachedFromTarget(self._guard(self._on_detached))
|
|
101
|
+
if self._capture_headers:
|
|
102
|
+
register.Network.requestWillBeSent(self._guard(self._on_request))
|
|
103
|
+
register.Network.requestServedFromCache(self._guard(self._on_served_from_cache))
|
|
104
|
+
register.Network.responseReceived(self._guard(self._on_response))
|
|
105
|
+
register.Network.responseReceivedExtraInfo(self._guard(self._on_extra_info))
|
|
106
|
+
register.Network.loadingFinished(self._guard(self._on_loading_done))
|
|
107
|
+
register.Network.loadingFailed(self._guard(self._on_loading_done))
|
|
108
|
+
await asyncio.wait_for(self._client.send.Target.setDiscoverTargets(params={'discover': True}), CDP_TIMEOUT)
|
|
109
|
+
targets = await asyncio.wait_for(self._client.send.Target.getTargets(), CDP_TIMEOUT)
|
|
110
|
+
pending = [self._ensure_attached(info) for info in targets.get('targetInfos', [])
|
|
111
|
+
if info.get('type') in OBSERVED_TARGET_TYPES]
|
|
112
|
+
# targetCreated may already have started some of these attaches; await all.
|
|
113
|
+
await asyncio.gather(*(task for task in pending if task is not None), return_exceptions=True)
|
|
114
|
+
if not self._contexts:
|
|
115
|
+
raise RuntimeError('no page target')
|
|
116
|
+
|
|
117
|
+
async def stop(self) -> None:
|
|
118
|
+
self._stopping = True
|
|
119
|
+
for task in list(self._tasks):
|
|
120
|
+
task.cancel()
|
|
121
|
+
for task in list(self._tasks):
|
|
122
|
+
try:
|
|
123
|
+
await task
|
|
124
|
+
except BaseException:
|
|
125
|
+
pass
|
|
126
|
+
self._tasks.clear()
|
|
127
|
+
client, self._client = self._client, None
|
|
128
|
+
if client is not None:
|
|
129
|
+
try:
|
|
130
|
+
await asyncio.wait_for(client.stop(), CDP_TIMEOUT)
|
|
131
|
+
except BaseException:
|
|
132
|
+
pass
|
|
133
|
+
self._discard_buffers()
|
|
134
|
+
|
|
135
|
+
def _discard_buffers(self) -> None:
|
|
136
|
+
for request in self._requests.values():
|
|
137
|
+
request.pending.clear()
|
|
138
|
+
self._requests.clear()
|
|
139
|
+
|
|
140
|
+
def _incomplete(self, reason: str) -> None:
|
|
141
|
+
self._reasons.add(reason)
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def reasons(self) -> tuple[str, ...]:
|
|
145
|
+
return tuple(sorted(self._reasons))
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def header_cookie_names(self) -> frozenset[str]:
|
|
149
|
+
return frozenset(self._header_names)
|
|
150
|
+
|
|
151
|
+
def stats(self) -> dict[str, Any]:
|
|
152
|
+
return {'headerCapture': 'enabled' if self._capture_headers else 'disabled',
|
|
153
|
+
'targetsObserved': self._attached_total, 'setCookieResponsesForwarded': self._forwarded,
|
|
154
|
+
'unattributedSetCookieResponses': self._unattributed, 'attachFailures': self._attach_failures}
|
|
155
|
+
|
|
156
|
+
def _guard(self, handler):
|
|
157
|
+
# cdp-use logs handler exceptions with their text. Never let one escape.
|
|
158
|
+
def wrapped(event, session_id=None):
|
|
159
|
+
try:
|
|
160
|
+
result = handler(event, session_id)
|
|
161
|
+
if asyncio.iscoroutine(result):
|
|
162
|
+
return self._swallow(result)
|
|
163
|
+
except Exception:
|
|
164
|
+
self._incomplete('collector-error')
|
|
165
|
+
return None
|
|
166
|
+
return wrapped
|
|
167
|
+
|
|
168
|
+
async def _swallow(self, coroutine) -> None:
|
|
169
|
+
try:
|
|
170
|
+
await coroutine
|
|
171
|
+
except Exception:
|
|
172
|
+
self._incomplete('collector-error')
|
|
173
|
+
|
|
174
|
+
# ----- targets ---------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
def _on_target_created(self, event, session_id=None) -> None:
|
|
177
|
+
info = event.get('targetInfo', {})
|
|
178
|
+
if info.get('type') in OBSERVED_TARGET_TYPES:
|
|
179
|
+
self._ensure_attached(info)
|
|
180
|
+
|
|
181
|
+
def _ensure_attached(self, info: dict[str, Any]) -> asyncio.Task | None:
|
|
182
|
+
"""One attach task per target, shared by discovery events and the initial scan."""
|
|
183
|
+
target_id = info.get('targetId')
|
|
184
|
+
if not isinstance(target_id, str) or self._stopping or self._client is None:
|
|
185
|
+
return None
|
|
186
|
+
task = self._attach_tasks.get(target_id)
|
|
187
|
+
if task is None:
|
|
188
|
+
task = asyncio.create_task(self._swallow(self._attach(target_id, info)))
|
|
189
|
+
self._attach_tasks[target_id] = task
|
|
190
|
+
self._tasks.add(task)
|
|
191
|
+
task.add_done_callback(self._tasks.discard)
|
|
192
|
+
return task
|
|
193
|
+
|
|
194
|
+
async def _attach(self, target_id: str, info: dict[str, Any]) -> None:
|
|
195
|
+
try:
|
|
196
|
+
result = await asyncio.wait_for(
|
|
197
|
+
self._client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True}), CDP_TIMEOUT)
|
|
198
|
+
session_id = result['sessionId']
|
|
199
|
+
if self._capture_headers:
|
|
200
|
+
await asyncio.wait_for(self._client.send.Network.enable(session_id=session_id), CDP_TIMEOUT)
|
|
201
|
+
except Exception:
|
|
202
|
+
# Usually a target that closed before attachment. Missed headers can
|
|
203
|
+
# only remove declaration evidence; store evidence is read at finish.
|
|
204
|
+
self._attach_failures += 1
|
|
205
|
+
return
|
|
206
|
+
self._targets[target_id] = session_id
|
|
207
|
+
self._sessions.add(session_id)
|
|
208
|
+
self._attached_total += 1
|
|
209
|
+
if info.get('type') == 'page' and isinstance(info.get('browserContextId'), str):
|
|
210
|
+
self._contexts.add(info['browserContextId'])
|
|
211
|
+
|
|
212
|
+
def _on_detached(self, event, session_id=None) -> None:
|
|
213
|
+
self._sessions.discard(event.get('sessionId'))
|
|
214
|
+
|
|
215
|
+
# ----- network: Set-Cookie attribution ---------------------------------
|
|
216
|
+
|
|
217
|
+
def _request_for(self, key, create: bool) -> _Request | None:
|
|
218
|
+
request = self._requests.get(key)
|
|
219
|
+
if request is None and create:
|
|
220
|
+
request = _Request()
|
|
221
|
+
self._requests[key] = request
|
|
222
|
+
while len(self._requests) > MAX_TRACKED_REQUESTS:
|
|
223
|
+
_, evicted = self._requests.popitem(last=False)
|
|
224
|
+
if evicted.pending:
|
|
225
|
+
self._drop_unattributed(evicted)
|
|
226
|
+
return request
|
|
227
|
+
|
|
228
|
+
def _authorized(self, url: str) -> bool:
|
|
229
|
+
return origin_of(url) in self._origins
|
|
230
|
+
|
|
231
|
+
async def _on_request(self, event, session_id=None) -> None:
|
|
232
|
+
key = (session_id, event.get('requestId'))
|
|
233
|
+
url = event.get('request', {}).get('url', '')
|
|
234
|
+
redirect = event.get('redirectResponse')
|
|
235
|
+
request = self._request_for(key, create=True)
|
|
236
|
+
if isinstance(redirect, dict):
|
|
237
|
+
if not request.hops:
|
|
238
|
+
# Attached mid-flight: the redirect response names the earlier hop.
|
|
239
|
+
request.hops.append(_Hop(url=redirect.get('url', '') if isinstance(redirect.get('url'), str) else ''))
|
|
240
|
+
previous = request.hops[-1]
|
|
241
|
+
previous.cached = previous.cached or _from_cache(redirect)
|
|
242
|
+
self._set_status(request, previous, redirect.get('status'))
|
|
243
|
+
if redirect.get('fromServiceWorker') and self._authorized(previous.url):
|
|
244
|
+
self._incomplete('service-worker-response')
|
|
245
|
+
elif request.hops:
|
|
246
|
+
self._drop_unattributed(request)
|
|
247
|
+
request.hops.clear()
|
|
248
|
+
request.attributed = 0
|
|
249
|
+
request.hops.append(_Hop(url=url if isinstance(url, str) else ''))
|
|
250
|
+
await self._resolve(request)
|
|
251
|
+
|
|
252
|
+
def _on_served_from_cache(self, event, session_id=None) -> None:
|
|
253
|
+
request = self._request_for((session_id, event.get('requestId')), create=False)
|
|
254
|
+
if request and request.hops:
|
|
255
|
+
request.hops[-1].cached = True
|
|
256
|
+
|
|
257
|
+
async def _on_response(self, event, session_id=None) -> None:
|
|
258
|
+
request = self._request_for((session_id, event.get('requestId')), create=False)
|
|
259
|
+
response = event.get('response', {})
|
|
260
|
+
url = response.get('url', '') if isinstance(response.get('url'), str) else ''
|
|
261
|
+
if response.get('fromServiceWorker') and self._authorized(url):
|
|
262
|
+
self._incomplete('service-worker-response')
|
|
263
|
+
if request is None:
|
|
264
|
+
return
|
|
265
|
+
if not request.hops:
|
|
266
|
+
if len(request.pending) > 1:
|
|
267
|
+
# Missed earlier hops cannot be ordered safely.
|
|
268
|
+
self._drop_unattributed(request)
|
|
269
|
+
return
|
|
270
|
+
request.hops.append(_Hop(url=url))
|
|
271
|
+
hop = request.hops[-1]
|
|
272
|
+
hop.cached = hop.cached or _from_cache(response)
|
|
273
|
+
self._set_status(request, hop, response.get('status'))
|
|
274
|
+
await self._resolve(request)
|
|
275
|
+
|
|
276
|
+
async def _on_extra_info(self, event, session_id=None) -> None:
|
|
277
|
+
request = self._request_for((session_id, event.get('requestId')), create=True)
|
|
278
|
+
status = event.get('statusCode')
|
|
279
|
+
lines = _set_cookie_lines(event.get('headers'))
|
|
280
|
+
request.had_lines = request.had_lines or bool(lines)
|
|
281
|
+
# Every ExtraInfo is queued, with or without Set-Cookie, to keep hop order.
|
|
282
|
+
request.pending.append((status if isinstance(status, int) else None, lines))
|
|
283
|
+
if len(request.pending) > MAX_LINES_PER_RESPONSE:
|
|
284
|
+
self._drop_unattributed(request)
|
|
285
|
+
await self._resolve(request)
|
|
286
|
+
|
|
287
|
+
def _on_loading_done(self, event, session_id=None) -> None:
|
|
288
|
+
key = (session_id, event.get('requestId'))
|
|
289
|
+
if key in self._requests:
|
|
290
|
+
self._requests.move_to_end(key)
|
|
291
|
+
|
|
292
|
+
def _set_status(self, request: _Request, hop: _Hop, status: Any) -> None:
|
|
293
|
+
if isinstance(status, int):
|
|
294
|
+
hop.status = status
|
|
295
|
+
if hop.extra_status is not None and hop.extra_status != status:
|
|
296
|
+
request.mismatch = True
|
|
297
|
+
self._check_mismatch(request)
|
|
298
|
+
|
|
299
|
+
def _check_mismatch(self, request: _Request) -> None:
|
|
300
|
+
# Order-based attribution is wrong for this request. That only matters if
|
|
301
|
+
# it carried Set-Cookie and touched an authorized origin.
|
|
302
|
+
if request.mismatch and request.had_lines and any(self._authorized(hop.url) for hop in request.hops):
|
|
303
|
+
self._incomplete('header-attribution-mismatch')
|
|
304
|
+
|
|
305
|
+
async def _resolve(self, request: _Request) -> None:
|
|
306
|
+
network_hops = [hop for hop in request.hops if not hop.cached]
|
|
307
|
+
while request.pending and request.attributed < len(network_hops):
|
|
308
|
+
hop = network_hops[request.attributed]
|
|
309
|
+
status, lines = request.pending.pop(0)
|
|
310
|
+
request.attributed += 1
|
|
311
|
+
hop.extra_status = status
|
|
312
|
+
if hop.status is not None and status is not None and hop.status != status:
|
|
313
|
+
request.mismatch = True
|
|
314
|
+
self._check_mismatch(request)
|
|
315
|
+
if request.mismatch:
|
|
316
|
+
continue
|
|
317
|
+
if lines and self._authorized(hop.url):
|
|
318
|
+
await self._forward(hop.url, lines)
|
|
319
|
+
|
|
320
|
+
def _drop_unattributed(self, request: _Request) -> None:
|
|
321
|
+
if any(lines for _, lines in request.pending):
|
|
322
|
+
self._unattributed += 1
|
|
323
|
+
if any(self._authorized(hop.url) for hop in request.hops):
|
|
324
|
+
self._incomplete('unattributed-set-cookie')
|
|
325
|
+
request.pending.clear()
|
|
326
|
+
|
|
327
|
+
async def _forward(self, url: str, lines: list[str]) -> None:
|
|
328
|
+
if self._stopping:
|
|
329
|
+
return
|
|
330
|
+
if self._forwarded >= MAX_FORWARDED_RESPONSES:
|
|
331
|
+
self._incomplete('response-limit')
|
|
332
|
+
return
|
|
333
|
+
if len(lines) > MAX_LINES_PER_RESPONSE or any(len(line) > MAX_HEADER_LENGTH for line in lines):
|
|
334
|
+
self._incomplete('oversized-set-cookie')
|
|
335
|
+
return
|
|
336
|
+
self._forwarded += 1
|
|
337
|
+
for line in lines:
|
|
338
|
+
name = line.split('=', 1)[0].strip() if '=' in line else ''
|
|
339
|
+
if name:
|
|
340
|
+
self._header_names.add(name[:256])
|
|
341
|
+
await self._engine.response(url, lines, False)
|
|
342
|
+
|
|
343
|
+
# ----- finish ----------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
async def healthy(self) -> bool:
|
|
346
|
+
"""A CDP round trip also guarantees that events sent before it were delivered."""
|
|
347
|
+
if self._client is None:
|
|
348
|
+
return False
|
|
349
|
+
try:
|
|
350
|
+
await asyncio.wait_for(self._client.send.Browser.getVersion(), CDP_TIMEOUT)
|
|
351
|
+
return True
|
|
352
|
+
except Exception:
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
def settle(self) -> None:
|
|
356
|
+
for request in self._requests.values():
|
|
357
|
+
self._drop_unattributed(request)
|
|
358
|
+
|
|
359
|
+
async def store_cookies(self) -> tuple[list[dict[str, Any]] | None, list[str]]:
|
|
360
|
+
"""Return engine-ready store cookies for authorized hosts, plus their values for output guarding."""
|
|
361
|
+
if self._client is None:
|
|
362
|
+
return None, []
|
|
363
|
+
cookies: list[dict[str, Any]] = []
|
|
364
|
+
failed = False
|
|
365
|
+
try:
|
|
366
|
+
raw: list[dict[str, Any]] = []
|
|
367
|
+
for context in sorted(self._contexts):
|
|
368
|
+
result = await asyncio.wait_for(
|
|
369
|
+
self._client.send.Storage.getCookies(params={'browserContextId': context}), CDP_TIMEOUT)
|
|
370
|
+
raw.extend(result.get('cookies', []))
|
|
371
|
+
if len(raw) > MAX_STORE_COOKIES * 4:
|
|
372
|
+
raise ValueError
|
|
373
|
+
for cookie in raw:
|
|
374
|
+
if not cookie_matches_origins(cookie.get('domain'), self._origins):
|
|
375
|
+
continue
|
|
376
|
+
same_site = cookie.get('sameSite')
|
|
377
|
+
cookies.append({'name': cookie.get('name'), 'value': cookie.get('value', ''), 'domain': cookie.get('domain'),
|
|
378
|
+
'path': cookie.get('path'), 'expires': cookie.get('expires', -1), 'httpOnly': cookie.get('httpOnly'),
|
|
379
|
+
'secure': cookie.get('secure'), **({'sameSite': same_site} if isinstance(same_site, str) else {}),
|
|
380
|
+
'partitioned': 'partitionKey' in cookie})
|
|
381
|
+
del raw
|
|
382
|
+
if len(cookies) > MAX_STORE_COOKIES:
|
|
383
|
+
raise ValueError
|
|
384
|
+
except Exception:
|
|
385
|
+
failed = True
|
|
386
|
+
if failed:
|
|
387
|
+
return None, []
|
|
388
|
+
return cookies, [c['value'] for c in cookies if isinstance(c.get('value'), str)]
|