krill-collector 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- krill_collector-0.1.2/PKG-INFO +89 -0
- krill_collector-0.1.2/README.md +79 -0
- krill_collector-0.1.2/pyproject.toml +23 -0
- krill_collector-0.1.2/src/krill_collector/__init__.py +22 -0
- krill_collector-0.1.2/src/krill_collector/_cli.py +109 -0
- krill_collector-0.1.2/src/krill_collector/_collector.py +290 -0
- krill_collector-0.1.2/src/krill_collector/_errors.py +9 -0
- krill_collector-0.1.2/src/krill_collector/_exchange_merge.py +48 -0
- krill_collector-0.1.2/src/krill_collector/_interceptor.py +142 -0
- krill_collector-0.1.2/src/krill_collector/_recorder.py +38 -0
- krill_collector-0.1.2/src/krill_collector/_resources/__init__.py +0 -0
- krill_collector-0.1.2/src/krill_collector/_resources/krill_event_capture.js +27 -0
- krill_collector-0.1.2/src/krill_collector/_resources/krill_request_capture.php +60 -0
- krill_collector-0.1.2/src/krill_collector/_server_recorder.py +246 -0
- krill_collector-0.1.2/src/krill_collector/py.typed +0 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: krill-collector
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Record browser workflow sessions into krill recordings
|
|
5
|
+
Author: anonymous
|
|
6
|
+
Requires-Dist: krill-core>=0.1.0,<0.2.0
|
|
7
|
+
Requires-Dist: playwright>=1.49
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# krill-collector
|
|
12
|
+
|
|
13
|
+
> A krill feeds by filtering everything that drifts past it — this package filters a live browser session, straining out the HTTP exchanges and user events that make up a workflow.
|
|
14
|
+
|
|
15
|
+
Record a browser workflow session, including server-side request capture, into a krill `WorkflowRecording`.
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
Requires Python 3.10+ and [uv](https://docs.astral.sh/uv/). Playwright needs a browser binary the first time:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
uv add krill-collector
|
|
23
|
+
uv run playwright install chromium
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
Record from the command line — drive the browser by hand, then close the tab to finish:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv run krill-collect http://target-app.local -o logicflow.json
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or from Python:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from krill_collector import WorkflowCollector, save_recording
|
|
38
|
+
|
|
39
|
+
collector = WorkflowCollector("http://target-app.local")
|
|
40
|
+
recording = collector.collect() # opens a browser; records until the tab closes
|
|
41
|
+
save_recording(recording, "logicflow.json")
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The recording is consumed by the companion [krill-fuzzer](../krill-fuzzer/README.md) package.
|
|
45
|
+
|
|
46
|
+
## API
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
browser session ──▶ intercept ──▶ record ──▶ WorkflowRecording ──▶ JSON
|
|
50
|
+
├── network (RequestInterceptor)
|
|
51
|
+
├── DOM events (EventRecorder)
|
|
52
|
+
└── server-side (ServerRequestRecorder)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### `krill_collector`
|
|
56
|
+
|
|
57
|
+
| Symbol | Type | Description |
|
|
58
|
+
|--------|------|-------------|
|
|
59
|
+
| `WorkflowCollector(target_url, server_project_roots)` | class | Orchestrates a recording session: `collect()` (sync), `async_collect()` |
|
|
60
|
+
| `ServerRequestRecorder(project_root, output_dir)` | class | Standalone server-side PHP request capture: `instrument()`, `deinstrument()`, `collect()` |
|
|
61
|
+
| `CollectorError` | Exception | Browser launch failure or zero exchanges captured; subclass of `KrillError` |
|
|
62
|
+
| `WorkflowRecording` | dataclass | The recording output: exchanges, events, target, timings (from `krill-core`) |
|
|
63
|
+
| `RecordedEvent` / `EventType` | recorded interaction + its kind (from `krill-core`) |
|
|
64
|
+
| `save_recording` | function | Write a recording to JSON (from `krill-core`) |
|
|
65
|
+
|
|
66
|
+
### `krill-collect`
|
|
67
|
+
|
|
68
|
+
Command-line recorder. `krill-collect TARGET_URL [-o OUTPUT] [-s SERVER_ROOT ...]` — records a browser session against `TARGET_URL`, writing JSON to `OUTPUT` (default `logicflow.json`). Repeat `-s DIR` to capture server-side requests from each project root.
|
|
69
|
+
|
|
70
|
+
## Documentation
|
|
71
|
+
|
|
72
|
+
- [Docs index](docs/index.md) — collect module design and implementation notes.
|
|
73
|
+
|
|
74
|
+
## Citation
|
|
75
|
+
|
|
76
|
+
If you use this project in academic work, please cite our paper. BibTeX (placeholder until the paper is released):
|
|
77
|
+
|
|
78
|
+
```bibtex
|
|
79
|
+
@misc{beluga_placeholder,
|
|
80
|
+
title = {{TODO: paper title — not yet released}},
|
|
81
|
+
author = {{TODO: authors}},
|
|
82
|
+
year = {{TODO}},
|
|
83
|
+
note = {Paper not yet released. Citation entry will be updated on publication.}
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
GPL-2.0 — see [LICENSE](../../LICENSE).
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# krill-collector
|
|
2
|
+
|
|
3
|
+
> A krill feeds by filtering everything that drifts past it — this package filters a live browser session, straining out the HTTP exchanges and user events that make up a workflow.
|
|
4
|
+
|
|
5
|
+
Record a browser workflow session, including server-side request capture, into a krill `WorkflowRecording`.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
Requires Python 3.10+ and [uv](https://docs.astral.sh/uv/). Playwright needs a browser binary the first time:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
uv add krill-collector
|
|
13
|
+
uv run playwright install chromium
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
Record from the command line — drive the browser by hand, then close the tab to finish:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
uv run krill-collect http://target-app.local -o logicflow.json
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Or from Python:
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from krill_collector import WorkflowCollector, save_recording
|
|
28
|
+
|
|
29
|
+
collector = WorkflowCollector("http://target-app.local")
|
|
30
|
+
recording = collector.collect() # opens a browser; records until the tab closes
|
|
31
|
+
save_recording(recording, "logicflow.json")
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The recording is consumed by the companion [krill-fuzzer](../krill-fuzzer/README.md) package.
|
|
35
|
+
|
|
36
|
+
## API
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
browser session ──▶ intercept ──▶ record ──▶ WorkflowRecording ──▶ JSON
|
|
40
|
+
├── network (RequestInterceptor)
|
|
41
|
+
├── DOM events (EventRecorder)
|
|
42
|
+
└── server-side (ServerRequestRecorder)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### `krill_collector`
|
|
46
|
+
|
|
47
|
+
| Symbol | Type | Description |
|
|
48
|
+
|--------|------|-------------|
|
|
49
|
+
| `WorkflowCollector(target_url, server_project_roots)` | class | Orchestrates a recording session: `collect()` (sync), `async_collect()` |
|
|
50
|
+
| `ServerRequestRecorder(project_root, output_dir)` | class | Standalone server-side PHP request capture: `instrument()`, `deinstrument()`, `collect()` |
|
|
51
|
+
| `CollectorError` | Exception | Browser launch failure or zero exchanges captured; subclass of `KrillError` |
|
|
52
|
+
| `WorkflowRecording` | dataclass | The recording output: exchanges, events, target, timings (from `krill-core`) |
|
|
53
|
+
| `RecordedEvent` / `EventType` | recorded interaction + its kind (from `krill-core`) |
|
|
54
|
+
| `save_recording` | function | Write a recording to JSON (from `krill-core`) |
|
|
55
|
+
|
|
56
|
+
### `krill-collect`
|
|
57
|
+
|
|
58
|
+
Command-line recorder. `krill-collect TARGET_URL [-o OUTPUT] [-s SERVER_ROOT ...]` — records a browser session against `TARGET_URL`, writing JSON to `OUTPUT` (default `logicflow.json`). Repeat `-s DIR` to capture server-side requests from each project root.
|
|
59
|
+
|
|
60
|
+
## Documentation
|
|
61
|
+
|
|
62
|
+
- [Docs index](docs/index.md) — collect module design and implementation notes.
|
|
63
|
+
|
|
64
|
+
## Citation
|
|
65
|
+
|
|
66
|
+
If you use this project in academic work, please cite our paper. BibTeX (placeholder until the paper is released):
|
|
67
|
+
|
|
68
|
+
```bibtex
|
|
69
|
+
@misc{beluga_placeholder,
|
|
70
|
+
title = {{TODO: paper title — not yet released}},
|
|
71
|
+
author = {{TODO: authors}},
|
|
72
|
+
year = {{TODO}},
|
|
73
|
+
note = {Paper not yet released. Citation entry will be updated on publication.}
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
GPL-2.0 — see [LICENSE](../../LICENSE).
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "krill-collector"
|
|
3
|
+
version = "0.1.2"
|
|
4
|
+
description = "Record browser workflow sessions into krill recordings"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "anonymous" }
|
|
8
|
+
]
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"krill-core>=0.1.0,<0.2.0",
|
|
12
|
+
"playwright>=1.49",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
krill-collect = "krill_collector._cli:main"
|
|
17
|
+
|
|
18
|
+
[tool.uv.sources]
|
|
19
|
+
krill-core = { workspace = true }
|
|
20
|
+
|
|
21
|
+
[build-system]
|
|
22
|
+
requires = ["uv_build>=0.11.26,<0.12"]
|
|
23
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Record browser interactions into a WorkflowRecording.
|
|
2
|
+
|
|
3
|
+
RequestInterceptor, EventRecorder, and ServerRequestRecorder are internal to
|
|
4
|
+
WorkflowCollector; ServerRequestRecorder is also exported for standalone
|
|
5
|
+
server-side recording. The recording data model and its JSON persistence are
|
|
6
|
+
re-exported from ``krill_core``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from krill_collector._collector import WorkflowCollector
|
|
10
|
+
from krill_collector._errors import CollectorError
|
|
11
|
+
from krill_collector._server_recorder import ServerRequestRecorder
|
|
12
|
+
from krill_core import EventType, RecordedEvent, WorkflowRecording, save_recording
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"CollectorError",
|
|
16
|
+
"EventType",
|
|
17
|
+
"RecordedEvent",
|
|
18
|
+
"WorkflowRecording",
|
|
19
|
+
"WorkflowCollector",
|
|
20
|
+
"ServerRequestRecorder",
|
|
21
|
+
"save_recording",
|
|
22
|
+
]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""CLI entry point for ``krill-collect`` — record browser workflow sessions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from krill_collector._collector import WorkflowCollector
|
|
10
|
+
from krill_core import WorkflowRecording, save_recording
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(argv: list[str] | None = None) -> None:
|
|
14
|
+
"""Entry point for ``krill-collect``."""
|
|
15
|
+
parser = _build_parser()
|
|
16
|
+
args = parser.parse_args(argv)
|
|
17
|
+
_run(args)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
21
|
+
"""Construct the argument parser for the collect command."""
|
|
22
|
+
parser = argparse.ArgumentParser(
|
|
23
|
+
prog="krill-collect",
|
|
24
|
+
description="Record a browser workflow session and save as JSON.",
|
|
25
|
+
)
|
|
26
|
+
parser.add_argument("target_url", help="Base URL of the target application")
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"-o",
|
|
29
|
+
"--output",
|
|
30
|
+
default="logicflow.json",
|
|
31
|
+
help="Output file path (default: logicflow.json)",
|
|
32
|
+
)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"-s",
|
|
35
|
+
"--server-root",
|
|
36
|
+
action="append",
|
|
37
|
+
dest="server_roots",
|
|
38
|
+
metavar="DIR",
|
|
39
|
+
help="Server project root for server-side request capture (repeatable)",
|
|
40
|
+
)
|
|
41
|
+
return parser
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _run(args: argparse.Namespace) -> None:
|
|
45
|
+
"""Execute the collect command."""
|
|
46
|
+
server_roots = _resolve_server_roots(args.server_roots)
|
|
47
|
+
output_path = _resolve_output_path(args.output)
|
|
48
|
+
collector = WorkflowCollector(
|
|
49
|
+
target_url=args.target_url,
|
|
50
|
+
server_project_roots=server_roots or None,
|
|
51
|
+
)
|
|
52
|
+
_print_banner(args.target_url, server_roots, output_path)
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
recording = collector.collect()
|
|
56
|
+
except KeyboardInterrupt:
|
|
57
|
+
print("\nRecording cancelled.")
|
|
58
|
+
sys.exit(1)
|
|
59
|
+
|
|
60
|
+
_print_summary(recording)
|
|
61
|
+
save_recording(recording, output_path)
|
|
62
|
+
print(f"\nSaved to {output_path}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _resolve_server_roots(root_strs: list[str] | None) -> list[Path]:
|
|
66
|
+
"""Resolve and validate server-root arguments, exiting on a non-directory."""
|
|
67
|
+
server_roots: list[Path] = []
|
|
68
|
+
for root_str in root_strs or []:
|
|
69
|
+
root = Path(root_str).resolve()
|
|
70
|
+
if not root.is_dir():
|
|
71
|
+
print(f"Error: server root is not a directory: {root}", file=sys.stderr)
|
|
72
|
+
sys.exit(1)
|
|
73
|
+
server_roots.append(root)
|
|
74
|
+
return server_roots
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _resolve_output_path(output: str) -> Path:
|
|
78
|
+
"""Resolve the output path, exiting if its parent directory is missing."""
|
|
79
|
+
output_path = Path(output).resolve()
|
|
80
|
+
if not output_path.parent.is_dir():
|
|
81
|
+
print(
|
|
82
|
+
f"Error: output directory does not exist: {output_path.parent}",
|
|
83
|
+
file=sys.stderr,
|
|
84
|
+
)
|
|
85
|
+
sys.exit(1)
|
|
86
|
+
return output_path
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _print_banner(
|
|
90
|
+
target_url: str,
|
|
91
|
+
server_roots: list[Path],
|
|
92
|
+
output_path: Path,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""Print the pre-recording banner listing target, servers, and output."""
|
|
95
|
+
print(f"Target: {target_url}")
|
|
96
|
+
for root in server_roots:
|
|
97
|
+
print(f"Server: {root}")
|
|
98
|
+
print(f"Output: {output_path}")
|
|
99
|
+
print()
|
|
100
|
+
print("Launching browser... Complete the workflow, then close the tab to finish.")
|
|
101
|
+
print()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _print_summary(recording: WorkflowRecording) -> None:
|
|
105
|
+
"""Print post-recording capture statistics."""
|
|
106
|
+
print("Recording captured:")
|
|
107
|
+
print(f" Exchanges: {len(recording.exchanges)}")
|
|
108
|
+
print(f" Events: {len(recording.events)}")
|
|
109
|
+
print(f" Duration: {recording.ended_at - recording.started_at:.1f}s")
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""WorkflowCollector — orchestrates browser recording session via Playwright."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import importlib.resources
|
|
7
|
+
import logging
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import NoReturn
|
|
12
|
+
|
|
13
|
+
from playwright.async_api import BrowserContext, Cookie
|
|
14
|
+
from playwright.async_api import Error as PlaywrightError
|
|
15
|
+
from playwright.async_api import Request as AsyncRequest
|
|
16
|
+
from playwright.async_api import Route as AsyncRoute
|
|
17
|
+
from playwright.async_api import async_playwright
|
|
18
|
+
|
|
19
|
+
from krill_collector._errors import CollectorError
|
|
20
|
+
from krill_collector._exchange_merge import merge_exchanges
|
|
21
|
+
from krill_collector._interceptor import (
|
|
22
|
+
_STATIC_RESOURCE_TYPES,
|
|
23
|
+
CapturedRequest,
|
|
24
|
+
CapturedResponse,
|
|
25
|
+
RequestInterceptor,
|
|
26
|
+
)
|
|
27
|
+
from krill_collector._recorder import EventRecorder
|
|
28
|
+
from krill_collector._server_recorder import ServerRequestRecorder
|
|
29
|
+
from krill_core import EventType, HttpRequest, WorkflowRecording
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
_SET_COOKIE_HEADER = "set-cookie"
|
|
34
|
+
|
|
35
|
+
# Response headers describing a content or transfer coding that ``route.fetch``
|
|
36
|
+
# has already stripped from the body it returns. Storing them beside the decoded
|
|
37
|
+
# body would claim a coding the bytes no longer carry (model-recording.md:
|
|
38
|
+
# coding consistency).
|
|
39
|
+
_DECODED_CODING_HEADERS = frozenset(
|
|
40
|
+
{"content-encoding", "content-length", "transfer-encoding"}
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# Injected DOM-capture script filename in the _resources package-data directory.
|
|
44
|
+
_CAPTURE_SCRIPT_FILENAME = "krill_event_capture.js"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _load_capture_script() -> str:
|
|
48
|
+
"""Read the injected DOM-event capture script from package data."""
|
|
49
|
+
return (
|
|
50
|
+
importlib.resources.files("krill_collector._resources")
|
|
51
|
+
.joinpath(_CAPTURE_SCRIPT_FILENAME)
|
|
52
|
+
.read_text(encoding="utf-8")
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _serialize_set_cookie(cookies: list[Cookie]) -> str:
|
|
57
|
+
"""Serialize context cookies into one Set-Cookie header value."""
|
|
58
|
+
return "; ".join(f"{c['name']}={c['value']}" for c in cookies)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _decoded_response_headers(headers: dict[str, str]) -> dict[str, str]:
|
|
62
|
+
"""Drop headers describing a coding the decoded body no longer carries."""
|
|
63
|
+
return {
|
|
64
|
+
key: value
|
|
65
|
+
for key, value in headers.items()
|
|
66
|
+
if key.lower() not in _DECODED_CODING_HEADERS
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _raise_collector_error(exc: Exception) -> NoReturn:
|
|
71
|
+
"""Translate common Playwright errors into actionable CollectorError messages."""
|
|
72
|
+
msg = str(exc)
|
|
73
|
+
if "Executable doesn't exist" in msg:
|
|
74
|
+
raise CollectorError(
|
|
75
|
+
"Chromium browser not found. Run: playwright install chromium"
|
|
76
|
+
) from exc
|
|
77
|
+
raise CollectorError(f"Browser session failed: {exc}") from exc
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class WorkflowCollector:
|
|
81
|
+
"""Manages the browser-based workflow recording session, composing
|
|
82
|
+
RequestInterceptor and EventRecorder."""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
target_url: str,
|
|
87
|
+
server_project_roots: list[Path] | None = None,
|
|
88
|
+
) -> None:
|
|
89
|
+
if not target_url:
|
|
90
|
+
raise ValueError("target_url must not be empty")
|
|
91
|
+
self._target_url = target_url
|
|
92
|
+
self._interceptor = RequestInterceptor()
|
|
93
|
+
self._recorder = EventRecorder()
|
|
94
|
+
self._server_recorders: list[ServerRequestRecorder] = [
|
|
95
|
+
ServerRequestRecorder(root) for root in (server_project_roots or [])
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
def collect(self) -> WorkflowRecording:
|
|
99
|
+
"""Run a complete recording session (sync entry point).
|
|
100
|
+
|
|
101
|
+
Drives ``async_collect`` to completion via ``asyncio.run``. Launches a
|
|
102
|
+
visible Chromium browser, navigates to the target URL, and blocks until
|
|
103
|
+
the user closes the browser tab.
|
|
104
|
+
|
|
105
|
+
Raises:
|
|
106
|
+
CollectorError: Called inside a running event loop (e.g. Jupyter),
|
|
107
|
+
or the browser session failed.
|
|
108
|
+
"""
|
|
109
|
+
coro = self.async_collect()
|
|
110
|
+
try:
|
|
111
|
+
return asyncio.run(coro)
|
|
112
|
+
except RuntimeError as exc:
|
|
113
|
+
if "cannot be called from a running event loop" in str(exc):
|
|
114
|
+
coro.close()
|
|
115
|
+
raise CollectorError(
|
|
116
|
+
"Cannot use collect() inside an async event loop "
|
|
117
|
+
"(e.g. Jupyter). Use async_collect() instead."
|
|
118
|
+
) from exc
|
|
119
|
+
raise
|
|
120
|
+
|
|
121
|
+
async def async_collect(self) -> WorkflowRecording:
|
|
122
|
+
"""Run a complete recording session (async).
|
|
123
|
+
|
|
124
|
+
Use this instead of ``collect()`` when running inside an existing
|
|
125
|
+
asyncio event loop (e.g. Jupyter notebooks).
|
|
126
|
+
"""
|
|
127
|
+
self._instrument_servers()
|
|
128
|
+
started_at = time.time()
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
async with async_playwright() as pw:
|
|
132
|
+
browser = await pw.chromium.launch(headless=False)
|
|
133
|
+
# Block service workers so their requests take the network
|
|
134
|
+
# path and reach the context route handler.
|
|
135
|
+
context = await browser.new_context(service_workers="block")
|
|
136
|
+
await self._configure_capture(context)
|
|
137
|
+
|
|
138
|
+
page = await context.new_page()
|
|
139
|
+
await page.goto(self._target_url)
|
|
140
|
+
await page.wait_for_event("close", timeout=0)
|
|
141
|
+
await browser.close()
|
|
142
|
+
|
|
143
|
+
except PlaywrightError as exc:
|
|
144
|
+
_raise_collector_error(exc)
|
|
145
|
+
finally:
|
|
146
|
+
self._deinstrument_servers()
|
|
147
|
+
|
|
148
|
+
return self._build_recording(started_at)
|
|
149
|
+
|
|
150
|
+
async def _configure_capture(self, context: BrowserContext) -> None:
|
|
151
|
+
"""Register network and DOM capture at context scope.
|
|
152
|
+
|
|
153
|
+
Context-scope registration covers popups and newly-opened tabs, not
|
|
154
|
+
just the initial page; ``page.route`` misses a popup's first request.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
async def _route_handler(route: AsyncRoute) -> None:
|
|
158
|
+
await self._capture_route(route, context)
|
|
159
|
+
|
|
160
|
+
await context.route("**/*", _route_handler)
|
|
161
|
+
await context.expose_function("__krill_record_event", self._handle_dom_event)
|
|
162
|
+
await context.add_init_script(_load_capture_script())
|
|
163
|
+
|
|
164
|
+
async def _capture_route(self, route: AsyncRoute, context: BrowserContext) -> None:
|
|
165
|
+
"""Capture one route's response, fold in session cookies, then fulfill.
|
|
166
|
+
|
|
167
|
+
``route.fetch`` returns a decoded body, so headers naming the original
|
|
168
|
+
content or transfer coding are dropped before recording to keep body
|
|
169
|
+
and headers consistent. Static-asset resource types continue untouched.
|
|
170
|
+
On a Playwright error the attempted request is recorded as a
|
|
171
|
+
response-less exchange (status_code=0) — a failed request is fuzzing
|
|
172
|
+
signal, not noise — and the route continues so the page still loads.
|
|
173
|
+
"""
|
|
174
|
+
if route.request.resource_type in _STATIC_RESOURCE_TYPES:
|
|
175
|
+
await route.continue_()
|
|
176
|
+
return
|
|
177
|
+
try:
|
|
178
|
+
resp = await route.fetch()
|
|
179
|
+
headers = _decoded_response_headers(resp.headers)
|
|
180
|
+
cookies = await context.cookies(route.request.url)
|
|
181
|
+
if cookies:
|
|
182
|
+
headers[_SET_COOKIE_HEADER] = _serialize_set_cookie(cookies)
|
|
183
|
+
await self._record_response(
|
|
184
|
+
route.request,
|
|
185
|
+
CapturedResponse(
|
|
186
|
+
status_code=resp.status, headers=headers, body=await resp.body()
|
|
187
|
+
),
|
|
188
|
+
)
|
|
189
|
+
await route.fulfill(response=resp)
|
|
190
|
+
except PlaywrightError as exc:
|
|
191
|
+
logger.debug(
|
|
192
|
+
"Route fetch failed for %s %s: %s",
|
|
193
|
+
route.request.method,
|
|
194
|
+
route.request.url,
|
|
195
|
+
exc,
|
|
196
|
+
)
|
|
197
|
+
await self._record_response(
|
|
198
|
+
route.request, CapturedResponse(status_code=0, headers={}, body=b"")
|
|
199
|
+
)
|
|
200
|
+
await route.continue_()
|
|
201
|
+
|
|
202
|
+
async def _record_response(
|
|
203
|
+
self, request: AsyncRequest, response: CapturedResponse
|
|
204
|
+
) -> None:
|
|
205
|
+
"""Feed a captured request/response pair into the interceptor.
|
|
206
|
+
|
|
207
|
+
Resolves the full request headers via ``all_headers()`` so the Cookie
|
|
208
|
+
header — which Playwright omits from the synchronous ``headers``
|
|
209
|
+
property — is captured.
|
|
210
|
+
"""
|
|
211
|
+
self._interceptor.on_response(
|
|
212
|
+
CapturedRequest(
|
|
213
|
+
method=request.method,
|
|
214
|
+
url=request.url,
|
|
215
|
+
headers=await request.all_headers(),
|
|
216
|
+
body=request.post_data_buffer,
|
|
217
|
+
resource_type=request.resource_type,
|
|
218
|
+
),
|
|
219
|
+
response,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
def _build_recording(self, started_at: float) -> WorkflowRecording:
|
|
223
|
+
"""Validate and assemble the recording after a session ends.
|
|
224
|
+
|
|
225
|
+
Merges browser-side exchanges with server-side captured requests.
|
|
226
|
+
Server-side requests that were not seen by the browser (identified
|
|
227
|
+
by URL deduplication) are added as response-less exchanges.
|
|
228
|
+
"""
|
|
229
|
+
ended_at = time.time()
|
|
230
|
+
browser_exchanges = self._interceptor.get_exchanges()
|
|
231
|
+
server_requests = self._collect_server_requests()
|
|
232
|
+
|
|
233
|
+
exchanges = merge_exchanges(browser_exchanges, server_requests)
|
|
234
|
+
|
|
235
|
+
if not exchanges:
|
|
236
|
+
raise CollectorError("No HTTP exchanges captured during recording session")
|
|
237
|
+
|
|
238
|
+
return WorkflowRecording(
|
|
239
|
+
exchanges=exchanges,
|
|
240
|
+
events=self._recorder.get_events(),
|
|
241
|
+
target_url=self._target_url,
|
|
242
|
+
started_at=started_at,
|
|
243
|
+
ended_at=ended_at,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
def _handle_dom_event(
|
|
247
|
+
self, event_type_str: str, selector: str, value: str | None
|
|
248
|
+
) -> None:
|
|
249
|
+
"""Handle a DOM event forwarded from injected JavaScript."""
|
|
250
|
+
try:
|
|
251
|
+
event_type = EventType(event_type_str)
|
|
252
|
+
except ValueError:
|
|
253
|
+
return
|
|
254
|
+
self._recorder.on_event(event_type, selector, value)
|
|
255
|
+
|
|
256
|
+
def _instrument_servers(self) -> None:
|
|
257
|
+
"""Instrument all configured server project roots."""
|
|
258
|
+
self._apply_to_servers(ServerRequestRecorder.instrument, "instrumentation")
|
|
259
|
+
|
|
260
|
+
def _deinstrument_servers(self) -> None:
|
|
261
|
+
"""Deinstrument all configured server project roots."""
|
|
262
|
+
self._apply_to_servers(ServerRequestRecorder.deinstrument, "deinstrumentation")
|
|
263
|
+
|
|
264
|
+
def _apply_to_servers(
|
|
265
|
+
self,
|
|
266
|
+
operation: Callable[[ServerRequestRecorder], list[Path]],
|
|
267
|
+
action: str,
|
|
268
|
+
) -> None:
|
|
269
|
+
"""Apply one recorder operation to every server recorder.
|
|
270
|
+
|
|
271
|
+
A recorder that raises ``OSError`` is logged and skipped so one
|
|
272
|
+
unwritable project cannot abort the others.
|
|
273
|
+
"""
|
|
274
|
+
for recorder in self._server_recorders:
|
|
275
|
+
try:
|
|
276
|
+
operation(recorder)
|
|
277
|
+
except OSError as exc:
|
|
278
|
+
logger.warning(
|
|
279
|
+
"ServerRequestRecorder %s failed for %s: %s",
|
|
280
|
+
action,
|
|
281
|
+
recorder.project_root,
|
|
282
|
+
exc,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def _collect_server_requests(self) -> list[HttpRequest]:
|
|
286
|
+
"""Collect server-side captured requests from all recorders."""
|
|
287
|
+
all_requests: list[HttpRequest] = []
|
|
288
|
+
for recorder in self._server_recorders:
|
|
289
|
+
all_requests.extend(recorder.collect())
|
|
290
|
+
return all_requests
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Merge browser-side and server-side captured HTTP exchanges."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from krill_core import HttpExchange, HttpRequest, HttpResponse
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def merge_exchanges(
|
|
13
|
+
browser_exchanges: list[HttpExchange],
|
|
14
|
+
server_requests: list[HttpRequest],
|
|
15
|
+
) -> list[HttpExchange]:
|
|
16
|
+
"""Merge browser-side exchanges with server-side captured requests.
|
|
17
|
+
|
|
18
|
+
Server-side requests whose URL was not captured by the browser are
|
|
19
|
+
added as response-less exchanges (status_code=0, empty body).
|
|
20
|
+
The merged list preserves timestamp ordering.
|
|
21
|
+
"""
|
|
22
|
+
if not server_requests:
|
|
23
|
+
return browser_exchanges
|
|
24
|
+
|
|
25
|
+
browser_urls = {ex.request.url for ex in browser_exchanges}
|
|
26
|
+
server_only: list[HttpExchange] = []
|
|
27
|
+
|
|
28
|
+
for req in server_requests:
|
|
29
|
+
if req.url not in browser_urls:
|
|
30
|
+
server_only.append(
|
|
31
|
+
HttpExchange(
|
|
32
|
+
request=req,
|
|
33
|
+
response=HttpResponse(status_code=0),
|
|
34
|
+
timestamp=0.0,
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
if not server_only:
|
|
39
|
+
return browser_exchanges
|
|
40
|
+
|
|
41
|
+
logger.info(
|
|
42
|
+
"Server-side recording captured %d additional requests not seen by browser",
|
|
43
|
+
len(server_only),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
merged = list(browser_exchanges) + server_only
|
|
47
|
+
merged.sort(key=lambda ex: ex.timestamp)
|
|
48
|
+
return merged
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""RequestInterceptor — captures HTTP exchanges via Playwright response callback."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from krill_core import HttpExchange, HttpMethod, HttpRequest, HttpResponse
|
|
9
|
+
|
|
10
|
+
# Resource types the collector neither fetches nor captures: static assets with
|
|
11
|
+
# no bearing on workflow state. Owned here; imported by the collector.
|
|
12
|
+
_STATIC_RESOURCE_TYPES = frozenset({"image", "stylesheet", "font", "script", "media"})
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class CapturedRequest:
|
|
17
|
+
"""Raw request fields captured from the browser, before HttpRequest assembly."""
|
|
18
|
+
|
|
19
|
+
method: str
|
|
20
|
+
url: str
|
|
21
|
+
headers: dict[str, str]
|
|
22
|
+
body: bytes | None
|
|
23
|
+
resource_type: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class CapturedResponse:
|
|
28
|
+
"""Raw response fields captured from the browser."""
|
|
29
|
+
|
|
30
|
+
status_code: int
|
|
31
|
+
headers: dict[str, str]
|
|
32
|
+
body: bytes
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _parse_cookie_header(cookie_header: str) -> dict[str, str]:
|
|
36
|
+
"""Parse a Cookie request header into name=value pairs."""
|
|
37
|
+
cookies: dict[str, str] = {}
|
|
38
|
+
for morsel in cookie_header.split(";"):
|
|
39
|
+
name, _, value = morsel.partition("=")
|
|
40
|
+
if name.strip():
|
|
41
|
+
cookies[name.strip()] = value.strip()
|
|
42
|
+
return cookies
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _parse_set_cookie_names(headers: dict[str, str]) -> set[str]:
|
|
46
|
+
"""Extract cookie names from Set-Cookie response headers.
|
|
47
|
+
|
|
48
|
+
The collector serializes Set-Cookie as "; "-joined name=value pairs (see
|
|
49
|
+
WorkflowCollector's route handler), the same grammar as a Cookie request
|
|
50
|
+
header, so both are parsed identically.
|
|
51
|
+
"""
|
|
52
|
+
names: set[str] = set()
|
|
53
|
+
for key, value in headers.items():
|
|
54
|
+
if key.lower() == "set-cookie":
|
|
55
|
+
names.update(_parse_cookie_header(value))
|
|
56
|
+
return names
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _to_http_request(request: CapturedRequest, headers: dict[str, str]) -> HttpRequest:
|
|
60
|
+
"""Build an HttpRequest from a captured request and its cleaned headers."""
|
|
61
|
+
return HttpRequest(
|
|
62
|
+
method=HttpMethod(request.method),
|
|
63
|
+
url=request.url,
|
|
64
|
+
headers=headers,
|
|
65
|
+
body=request.body,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _to_http_response(response: CapturedResponse) -> HttpResponse:
|
|
70
|
+
"""Build an HttpResponse from a captured response."""
|
|
71
|
+
return HttpResponse(
|
|
72
|
+
status_code=response.status_code,
|
|
73
|
+
headers=response.headers,
|
|
74
|
+
body=response.body,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class RequestInterceptor:
|
|
79
|
+
"""Captures HTTP request-response pairs from an instrumented browser session.
|
|
80
|
+
|
|
81
|
+
Classifies request cookies: server-set cookies (from prior Set-Cookie
|
|
82
|
+
responses) are stripped; client-set cookies (from JavaScript
|
|
83
|
+
document.cookie) are retained as inject_cookies on the exchange.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
excluded_resource_types: set[str] | None = None,
|
|
89
|
+
) -> None:
|
|
90
|
+
self._exchanges: list[HttpExchange] = []
|
|
91
|
+
self._excluded_resource_types: frozenset[str] = (
|
|
92
|
+
frozenset(excluded_resource_types)
|
|
93
|
+
if excluded_resource_types is not None
|
|
94
|
+
else _STATIC_RESOURCE_TYPES
|
|
95
|
+
)
|
|
96
|
+
self._server_cookies: set[str] = set()
|
|
97
|
+
|
|
98
|
+
def _classify_request_cookies(
|
|
99
|
+
self,
|
|
100
|
+
request_headers: dict[str, str],
|
|
101
|
+
) -> tuple[dict[str, str], dict[str, str]]:
|
|
102
|
+
"""Separate request headers into clean headers and client cookies.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
(clean_headers without cookie key, inject_cookies for client-set cookies)
|
|
106
|
+
"""
|
|
107
|
+
inject_cookies: dict[str, str] = {}
|
|
108
|
+
clean_headers: dict[str, str] = {}
|
|
109
|
+
for key, header_value in request_headers.items():
|
|
110
|
+
if key.lower() == "cookie":
|
|
111
|
+
for name, value in _parse_cookie_header(header_value).items():
|
|
112
|
+
if name not in self._server_cookies:
|
|
113
|
+
inject_cookies[name] = value
|
|
114
|
+
else:
|
|
115
|
+
clean_headers[key] = header_value
|
|
116
|
+
return clean_headers, inject_cookies
|
|
117
|
+
|
|
118
|
+
def on_response(self, request: CapturedRequest, response: CapturedResponse) -> None:
|
|
119
|
+
"""Record a paired HTTP request-response exchange."""
|
|
120
|
+
if request.resource_type in self._excluded_resource_types:
|
|
121
|
+
return
|
|
122
|
+
|
|
123
|
+
self._server_cookies.update(_parse_set_cookie_names(response.headers))
|
|
124
|
+
clean_headers, inject_cookies = self._classify_request_cookies(request.headers)
|
|
125
|
+
|
|
126
|
+
self._exchanges.append(
|
|
127
|
+
HttpExchange(
|
|
128
|
+
request=_to_http_request(request, clean_headers),
|
|
129
|
+
response=_to_http_response(response),
|
|
130
|
+
timestamp=time.time(),
|
|
131
|
+
inject_cookies=inject_cookies,
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def get_exchanges(self) -> list[HttpExchange]:
|
|
136
|
+
"""Return a copy of all captured exchanges."""
|
|
137
|
+
return list(self._exchanges)
|
|
138
|
+
|
|
139
|
+
def reset(self) -> None:
|
|
140
|
+
"""Clear all accumulated state."""
|
|
141
|
+
self._exchanges.clear()
|
|
142
|
+
self._server_cookies.clear()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""EventRecorder — captures user interaction events from instrumented browser."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from krill_core import EventType, RecordedEvent
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class EventRecorder:
|
|
11
|
+
"""Captures user interaction events from an instrumented browser session."""
|
|
12
|
+
|
|
13
|
+
def __init__(self) -> None:
|
|
14
|
+
self._events: list[RecordedEvent] = []
|
|
15
|
+
|
|
16
|
+
def on_event(
|
|
17
|
+
self,
|
|
18
|
+
event_type: EventType,
|
|
19
|
+
selector: str,
|
|
20
|
+
value: str | None = None,
|
|
21
|
+
) -> None:
|
|
22
|
+
"""Record a user interaction event."""
|
|
23
|
+
self._events.append(
|
|
24
|
+
RecordedEvent(
|
|
25
|
+
event_type=event_type,
|
|
26
|
+
timestamp=time.time(),
|
|
27
|
+
selector=selector,
|
|
28
|
+
value=value,
|
|
29
|
+
)
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def get_events(self) -> list[RecordedEvent]:
|
|
33
|
+
"""Return a copy of all captured events."""
|
|
34
|
+
return list(self._events)
|
|
35
|
+
|
|
36
|
+
def reset(self) -> None:
|
|
37
|
+
"""Clear all accumulated state."""
|
|
38
|
+
self._events.clear()
|
|
File without changes
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// krill-fuzzer browser DOM-event capture script.
|
|
2
|
+
//
|
|
3
|
+
// Injected by WorkflowCollector via page.add_init_script to report DOM
|
|
4
|
+
// events (click, input, submit, scroll) to the exposed __krill_record_event
|
|
5
|
+
// binding. Playwright has no page.on("click"), so capture runs in-page.
|
|
6
|
+
|
|
7
|
+
function _krillSelector(el) {
|
|
8
|
+
return el.tagName.toLowerCase()
|
|
9
|
+
+ (el.id ? '#' + el.id : '')
|
|
10
|
+
+ (el.className ? '.' + String(el.className).split(' ').filter(Boolean).join('.') : '');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
document.addEventListener('click', (e) => {
|
|
14
|
+
window.__krill_record_event('CLICK', _krillSelector(e.target), null);
|
|
15
|
+
}, true);
|
|
16
|
+
|
|
17
|
+
document.addEventListener('input', (e) => {
|
|
18
|
+
window.__krill_record_event('INPUT', _krillSelector(e.target), e.target.value || null);
|
|
19
|
+
}, true);
|
|
20
|
+
|
|
21
|
+
document.addEventListener('submit', (e) => {
|
|
22
|
+
window.__krill_record_event('SUBMIT', _krillSelector(e.target), null);
|
|
23
|
+
}, true);
|
|
24
|
+
|
|
25
|
+
document.addEventListener('scroll', () => {
|
|
26
|
+
window.__krill_record_event('SCROLL', 'document', String(window.scrollY));
|
|
27
|
+
}, {capture: true, passive: true});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
<?php
|
|
2
|
+
/**
|
|
3
|
+
* krill-fuzzer server-side request capture.
|
|
4
|
+
*
|
|
5
|
+
* Injected by ServerRequestRecorder into PHP entry files to capture all
|
|
6
|
+
* incoming HTTP requests on the server side. Records method, URL, headers,
|
|
7
|
+
* body, and timestamp as a single JSONL line.
|
|
8
|
+
*
|
|
9
|
+
* Placeholders replaced at injection time:
|
|
10
|
+
* {{OUTPUT_DIR}} — directory for the log file (PHP expression)
|
|
11
|
+
* {{TIMESTAMP}} — unique session identifier
|
|
12
|
+
*/
|
|
13
|
+
(function () {
|
|
14
|
+
static $__krill_captured = false;
|
|
15
|
+
if ($__krill_captured) return;
|
|
16
|
+
$__krill_captured = true;
|
|
17
|
+
|
|
18
|
+
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
|
|
19
|
+
$scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
|
|
20
|
+
$host = $_SERVER['HTTP_HOST'] ?? ($_SERVER['SERVER_NAME'] ?? 'localhost');
|
|
21
|
+
$uri = $_SERVER['REQUEST_URI'] ?? '/';
|
|
22
|
+
$url = $scheme . '://' . $host . $uri;
|
|
23
|
+
|
|
24
|
+
$headers = [];
|
|
25
|
+
foreach ($_SERVER as $k => $v) {
|
|
26
|
+
if (strpos($k, 'HTTP_') === 0) {
|
|
27
|
+
$name = str_replace('_', '-', strtolower(substr($k, 5)));
|
|
28
|
+
$headers[$name] = $v;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (isset($_SERVER['CONTENT_TYPE'])) {
|
|
32
|
+
$headers['content-type'] = $_SERVER['CONTENT_TYPE'];
|
|
33
|
+
}
|
|
34
|
+
if (isset($_SERVER['CONTENT_LENGTH'])) {
|
|
35
|
+
$headers['content-length'] = $_SERVER['CONTENT_LENGTH'];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
$body = file_get_contents('php://input');
|
|
39
|
+
if ($body === false || $body === '') {
|
|
40
|
+
$body = null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
$entry = json_encode([
|
|
44
|
+
'method' => $method,
|
|
45
|
+
'url' => $url,
|
|
46
|
+
'headers' => $headers,
|
|
47
|
+
'body' => $body,
|
|
48
|
+
'ts' => microtime(true),
|
|
49
|
+
], JSON_UNESCAPED_SLASHES);
|
|
50
|
+
|
|
51
|
+
$log_file = {{OUTPUT_DIR}} . '/.krill_request_{{TIMESTAMP}}.jsonl';
|
|
52
|
+
$fp = @fopen($log_file, 'a');
|
|
53
|
+
if ($fp) {
|
|
54
|
+
if (flock($fp, LOCK_EX)) {
|
|
55
|
+
fwrite($fp, $entry . "\n");
|
|
56
|
+
flock($fp, LOCK_UN);
|
|
57
|
+
}
|
|
58
|
+
fclose($fp);
|
|
59
|
+
}
|
|
60
|
+
})();
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""ServerRequestRecorder — server-side HTTP request capture via PHP instrumentation.
|
|
2
|
+
|
|
3
|
+
Injects a lightweight PHP capture snippet into every PHP entry file in a project
|
|
4
|
+
directory. Each instrumented file records its incoming HTTP request (method, URL,
|
|
5
|
+
headers, body) as a JSONL line. After the recording session, the log is parsed
|
|
6
|
+
into HttpExchange objects (response-less, server-side only).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import importlib.resources
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from krill_core import HttpMethod, HttpRequest
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
_RUNTIME_FILENAME = "krill_request_capture.php"
|
|
24
|
+
_LOG_PREFIX = ".krill_request_"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ServerRequestRecorder:
|
|
28
|
+
"""Instruments PHP files to log incoming HTTP requests server-side.
|
|
29
|
+
|
|
30
|
+
Lifecycle: instrument() → (recording session happens) → collect() → deinstrument().
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, project_root: Path, output_dir: Path | None = None) -> None:
|
|
34
|
+
if not project_root.is_dir():
|
|
35
|
+
raise ValueError(f"project_root is not a directory: {project_root}")
|
|
36
|
+
self._project_root = project_root.resolve()
|
|
37
|
+
self._output_dir = output_dir.resolve() if output_dir else self._project_root
|
|
38
|
+
self._timestamp = f"{time.strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
|
39
|
+
self._runtime_dest = self._project_root / _RUNTIME_FILENAME
|
|
40
|
+
self._injected_files: list[Path] = []
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def project_root(self) -> Path:
|
|
44
|
+
"""Resolved root directory of the instrumented PHP project."""
|
|
45
|
+
return self._project_root
|
|
46
|
+
|
|
47
|
+
def instrument(self) -> list[Path]:
|
|
48
|
+
"""Inject request-logging snippet into all PHP files.
|
|
49
|
+
|
|
50
|
+
Copies the runtime helper to the project root (with placeholders
|
|
51
|
+
substituted), then prepends a ``require_once`` to every ``.php``
|
|
52
|
+
file in the project directory tree.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
List of modified PHP file paths.
|
|
56
|
+
"""
|
|
57
|
+
self._injected_files = []
|
|
58
|
+
self._deploy_runtime_helper()
|
|
59
|
+
|
|
60
|
+
for php_file in sorted(self._project_root.rglob("*.php")):
|
|
61
|
+
if php_file.name == _RUNTIME_FILENAME:
|
|
62
|
+
continue
|
|
63
|
+
if self._inject_require(php_file):
|
|
64
|
+
self._injected_files.append(php_file)
|
|
65
|
+
|
|
66
|
+
logger.debug(
|
|
67
|
+
"ServerRequestRecorder: instrumented %d files in %s",
|
|
68
|
+
len(self._injected_files),
|
|
69
|
+
self._project_root,
|
|
70
|
+
)
|
|
71
|
+
return list(self._injected_files)
|
|
72
|
+
|
|
73
|
+
def collect(self) -> list[HttpRequest]:
|
|
74
|
+
"""Read server-side request logs and return parsed HttpRequest objects.
|
|
75
|
+
|
|
76
|
+
Reads all ``.krill_request_*.jsonl`` files from the output directory,
|
|
77
|
+
parses each JSONL line into an HttpRequest, and returns them sorted
|
|
78
|
+
by timestamp.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
List of HttpRequest captured server-side, ordered by timestamp.
|
|
82
|
+
"""
|
|
83
|
+
log_files = sorted(self._output_dir.glob(f"{_LOG_PREFIX}*.jsonl"))
|
|
84
|
+
if not log_files:
|
|
85
|
+
logger.debug("ServerRequestRecorder: no request log files found")
|
|
86
|
+
return []
|
|
87
|
+
|
|
88
|
+
entries: list[tuple[float, HttpRequest]] = []
|
|
89
|
+
for log_file in log_files:
|
|
90
|
+
entries.extend(self._parse_log_file(log_file))
|
|
91
|
+
|
|
92
|
+
entries.sort(key=lambda pair: pair[0])
|
|
93
|
+
requests = [r for _, r in entries]
|
|
94
|
+
logger.debug(
|
|
95
|
+
"ServerRequestRecorder: collected %d server-side requests",
|
|
96
|
+
len(requests),
|
|
97
|
+
)
|
|
98
|
+
return requests
|
|
99
|
+
|
|
100
|
+
def _parse_log_file(self, log_file: Path) -> list[tuple[float, HttpRequest]]:
|
|
101
|
+
"""Parse one JSONL log file into (timestamp, request) pairs.
|
|
102
|
+
|
|
103
|
+
Blank lines are skipped; a line that is not valid JSON is logged and
|
|
104
|
+
skipped so one corrupt record cannot discard the whole file.
|
|
105
|
+
"""
|
|
106
|
+
entries: list[tuple[float, HttpRequest]] = []
|
|
107
|
+
for line_num, line in enumerate(
|
|
108
|
+
log_file.read_text(encoding="utf-8").splitlines(),
|
|
109
|
+
start=1,
|
|
110
|
+
):
|
|
111
|
+
entry = _parse_log_line(log_file.name, line_num, line)
|
|
112
|
+
if entry is not None:
|
|
113
|
+
entries.append(entry)
|
|
114
|
+
return entries
|
|
115
|
+
|
|
116
|
+
def deinstrument(self) -> list[Path]:
|
|
117
|
+
"""Remove injected require_once lines and runtime helper.
|
|
118
|
+
|
|
119
|
+
Scans the project tree rather than the in-memory injected-file list, so
|
|
120
|
+
instrumentation left by a crashed or separate session is still removed.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
List of restored PHP file paths.
|
|
124
|
+
"""
|
|
125
|
+
restored: list[Path] = []
|
|
126
|
+
for php_file in sorted(self._project_root.rglob("*.php")):
|
|
127
|
+
if php_file.name == _RUNTIME_FILENAME:
|
|
128
|
+
continue
|
|
129
|
+
if self._remove_require(php_file):
|
|
130
|
+
restored.append(php_file)
|
|
131
|
+
|
|
132
|
+
if self._runtime_dest.exists():
|
|
133
|
+
self._runtime_dest.unlink()
|
|
134
|
+
|
|
135
|
+
self._clean_log_files()
|
|
136
|
+
self._injected_files = []
|
|
137
|
+
|
|
138
|
+
logger.debug(
|
|
139
|
+
"ServerRequestRecorder: deinstrumented %d files",
|
|
140
|
+
len(restored),
|
|
141
|
+
)
|
|
142
|
+
return restored
|
|
143
|
+
|
|
144
|
+
def _deploy_runtime_helper(self) -> None:
|
|
145
|
+
"""Copy the PHP runtime snippet to the project root with placeholders replaced."""
|
|
146
|
+
template_ref = importlib.resources.files(
|
|
147
|
+
"krill_collector._resources",
|
|
148
|
+
).joinpath(_RUNTIME_FILENAME)
|
|
149
|
+
template = template_ref.read_text(encoding="utf-8")
|
|
150
|
+
|
|
151
|
+
if self._output_dir != self._project_root:
|
|
152
|
+
php_output_dir = f"'{self._output_dir}'"
|
|
153
|
+
else:
|
|
154
|
+
php_output_dir = "__DIR__"
|
|
155
|
+
|
|
156
|
+
content = template.replace("{{OUTPUT_DIR}}", php_output_dir).replace(
|
|
157
|
+
"{{TIMESTAMP}}", self._timestamp
|
|
158
|
+
)
|
|
159
|
+
self._runtime_dest.write_text(content, encoding="utf-8")
|
|
160
|
+
|
|
161
|
+
def _inject_require(self, php_file: Path) -> bool:
|
|
162
|
+
"""Prepend require_once for the runtime helper after <?php tag."""
|
|
163
|
+
content = php_file.read_text(encoding="utf-8")
|
|
164
|
+
require_line = f"require_once __DIR__ . '/{_RUNTIME_FILENAME}';"
|
|
165
|
+
|
|
166
|
+
if require_line in content:
|
|
167
|
+
return False
|
|
168
|
+
if _RUNTIME_FILENAME in content:
|
|
169
|
+
return False
|
|
170
|
+
|
|
171
|
+
if "<?php" in content:
|
|
172
|
+
content = content.replace(
|
|
173
|
+
"<?php",
|
|
174
|
+
f"<?php\n{require_line}\n",
|
|
175
|
+
1,
|
|
176
|
+
)
|
|
177
|
+
php_file.write_text(content, encoding="utf-8")
|
|
178
|
+
return True
|
|
179
|
+
return False
|
|
180
|
+
|
|
181
|
+
def _remove_require(self, php_file: Path) -> bool:
|
|
182
|
+
"""Remove the injected require_once line from a PHP file."""
|
|
183
|
+
if not php_file.exists():
|
|
184
|
+
return False
|
|
185
|
+
content = php_file.read_text(encoding="utf-8")
|
|
186
|
+
if _RUNTIME_FILENAME not in content:
|
|
187
|
+
return False
|
|
188
|
+
|
|
189
|
+
lines = content.splitlines(keepends=True)
|
|
190
|
+
new_lines = [
|
|
191
|
+
line
|
|
192
|
+
for line in lines
|
|
193
|
+
if not ("require_once" in line and _RUNTIME_FILENAME in line)
|
|
194
|
+
]
|
|
195
|
+
new_content = "".join(new_lines)
|
|
196
|
+
if new_content == content:
|
|
197
|
+
return False
|
|
198
|
+
php_file.write_text(new_content, encoding="utf-8")
|
|
199
|
+
return True
|
|
200
|
+
|
|
201
|
+
def _clean_log_files(self) -> None:
|
|
202
|
+
"""Remove server-side request log files."""
|
|
203
|
+
for log_file in self._output_dir.glob(f"{_LOG_PREFIX}*.jsonl"):
|
|
204
|
+
log_file.unlink(missing_ok=True)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _parse_log_line(
|
|
208
|
+
log_name: str,
|
|
209
|
+
line_num: int,
|
|
210
|
+
line: str,
|
|
211
|
+
) -> tuple[float, HttpRequest] | None:
|
|
212
|
+
"""Parse one JSONL log line into a (timestamp, request) pair, or None to skip."""
|
|
213
|
+
stripped = line.strip()
|
|
214
|
+
if not stripped:
|
|
215
|
+
return None
|
|
216
|
+
try:
|
|
217
|
+
entry = json.loads(stripped)
|
|
218
|
+
except json.JSONDecodeError:
|
|
219
|
+
logger.warning(
|
|
220
|
+
"ServerRequestRecorder: invalid JSON at %s:%d", log_name, line_num
|
|
221
|
+
)
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
request = _parse_request_entry(entry)
|
|
225
|
+
if request is None:
|
|
226
|
+
return None
|
|
227
|
+
return (entry.get("ts", 0.0), request)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _parse_request_entry(entry: dict[str, Any]) -> HttpRequest | None:
|
|
231
|
+
"""Parse a JSONL entry into an HttpRequest, or None if malformed."""
|
|
232
|
+
method_str = entry.get("method")
|
|
233
|
+
url = entry.get("url")
|
|
234
|
+
if not method_str or not url:
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
headers = entry.get("headers", {})
|
|
238
|
+
body_str = entry.get("body")
|
|
239
|
+
body = body_str.encode("utf-8") if body_str is not None else None
|
|
240
|
+
|
|
241
|
+
return HttpRequest(
|
|
242
|
+
method=HttpMethod(method_str),
|
|
243
|
+
url=url,
|
|
244
|
+
headers=headers,
|
|
245
|
+
body=body,
|
|
246
|
+
)
|
|
File without changes
|