repoburg 1.3.255 → 1.3.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/dist/index.js +61 -3
- package/daemon/dist/staticServing.js +2 -2
- package/daemon/dist/tabtape/api/follow.js +174 -0
- package/daemon/dist/tabtape/api/http.js +221 -0
- package/daemon/dist/tabtape/api/record.js +149 -0
- package/daemon/dist/tabtape/api/router.js +519 -0
- package/daemon/dist/tabtape/api/router.spec.js +604 -0
- package/daemon/dist/tabtape/api/static.js +70 -0
- package/daemon/dist/tabtape/api/ws.js +166 -0
- package/daemon/dist/tabtape/buffer/buffer.js +347 -0
- package/daemon/dist/tabtape/buffer/buffer.spec.js +137 -0
- package/daemon/dist/tabtape/config.js +108 -0
- package/daemon/dist/tabtape/e2e-server.js +96 -0
- package/daemon/dist/tabtape/errors.js +77 -0
- package/daemon/dist/tabtape/export/export.js +331 -0
- package/daemon/dist/tabtape/export/export.spec.js +262 -0
- package/daemon/dist/tabtape/har/gostd.js +395 -0
- package/daemon/dist/tabtape/har/gostd.spec.js +139 -0
- package/daemon/dist/tabtape/har/har.js +582 -0
- package/daemon/dist/tabtape/har/har.spec.js +325 -0
- package/daemon/dist/tabtape/index.js +134 -0
- package/daemon/dist/tabtape/index.spec.js +104 -0
- package/daemon/dist/tabtape/lock.js +67 -0
- package/daemon/dist/tabtape/log.js +32 -0
- package/daemon/dist/tabtape/mcp/definitions.js +362 -0
- package/daemon/dist/tabtape/mcp/gofmt.js +227 -0
- package/daemon/dist/tabtape/mcp/params.js +345 -0
- package/daemon/dist/tabtape/mcp/params.spec.js +172 -0
- package/daemon/dist/tabtape/mcp/recordLink.js +122 -0
- package/daemon/dist/tabtape/mcp/server.js +411 -0
- package/daemon/dist/tabtape/mcp/server.spec.js +722 -0
- package/daemon/dist/tabtape/mcp/tools.js +1085 -0
- package/daemon/dist/tabtape/query/format.js +195 -0
- package/daemon/dist/tabtape/query/network.js +133 -0
- package/daemon/dist/tabtape/query/perf.js +93 -0
- package/daemon/dist/tabtape/query/query.spec.js +233 -0
- package/daemon/dist/tabtape/query/reader.js +602 -0
- package/daemon/dist/tabtape/query/types.js +274 -0
- package/daemon/dist/tabtape/redact.js +37 -0
- package/daemon/dist/tabtape/repro/lang.js +95 -0
- package/daemon/dist/tabtape/repro/repro.js +422 -0
- package/daemon/dist/tabtape/repro/repro.spec.js +210 -0
- package/daemon/dist/tabtape/sink.js +24 -0
- package/daemon/dist/tabtape/store/record.js +99 -0
- package/daemon/dist/tabtape/store/segment.js +267 -0
- package/daemon/dist/tabtape/store/store.js +846 -0
- package/daemon/dist/tabtape/store/store.spec.js +485 -0
- package/daemon/dist/tabtape/testing/querytest.js +119 -0
- package/daemon/dist/tabtape/testing/tmp.js +33 -0
- package/package.json +3 -2
- package/tabtape/extension/dist/background.js +8 -0
- package/tabtape/extension/dist/background.js.map +7 -0
- package/tabtape/extension/dist/bridge.js +3 -0
- package/tabtape/extension/dist/bridge.js.map +7 -0
- package/tabtape/extension/dist/icons/128.png +0 -0
- package/tabtape/extension/dist/icons/16.png +0 -0
- package/tabtape/extension/dist/icons/32.png +0 -0
- package/tabtape/extension/dist/icons/48.png +0 -0
- package/tabtape/extension/dist/manifest.json +28 -0
- package/tabtape/extension/dist/popup.css +62 -0
- package/tabtape/extension/dist/popup.html +34 -0
- package/tabtape/extension/dist/popup.js +4 -0
- package/tabtape/extension/dist/popup.js.map +7 -0
- package/tabtape/extension/dist/recorder.js +133 -0
- package/tabtape/extension/dist/recorder.js.map +1 -0
- package/tabtape/extension/dist/state.js +2 -0
- package/tabtape/extension/dist/state.js.map +1 -0
- package/tabtape/playwright/mcp.mjs +58 -0
- package/tabtape/playwright/tabtape.d.mts +37 -0
- package/tabtape/playwright/tabtape.mjs +146 -0
- package/tabtape/ui/dist/app/assets/index-Canv3Ljk.css +1 -0
- package/tabtape/ui/dist/app/assets/index-iWNSSrAE.js +145 -0
- package/tabtape/ui/dist/app/index.html +14 -0
- package/tabtape/ui/dist/export/viewer.css +2 -0
- package/tabtape/ui/dist/export/viewer.js +143 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FollowUpgrade = void 0;
|
|
4
|
+
exports.allowedOrigin = allowedOrigin;
|
|
5
|
+
const ws_1 = require("ws");
|
|
6
|
+
const follow_1 = require("./follow");
|
|
7
|
+
const http_1 = require("./http");
|
|
8
|
+
const STALL_MS = 60000;
|
|
9
|
+
function reject(socket, status, text, message) {
|
|
10
|
+
const body = message + '\n';
|
|
11
|
+
socket.write(`HTTP/1.1 ${status} ${text}\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`);
|
|
12
|
+
socket.destroy();
|
|
13
|
+
}
|
|
14
|
+
/** The page's origin must be the one serving it (the replay UI) or the extension. */
|
|
15
|
+
function allowedOrigin(req) {
|
|
16
|
+
const origin = req.headers.origin;
|
|
17
|
+
if (origin === undefined)
|
|
18
|
+
return true; // not a browser
|
|
19
|
+
if (origin.startsWith('chrome-extension://'))
|
|
20
|
+
return true;
|
|
21
|
+
let u;
|
|
22
|
+
try {
|
|
23
|
+
u = new URL(origin);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const host = (req.headers.host ?? '').toLowerCase();
|
|
29
|
+
return (u.protocol === 'http:' || u.protocol === 'https:') && u.host.toLowerCase() === host;
|
|
30
|
+
}
|
|
31
|
+
class FollowUpgrade {
|
|
32
|
+
constructor(opts) {
|
|
33
|
+
this.opts = opts;
|
|
34
|
+
this.wss = new ws_1.WebSocketServer({ noServer: true, perMessageDeflate: false, maxPayload: 64 * 1024 });
|
|
35
|
+
const mount = opts.mount.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
36
|
+
this.pattern = new RegExp(`^${mount}/api/sessions/([^/?#]+)/events/?$`);
|
|
37
|
+
}
|
|
38
|
+
/** Whether the upgrade request is tabtape's. */
|
|
39
|
+
matches(req) {
|
|
40
|
+
const url = req.url ?? '';
|
|
41
|
+
return url === this.opts.mount || url.startsWith(this.opts.mount + '/') || url.startsWith(this.opts.mount + '?');
|
|
42
|
+
}
|
|
43
|
+
handle(req, socket, head) {
|
|
44
|
+
if (!(0, http_1.loopbackHost)(req.headers.host) || (0, http_1.viaTunnel)(req)) {
|
|
45
|
+
reject(socket, 403, 'Forbidden', 'forbidden host');
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (!allowedOrigin(req)) {
|
|
49
|
+
reject(socket, 403, 'Forbidden', 'forbidden origin');
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const pathname = (req.url ?? '').split('?')[0];
|
|
53
|
+
const m = this.pattern.exec(pathname);
|
|
54
|
+
if (!m) {
|
|
55
|
+
reject(socket, 404, 'Not Found', 'not found');
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
let id;
|
|
59
|
+
try {
|
|
60
|
+
id = decodeURIComponent(m[1]);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
reject(socket, 400, 'Bad Request', 'invalid session id');
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.wss.handleUpgrade(req, socket, head, (ws) => {
|
|
67
|
+
void this.run(ws, id);
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
async run(ws, id) {
|
|
71
|
+
const { st, followers, stop, logger } = this.opts;
|
|
72
|
+
if (!st.has(id)) {
|
|
73
|
+
ws.close(4404, 'session not found');
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (!followers.acquire(id)) {
|
|
77
|
+
ws.close(4429, `too many live viewers (at most ${followers.max})`);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const out = wsOut(ws);
|
|
82
|
+
const end = await (0, follow_1.follow)(st, id, out, stop, this.opts.keepAliveMs);
|
|
83
|
+
switch (end) {
|
|
84
|
+
case 'ended':
|
|
85
|
+
ws.close(1000, 'ended');
|
|
86
|
+
break;
|
|
87
|
+
case 'gone':
|
|
88
|
+
ws.close(1000, 'session deleted');
|
|
89
|
+
break;
|
|
90
|
+
case 'stopped':
|
|
91
|
+
ws.close(1001, 'server shutting down');
|
|
92
|
+
break;
|
|
93
|
+
case 'left':
|
|
94
|
+
if (ws.readyState === ws_1.WebSocket.OPEN)
|
|
95
|
+
ws.terminate();
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
logger.warn('live viewer failed', { session: id, err });
|
|
101
|
+
if (ws.readyState === ws_1.WebSocket.OPEN)
|
|
102
|
+
ws.close(1011, 'internal error');
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
followers.release(id);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
close() {
|
|
109
|
+
this.wss.close();
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.FollowUpgrade = FollowUpgrade;
|
|
113
|
+
/** Sends complete lines only: segment chunks may end inside a line. */
|
|
114
|
+
function wsOut(ws) {
|
|
115
|
+
let parts = [];
|
|
116
|
+
let partsLen = 0;
|
|
117
|
+
const left = new Promise((resolve) => {
|
|
118
|
+
ws.once('close', () => resolve());
|
|
119
|
+
ws.once('error', () => resolve());
|
|
120
|
+
});
|
|
121
|
+
const send = (data) => new Promise((resolve, rejectSend) => {
|
|
122
|
+
if (ws.readyState !== ws_1.WebSocket.OPEN) {
|
|
123
|
+
rejectSend(new http_1.ClientGoneError());
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const timer = setTimeout(() => {
|
|
127
|
+
ws.terminate();
|
|
128
|
+
rejectSend(new http_1.ClientGoneError('client stopped reading'));
|
|
129
|
+
}, STALL_MS);
|
|
130
|
+
ws.send(data, { binary: false }, (err) => {
|
|
131
|
+
clearTimeout(timer);
|
|
132
|
+
if (err)
|
|
133
|
+
rejectSend(err);
|
|
134
|
+
else
|
|
135
|
+
resolve();
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
return {
|
|
139
|
+
left,
|
|
140
|
+
write(chunk) {
|
|
141
|
+
const nl = chunk.lastIndexOf(0x0a);
|
|
142
|
+
if (nl < 0) {
|
|
143
|
+
parts.push(chunk);
|
|
144
|
+
partsLen += chunk.length;
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const head = chunk.subarray(0, nl + 1);
|
|
148
|
+
const message = partsLen ? Buffer.concat([...parts, head], partsLen + head.length) : head;
|
|
149
|
+
parts = nl + 1 < chunk.length ? [chunk.subarray(nl + 1)] : [];
|
|
150
|
+
partsLen = parts.length ? parts[0].length : 0;
|
|
151
|
+
return send(message);
|
|
152
|
+
},
|
|
153
|
+
flush() {
|
|
154
|
+
if (!partsLen)
|
|
155
|
+
return;
|
|
156
|
+
const message = Buffer.concat(parts, partsLen);
|
|
157
|
+
parts = [];
|
|
158
|
+
partsLen = 0;
|
|
159
|
+
return send(message);
|
|
160
|
+
},
|
|
161
|
+
keepAlive() {
|
|
162
|
+
if (ws.readyState === ws_1.WebSocket.OPEN)
|
|
163
|
+
ws.ping();
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.BufferStore = exports.DEFAULT_WINDOW_MS = void 0;
|
|
7
|
+
exports.newSessionID = newSessionID;
|
|
8
|
+
/**
|
|
9
|
+
* Keeps the last minutes of tabs in buffer mode, apart from the recordings:
|
|
10
|
+
* the extension streams allow-listed tabs here continuously, and "save"
|
|
11
|
+
* copies the last N minutes of one tab into a normal session.
|
|
12
|
+
*
|
|
13
|
+
* Layout: <dir>/<key>/<seq>.jsonl. A tab's data is split into segments that
|
|
14
|
+
* each start at an rrweb checkout (Meta event + full snapshot), so a save can
|
|
15
|
+
* start at any segment and still replay. Segments that are no longer needed
|
|
16
|
+
* for the tab's window are deleted as new ones arrive, and the combined size
|
|
17
|
+
* of all buffers is capped (oldest segments first). Buffers are ephemeral:
|
|
18
|
+
* the directory is cleared on start.
|
|
19
|
+
*/
|
|
20
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
21
|
+
const fs_1 = __importDefault(require("fs"));
|
|
22
|
+
const path_1 = __importDefault(require("path"));
|
|
23
|
+
const errors_1 = require("../errors");
|
|
24
|
+
const log_1 = require("../log");
|
|
25
|
+
const record_1 = require("../store/record");
|
|
26
|
+
const segment_1 = require("../store/segment");
|
|
27
|
+
const store_1 = require("../store/store");
|
|
28
|
+
/** The buffered time when the extension does not say. */
|
|
29
|
+
exports.DEFAULT_WINDOW_MS = 2 * 60000;
|
|
30
|
+
const MIN_WINDOW_MS = 30000;
|
|
31
|
+
const MAX_WINDOW_MS = 30 * 60000;
|
|
32
|
+
/** Buffers whose tab stopped sending (browser closed or crashed) are deleted. */
|
|
33
|
+
const IDLE_EXPIRY_MS = 10 * 60000;
|
|
34
|
+
/** Saved records are appended to the session in batches of about this size. */
|
|
35
|
+
const COPY_BATCH = 4 << 20;
|
|
36
|
+
class BufferStore {
|
|
37
|
+
constructor(opts) {
|
|
38
|
+
this.opts = opts;
|
|
39
|
+
this.bufs = new Map();
|
|
40
|
+
/** All segments, oldest first. */
|
|
41
|
+
this.segments = [];
|
|
42
|
+
this.total = 0;
|
|
43
|
+
this.nextSeq = 1;
|
|
44
|
+
this.log = opts.logger ?? (0, log_1.consoleLogger)();
|
|
45
|
+
this.now = opts.now ?? Date.now;
|
|
46
|
+
}
|
|
47
|
+
/** Clears dir and returns an empty store. */
|
|
48
|
+
static open(opts) {
|
|
49
|
+
if (!opts.dir || !(opts.maxBytes > 0))
|
|
50
|
+
throw new Error('buffer: dir and maxBytes are required');
|
|
51
|
+
fs_1.default.rmSync(opts.dir, { recursive: true, force: true });
|
|
52
|
+
fs_1.default.mkdirSync(opts.dir, { recursive: true, mode: 0o755 });
|
|
53
|
+
return new BufferStore(opts);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Adds records to a tab's buffer, creating it if needed. windowMs is how
|
|
57
|
+
* much the tab keeps (0 keeps the current or default window); userAgent
|
|
58
|
+
* describes the browser for saved sessions.
|
|
59
|
+
*/
|
|
60
|
+
append(key, recs, windowMs, userAgent) {
|
|
61
|
+
if (!(0, store_1.validID)(key))
|
|
62
|
+
throw new errors_1.InvalidIDError();
|
|
63
|
+
this.expire();
|
|
64
|
+
const b = this.bufferFor(key);
|
|
65
|
+
if (windowMs > 0)
|
|
66
|
+
b.windowMs = Math.min(Math.max(windowMs, MIN_WINDOW_MS), MAX_WINDOW_MS);
|
|
67
|
+
b.ua = userAgent || b.ua;
|
|
68
|
+
b.lastWrite = this.now();
|
|
69
|
+
for (const r of recs) {
|
|
70
|
+
if ((0, record_1.rrwebTypeOf)(r) === record_1.RRWEB_META || !b.cur)
|
|
71
|
+
this.rotate(b);
|
|
72
|
+
const cur = b.cur;
|
|
73
|
+
const line = Buffer.from(r.raw + '\n', 'utf8');
|
|
74
|
+
let n = 0;
|
|
75
|
+
try {
|
|
76
|
+
while (n < line.length)
|
|
77
|
+
n += fs_1.default.writeSync(b.fd, line, n, line.length - n);
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
cur.size += n;
|
|
81
|
+
this.total += n;
|
|
82
|
+
this.closeFile(b);
|
|
83
|
+
throw new Error(`write buffer: ${(0, errors_1.errorMessage)(err)}`);
|
|
84
|
+
}
|
|
85
|
+
cur.size += n;
|
|
86
|
+
this.total += n;
|
|
87
|
+
if (cur.firstTs === 0 || r.ts < cur.firstTs)
|
|
88
|
+
cur.firstTs = r.ts;
|
|
89
|
+
cur.lastTs = Math.max(cur.lastTs, r.ts);
|
|
90
|
+
if (r.type === 'event')
|
|
91
|
+
notePage(b, r.raw);
|
|
92
|
+
}
|
|
93
|
+
this.retain(b);
|
|
94
|
+
this.enforceBudget(b.cur);
|
|
95
|
+
}
|
|
96
|
+
bufferFor(key) {
|
|
97
|
+
const existing = this.bufs.get(key);
|
|
98
|
+
if (existing)
|
|
99
|
+
return existing;
|
|
100
|
+
const dir = path_1.default.join(this.opts.dir, key);
|
|
101
|
+
try {
|
|
102
|
+
fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o755 });
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
throw new Error(`create buffer: ${(0, errors_1.errorMessage)(err)}`);
|
|
106
|
+
}
|
|
107
|
+
const b = { key, dir, segs: [], cur: null, fd: null, windowMs: exports.DEFAULT_WINDOW_MS, url: '', title: '', ua: '', lastWrite: 0 };
|
|
108
|
+
this.bufs.set(key, b);
|
|
109
|
+
return b;
|
|
110
|
+
}
|
|
111
|
+
rotate(b) {
|
|
112
|
+
this.closeFile(b);
|
|
113
|
+
const seq = this.nextSeq;
|
|
114
|
+
const p = path_1.default.join(b.dir, `${String(seq).padStart(16, '0')}.jsonl`);
|
|
115
|
+
let fd;
|
|
116
|
+
try {
|
|
117
|
+
fd = fs_1.default.openSync(p, 'ax', 0o644);
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
throw new Error(`create buffer segment: ${(0, errors_1.errorMessage)(err)}`);
|
|
121
|
+
}
|
|
122
|
+
this.nextSeq++;
|
|
123
|
+
const seg = { seq, key: b.key, path: p, size: 0, firstTs: 0, lastTs: 0 };
|
|
124
|
+
this.segments.push(seg);
|
|
125
|
+
b.segs.push(seg);
|
|
126
|
+
b.cur = seg;
|
|
127
|
+
b.fd = fd;
|
|
128
|
+
}
|
|
129
|
+
closeFile(b) {
|
|
130
|
+
if (b.fd === null)
|
|
131
|
+
return;
|
|
132
|
+
try {
|
|
133
|
+
fs_1.default.closeSync(b.fd);
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
this.log.warn('close buffer segment', { path: b.cur?.path, err });
|
|
137
|
+
}
|
|
138
|
+
b.fd = null;
|
|
139
|
+
b.cur = null;
|
|
140
|
+
}
|
|
141
|
+
/** Deletes segments a save can no longer need: a segment goes once the next one starts early enough to cover the whole window. */
|
|
142
|
+
retain(b) {
|
|
143
|
+
const cutoff = this.now() - b.windowMs;
|
|
144
|
+
while (b.segs.length > 1 && b.segs[1].firstTs <= cutoff)
|
|
145
|
+
this.remove(b.segs[0]);
|
|
146
|
+
}
|
|
147
|
+
enforceBudget(keep) {
|
|
148
|
+
while (this.total > this.opts.maxBytes) {
|
|
149
|
+
const seg = this.segments.find((s) => s !== keep);
|
|
150
|
+
if (!seg)
|
|
151
|
+
return;
|
|
152
|
+
this.remove(seg);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
remove(seg) {
|
|
156
|
+
const b = this.bufs.get(seg.key);
|
|
157
|
+
if (b && b.cur === seg)
|
|
158
|
+
this.closeFile(b);
|
|
159
|
+
try {
|
|
160
|
+
fs_1.default.unlinkSync(seg.path);
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
if (!(0, errors_1.isNotExist)(err))
|
|
164
|
+
this.log.warn('delete buffer segment', { path: seg.path, err });
|
|
165
|
+
}
|
|
166
|
+
this.total -= seg.size;
|
|
167
|
+
this.segments = this.segments.filter((x) => x !== seg);
|
|
168
|
+
if (b)
|
|
169
|
+
b.segs = b.segs.filter((x) => x !== seg);
|
|
170
|
+
}
|
|
171
|
+
expire() {
|
|
172
|
+
const cutoff = this.now() - IDLE_EXPIRY_MS;
|
|
173
|
+
for (const [key, b] of [...this.bufs]) {
|
|
174
|
+
if (b.lastWrite < cutoff)
|
|
175
|
+
this.deleteBuffer(key);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/** Removes a buffer and its files. */
|
|
179
|
+
delete(key) {
|
|
180
|
+
if (!this.bufs.has(key))
|
|
181
|
+
throw new errors_1.BufferNotFoundError();
|
|
182
|
+
this.deleteBuffer(key);
|
|
183
|
+
}
|
|
184
|
+
deleteBuffer(key) {
|
|
185
|
+
const b = this.bufs.get(key);
|
|
186
|
+
this.closeFile(b);
|
|
187
|
+
for (const seg of [...b.segs])
|
|
188
|
+
this.remove(seg);
|
|
189
|
+
this.bufs.delete(key);
|
|
190
|
+
try {
|
|
191
|
+
fs_1.default.rmSync(b.dir, { recursive: true, force: true });
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
this.log.warn('delete buffer', { key, err });
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/** Every buffer, most recently written first. */
|
|
198
|
+
list() {
|
|
199
|
+
this.expire();
|
|
200
|
+
return [...this.bufs.values()].map(infoOf).sort((a, b) => b.lastWriteAt - a.lastWriteAt);
|
|
201
|
+
}
|
|
202
|
+
/** The combined size of all buffers. */
|
|
203
|
+
stats() {
|
|
204
|
+
return { buffers: this.bufs.size, bytes: this.total, maxBytes: this.opts.maxBytes };
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Copies the buffer's last window into a new session of st and returns it.
|
|
208
|
+
* The copy starts at the newest checkout at or before now − window (or at
|
|
209
|
+
* the start of a younger buffer), so it covers at least the window and
|
|
210
|
+
* replays from a full snapshot. A marker event records the moment of
|
|
211
|
+
* saving.
|
|
212
|
+
*/
|
|
213
|
+
async save(key, st, opts = {}) {
|
|
214
|
+
const b = this.bufs.get(key);
|
|
215
|
+
if (!b)
|
|
216
|
+
throw new errors_1.BufferNotFoundError();
|
|
217
|
+
if (b.segs.length === 0)
|
|
218
|
+
throw new errors_1.BufferEmptyError();
|
|
219
|
+
const windowMs = opts.windowMs && opts.windowMs > 0 ? opts.windowMs : b.windowMs;
|
|
220
|
+
const now = this.now();
|
|
221
|
+
const cutoff = now - windowMs;
|
|
222
|
+
let first = 0;
|
|
223
|
+
b.segs.forEach((seg, i) => {
|
|
224
|
+
if (seg.firstTs <= cutoff)
|
|
225
|
+
first = i;
|
|
226
|
+
});
|
|
227
|
+
// Sizes are taken now: appends that arrive while copying are not part of the save.
|
|
228
|
+
const segs = b.segs.slice(first).map((seg) => ({ path: seg.path, size: seg.size }));
|
|
229
|
+
const id = newSessionID(now);
|
|
230
|
+
st.start({ id, startedAt: b.segs[first].firstTs, url: b.url, title: b.title, userAgent: b.ua, source: 'buffer' });
|
|
231
|
+
try {
|
|
232
|
+
for (const seg of segs)
|
|
233
|
+
await copySegment(seg.path, seg.size, id, st);
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
try {
|
|
237
|
+
st.delete(id); // best effort: do not leave half a session behind
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
// already gone
|
|
241
|
+
}
|
|
242
|
+
throw err;
|
|
243
|
+
}
|
|
244
|
+
const marker = (0, record_1.parseRecord)(`{"ts":${now},"type":"event","data":{"name":"marker","label":"Saved from buffer mode"}}`);
|
|
245
|
+
st.append(id, [marker]);
|
|
246
|
+
return opts.continue ? st.get(id) : st.end(id, now);
|
|
247
|
+
}
|
|
248
|
+
/** Closes open files. */
|
|
249
|
+
close() {
|
|
250
|
+
for (const b of this.bufs.values())
|
|
251
|
+
this.closeFile(b);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
exports.BufferStore = BufferStore;
|
|
255
|
+
/** Keeps the buffer's URL and title in step with page views. */
|
|
256
|
+
function notePage(b, raw) {
|
|
257
|
+
if (!raw.includes('"page_view"'))
|
|
258
|
+
return;
|
|
259
|
+
try {
|
|
260
|
+
const ev = JSON.parse(raw);
|
|
261
|
+
const d = ev.data;
|
|
262
|
+
if (d && d.name === 'page_view') {
|
|
263
|
+
b.url = typeof d.url === 'string' ? d.url : '';
|
|
264
|
+
b.title = (typeof d.title === 'string' && d.title) || b.title;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
catch {
|
|
268
|
+
// not a record we understand
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
function infoOf(b) {
|
|
272
|
+
let bytes = 0;
|
|
273
|
+
let firstTs = 0;
|
|
274
|
+
let lastTs = 0;
|
|
275
|
+
for (const seg of b.segs) {
|
|
276
|
+
bytes += seg.size;
|
|
277
|
+
if (firstTs === 0 || seg.firstTs < firstTs)
|
|
278
|
+
firstTs = seg.firstTs;
|
|
279
|
+
lastTs = Math.max(lastTs, seg.lastTs);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
key: b.key,
|
|
283
|
+
...(b.url ? { url: b.url } : {}),
|
|
284
|
+
...(b.title ? { title: b.title } : {}),
|
|
285
|
+
windowMs: b.windowMs,
|
|
286
|
+
...(firstTs ? { firstTs } : {}),
|
|
287
|
+
...(lastTs ? { lastTs } : {}),
|
|
288
|
+
durationMs: Math.max(lastTs - firstTs, 0),
|
|
289
|
+
bytes,
|
|
290
|
+
segments: b.segs.length,
|
|
291
|
+
lastWriteAt: b.lastWrite,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/** Copies a buffer segment's complete lines into the session; damaged lines are skipped. */
|
|
295
|
+
async function copySegment(p, size, id, st) {
|
|
296
|
+
let batch = [];
|
|
297
|
+
let batchBytes = 0;
|
|
298
|
+
const flush = () => {
|
|
299
|
+
if (batch.length === 0)
|
|
300
|
+
return;
|
|
301
|
+
st.append(id, batch);
|
|
302
|
+
batch = [];
|
|
303
|
+
batchBytes = 0;
|
|
304
|
+
};
|
|
305
|
+
let rest = Buffer.alloc(0);
|
|
306
|
+
const take = (line) => {
|
|
307
|
+
if (line.length === 0)
|
|
308
|
+
return;
|
|
309
|
+
let r;
|
|
310
|
+
try {
|
|
311
|
+
r = (0, record_1.parseRecord)(line);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
return; // a damaged line; the rest is still worth saving
|
|
315
|
+
}
|
|
316
|
+
batch.push(r);
|
|
317
|
+
batchBytes += r.raw.length;
|
|
318
|
+
if (batchBytes >= COPY_BATCH)
|
|
319
|
+
flush();
|
|
320
|
+
};
|
|
321
|
+
try {
|
|
322
|
+
for await (const chunk of (0, segment_1.readSegment)(p, 0, size)) {
|
|
323
|
+
let data = rest.length ? Buffer.concat([rest, chunk]) : chunk;
|
|
324
|
+
let nl = data.indexOf(0x0a);
|
|
325
|
+
while (nl >= 0) {
|
|
326
|
+
take(data.subarray(0, nl));
|
|
327
|
+
data = data.subarray(nl + 1);
|
|
328
|
+
nl = data.indexOf(0x0a);
|
|
329
|
+
}
|
|
330
|
+
rest = Buffer.from(data);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
catch (err) {
|
|
334
|
+
if ((0, errors_1.isNotExist)(err))
|
|
335
|
+
return; // deleted meanwhile
|
|
336
|
+
throw new Error(`read buffer segment: ${(0, errors_1.errorMessage)(err)}`);
|
|
337
|
+
}
|
|
338
|
+
take(rest);
|
|
339
|
+
flush();
|
|
340
|
+
}
|
|
341
|
+
/** Follows the extension's format: YYYYMMDD-HHMMSS-xxxxxx (local time). */
|
|
342
|
+
function newSessionID(now) {
|
|
343
|
+
const d = new Date(now);
|
|
344
|
+
const p = (n, w = 2) => String(n).padStart(w, '0');
|
|
345
|
+
const stamp = `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
|
|
346
|
+
return `${stamp}-${crypto_1.default.randomBytes(3).toString('hex')}`;
|
|
347
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
const fs_1 = __importDefault(require("fs"));
|
|
40
|
+
const path_1 = __importDefault(require("path"));
|
|
41
|
+
const errors_1 = require("../errors");
|
|
42
|
+
const log_1 = require("../log");
|
|
43
|
+
const reader_1 = require("../query/reader");
|
|
44
|
+
const record_1 = require("../store/record");
|
|
45
|
+
const qt = __importStar(require("../testing/querytest"));
|
|
46
|
+
const tmp_1 = require("../testing/tmp");
|
|
47
|
+
const buffer_1 = require("./buffer");
|
|
48
|
+
let clock = 0;
|
|
49
|
+
const stores = [];
|
|
50
|
+
function open(maxBytes) {
|
|
51
|
+
clock = 1759650432000;
|
|
52
|
+
const dir = path_1.default.join((0, tmp_1.tempDir)(), 'buffer');
|
|
53
|
+
const b = buffer_1.BufferStore.open({ dir, maxBytes, logger: log_1.silentLogger, now: () => clock });
|
|
54
|
+
stores.push(b);
|
|
55
|
+
return { b, dir };
|
|
56
|
+
}
|
|
57
|
+
afterEach(async () => {
|
|
58
|
+
while (stores.length)
|
|
59
|
+
stores.pop().close();
|
|
60
|
+
await qt.closeTestStores();
|
|
61
|
+
(0, tmp_1.cleanupTempDirs)();
|
|
62
|
+
});
|
|
63
|
+
const rec = (line) => (0, record_1.parseRecord)(line);
|
|
64
|
+
/** Appends one second of a tab: a checkout every 30 s, a click every second. */
|
|
65
|
+
function tick(b, key, sec) {
|
|
66
|
+
const ts = clock;
|
|
67
|
+
const recs = [];
|
|
68
|
+
if (sec % 30 === 0) {
|
|
69
|
+
recs.push(rec(`{"ts":${ts},"type":"rrweb","data":{"type":4,"data":{"href":"http://localhost/","width":800,"height":600},"timestamp":${ts}}}`), rec(`{"ts":${ts},"type":"rrweb","data":{"type":2,"data":{"node":{"type":0,"childNodes":[],"id":1}},"timestamp":${ts}}}`));
|
|
70
|
+
}
|
|
71
|
+
if (sec === 0)
|
|
72
|
+
recs.push(rec(`{"ts":${ts},"type":"event","data":{"name":"page_view","kind":"load","url":"http://localhost/shop","title":"Shop"}}`));
|
|
73
|
+
recs.push(rec(`{"ts":${ts},"type":"event","data":{"name":"click","url":"http://localhost/shop","target":{"tag":"button","selector":"button","text":"s${sec}"}}}`));
|
|
74
|
+
b.append(key, recs, 2 * 60000, 'UA/1');
|
|
75
|
+
clock += 1000;
|
|
76
|
+
}
|
|
77
|
+
describe('BufferStore', () => {
|
|
78
|
+
it('keeps only the window', () => {
|
|
79
|
+
const { b, dir } = open(100 << 20);
|
|
80
|
+
for (let sec = 0; sec < 15 * 60; sec++)
|
|
81
|
+
tick(b, 'tab-1', sec);
|
|
82
|
+
const info = b.list()[0];
|
|
83
|
+
// Window 2:00 plus at most one checkout interval.
|
|
84
|
+
expect(info.durationMs).toBeGreaterThanOrEqual(120000);
|
|
85
|
+
expect(info.durationMs).toBeLessThanOrEqual(150000);
|
|
86
|
+
expect(fs_1.default.readdirSync(path_1.default.join(dir, 'tab-1')).length).toBe(info.segments);
|
|
87
|
+
expect(info.segments).toBeLessThanOrEqual(6);
|
|
88
|
+
expect([info.url, info.title]).toEqual(['http://localhost/shop', 'Shop']);
|
|
89
|
+
});
|
|
90
|
+
it('saves the last minutes', async () => {
|
|
91
|
+
const { b } = open(100 << 20);
|
|
92
|
+
const st = qt.testStore();
|
|
93
|
+
for (let sec = 0; sec < 5 * 60; sec++)
|
|
94
|
+
tick(b, 'tab-1', sec);
|
|
95
|
+
const sess = await b.save('tab-1', st);
|
|
96
|
+
const dur = (sess.endedAt ?? 0) - sess.startedAt;
|
|
97
|
+
expect(sess.source).toBe('buffer');
|
|
98
|
+
expect(sess.userAgent).toBe('UA/1');
|
|
99
|
+
expect(dur).toBeGreaterThanOrEqual(120000);
|
|
100
|
+
expect(dur).toBeLessThanOrEqual(151000);
|
|
101
|
+
const v = await new reader_1.Reader(st).view(sess.id);
|
|
102
|
+
expect(v.replayable).toBe(true);
|
|
103
|
+
expect(v.lines[0].rr).toBe(reader_1.RR_META);
|
|
104
|
+
expect(v.lines[v.lines.length - 1].ev?.name).toBe('marker');
|
|
105
|
+
// Saving does not stop the buffer.
|
|
106
|
+
tick(b, 'tab-1', 300);
|
|
107
|
+
expect(b.list().length).toBe(1);
|
|
108
|
+
});
|
|
109
|
+
it('saves a young buffer and can continue', async () => {
|
|
110
|
+
const { b } = open(100 << 20);
|
|
111
|
+
const st = qt.testStore();
|
|
112
|
+
for (let sec = 0; sec < 40; sec++)
|
|
113
|
+
tick(b, 'tab-1', sec);
|
|
114
|
+
const sess = await b.save('tab-1', st, { continue: true });
|
|
115
|
+
expect(sess.endedAt).toBeUndefined();
|
|
116
|
+
expect(sess.counts.clicks).toBe(40);
|
|
117
|
+
});
|
|
118
|
+
it('keeps to its budget and deletes buffers', async () => {
|
|
119
|
+
const { b, dir } = open(20000);
|
|
120
|
+
for (let sec = 0; sec < 200; sec++) {
|
|
121
|
+
tick(b, 'tab-1', sec);
|
|
122
|
+
tick(b, 'tab-2', sec);
|
|
123
|
+
}
|
|
124
|
+
expect(b.stats().bytes).toBeLessThanOrEqual(20000 + 2000);
|
|
125
|
+
b.delete('tab-1');
|
|
126
|
+
expect(fs_1.default.existsSync(path_1.default.join(dir, 'tab-1'))).toBe(false);
|
|
127
|
+
expect(() => b.delete('tab-1')).toThrow(errors_1.BufferNotFoundError);
|
|
128
|
+
await expect(b.save('nope', qt.testStore())).rejects.toThrow(errors_1.BufferNotFoundError);
|
|
129
|
+
expect(() => b.append('../x', [], 0, '')).toThrow(errors_1.InvalidIDError);
|
|
130
|
+
});
|
|
131
|
+
it('expires idle buffers', () => {
|
|
132
|
+
const { b } = open(100 << 20);
|
|
133
|
+
tick(b, 'tab-1', 0);
|
|
134
|
+
clock += 11 * 60000;
|
|
135
|
+
expect(b.list()).toEqual([]);
|
|
136
|
+
});
|
|
137
|
+
});
|