wacrawl-dashboard 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +220 -89
- package/apps/api/dist/demo/demoArchive.js +371 -0
- package/apps/api/dist/demo/demoCast.js +138 -0
- package/apps/api/dist/demo/demoMedia.js +108 -0
- package/apps/api/dist/index.js +109 -29
- package/apps/api/dist/lib/archiveSchema.js +57 -0
- package/apps/api/dist/lib/archiveSync.js +151 -0
- package/apps/api/dist/lib/chatKind.js +41 -0
- package/apps/api/dist/lib/displayName.js +37 -3
- package/apps/api/dist/lib/mediaFsPath.js +27 -0
- package/apps/api/dist/lib/percentile.js +23 -0
- package/apps/api/dist/lib/query.js +33 -7
- package/apps/api/dist/lib/rollupDb.js +253 -0
- package/apps/api/dist/lib/searchFilters.js +141 -0
- package/apps/api/dist/lib/searchIndex.js +3 -3
- package/apps/api/dist/lib/selfIdentity.js +42 -0
- package/apps/api/dist/lib/statsCache.js +83 -0
- package/apps/api/dist/lib/timezone.js +120 -5
- package/apps/api/dist/lib/urlExtract.js +111 -0
- package/apps/api/dist/lib/wordCloudUsefulStopWords.js +36 -3
- package/apps/api/dist/pathsResolve.js +17 -3
- package/apps/api/dist/routes/data.js +168 -78
- package/apps/api/dist/routes/settings.js +3 -0
- package/apps/api/dist/routes/stats/profiles.js +522 -0
- package/apps/api/dist/routes/stats/reports.js +555 -0
- package/apps/api/dist/routes/stats/shared.js +75 -0
- package/apps/api/dist/routes/stats.js +486 -97
- package/apps/api/dist/routes/sync.js +60 -0
- package/apps/api/dist/scripts/generateDemoArchive.js +24 -0
- package/apps/api/package.json +31 -0
- package/apps/web/dist/apple-touch-icon.png +0 -0
- package/apps/web/dist/assets/charts-BMGh5n-d.js +36 -0
- package/apps/web/dist/assets/icons-BdElrSaU.js +1 -0
- package/apps/web/dist/assets/index-Bb8Zj42D.css +1 -0
- package/apps/web/dist/assets/index-wrxENU1e.js +9 -0
- package/apps/web/dist/assets/react-D3RL6ob4.js +3 -0
- package/apps/web/dist/favicon-16x16.png +0 -0
- package/apps/web/dist/favicon-32x32.png +0 -0
- package/apps/web/dist/favicon-512x512.png +0 -0
- package/apps/web/dist/favicon.ico +0 -0
- package/apps/web/dist/favicon.svg +59 -0
- package/apps/web/dist/index.html +11 -2
- package/bin/wacrawl-dashboard.js +64 -15
- package/package.json +25 -4
- package/apps/web/dist/assets/index-1fGu93mF.css +0 -1
- package/apps/web/dist/assets/index-Dfsoi6vG.js +0 -43
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import Database from 'better-sqlite3';
|
|
4
|
+
import { getDb } from '../db.js';
|
|
5
|
+
/**
|
|
6
|
+
* A precomputed index of what the archive says, so the text reports can stop
|
|
7
|
+
* guessing.
|
|
8
|
+
*
|
|
9
|
+
* The word cloud and the emoji report both work by pulling message bodies into
|
|
10
|
+
* JS and running a regex over every one. That does not scale, so both were
|
|
11
|
+
* capped — the newest 30,000 and 50,000 messages — and every number they
|
|
12
|
+
* produced was therefore a sample rather than an answer. `ScanNote` says so on
|
|
13
|
+
* screen, which is honest but not a fix.
|
|
14
|
+
*
|
|
15
|
+
* Counting each term once, up front, turns those reports into an indexed sum:
|
|
16
|
+
* exact over the whole archive, and faster than the capped version it replaces.
|
|
17
|
+
*
|
|
18
|
+
* The archive itself is opened readonly and belongs to the wacrawl CLI, so the
|
|
19
|
+
* counts live in a sidecar file beside it. Everything here degrades to `null`
|
|
20
|
+
* rather than throwing: a read-only directory, a disk with no room, an archive
|
|
21
|
+
* that moved — any of those should cost the exact numbers, not the dashboard.
|
|
22
|
+
*/
|
|
23
|
+
/** Bump to force a rebuild when the tokenizer or the schema changes. */
|
|
24
|
+
const ROLLUP_VERSION = 1;
|
|
25
|
+
const HOUR_SECONDS = 3600;
|
|
26
|
+
/**
|
|
27
|
+
* Tokenizer, kept identical to the live one in stats.ts.
|
|
28
|
+
*
|
|
29
|
+
* If these ever disagree the rollup silently answers a different question than
|
|
30
|
+
* the fallback path, which is worse than having no rollup at all — so the
|
|
31
|
+
* version above must be bumped whenever this changes.
|
|
32
|
+
*/
|
|
33
|
+
const WORD_TOKEN_RE = /[\p{L}\p{N}]{3,}/gu;
|
|
34
|
+
const EMOJI_REGEX = /[\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu;
|
|
35
|
+
let cached = null;
|
|
36
|
+
/** One failed build per fingerprint is enough; do not retry on every request. */
|
|
37
|
+
let failedFingerprint = null;
|
|
38
|
+
function archiveFingerprint(primaryDb) {
|
|
39
|
+
const stat = fs.statSync(primaryDb);
|
|
40
|
+
return `${ROLLUP_VERSION}:${stat.mtimeMs}:${stat.size}`;
|
|
41
|
+
}
|
|
42
|
+
function rollupPath(primaryDb) {
|
|
43
|
+
const directory = path.dirname(primaryDb);
|
|
44
|
+
const base = path.basename(primaryDb, path.extname(primaryDb));
|
|
45
|
+
return path.join(directory, `${base}-rollups.db`);
|
|
46
|
+
}
|
|
47
|
+
function createSchema(db) {
|
|
48
|
+
db.exec(`
|
|
49
|
+
DROP TABLE IF EXISTS meta;
|
|
50
|
+
DROP TABLE IF EXISTS term_bucket;
|
|
51
|
+
DROP TABLE IF EXISTS emoji_bucket;
|
|
52
|
+
DROP TABLE IF EXISTS message_bucket;
|
|
53
|
+
|
|
54
|
+
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
55
|
+
|
|
56
|
+
-- Every token, stop words included: the word cloud's all/useful toggle is
|
|
57
|
+
-- applied when the report is built, so filtering here would remove a
|
|
58
|
+
-- choice the reader still has.
|
|
59
|
+
CREATE TABLE term_bucket (
|
|
60
|
+
term TEXT NOT NULL,
|
|
61
|
+
chat_jid TEXT NOT NULL,
|
|
62
|
+
hour INTEGER NOT NULL,
|
|
63
|
+
count INTEGER NOT NULL,
|
|
64
|
+
PRIMARY KEY (term, chat_jid, hour)
|
|
65
|
+
) WITHOUT ROWID;
|
|
66
|
+
|
|
67
|
+
-- person_jid is who sent it, which for my own messages is the chat rather
|
|
68
|
+
-- than me; it matches how getEmojiAnalytics attributes them.
|
|
69
|
+
CREATE TABLE emoji_bucket (
|
|
70
|
+
emoji TEXT NOT NULL,
|
|
71
|
+
person_jid TEXT NOT NULL,
|
|
72
|
+
from_me INTEGER NOT NULL,
|
|
73
|
+
hour INTEGER NOT NULL,
|
|
74
|
+
count INTEGER NOT NULL,
|
|
75
|
+
PRIMARY KEY (emoji, person_jid, from_me, hour)
|
|
76
|
+
) WITHOUT ROWID;
|
|
77
|
+
|
|
78
|
+
-- How many messages each hour was built from, so a report can state its
|
|
79
|
+
-- coverage exactly instead of inferring it.
|
|
80
|
+
CREATE TABLE message_bucket (
|
|
81
|
+
hour INTEGER PRIMARY KEY,
|
|
82
|
+
messages INTEGER NOT NULL
|
|
83
|
+
) WITHOUT ROWID;
|
|
84
|
+
|
|
85
|
+
CREATE INDEX idx_term_hour ON term_bucket(hour);
|
|
86
|
+
CREATE INDEX idx_emoji_hour ON emoji_bucket(hour);
|
|
87
|
+
`);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Count every term and emoji in the archive into the sidecar.
|
|
91
|
+
*
|
|
92
|
+
* Exported for the `optimize-db` script, which prebuilds so the first request
|
|
93
|
+
* after a sync does not pay for it.
|
|
94
|
+
*/
|
|
95
|
+
export function buildRollups(source, target) {
|
|
96
|
+
createSchema(target);
|
|
97
|
+
const rows = source.prepare(`
|
|
98
|
+
SELECT
|
|
99
|
+
chat_jid,
|
|
100
|
+
CASE WHEN from_me = 1 THEN chat_jid ELSE COALESCE(sender_jid, chat_jid) END AS person_jid,
|
|
101
|
+
from_me,
|
|
102
|
+
ts,
|
|
103
|
+
text
|
|
104
|
+
FROM messages
|
|
105
|
+
WHERE text IS NOT NULL AND TRIM(text) <> ''
|
|
106
|
+
`).all();
|
|
107
|
+
const terms = new Map();
|
|
108
|
+
const emoji = new Map();
|
|
109
|
+
const messages = new Map();
|
|
110
|
+
for (const row of rows) {
|
|
111
|
+
const hour = Math.floor(row.ts / HOUR_SECONDS);
|
|
112
|
+
messages.set(hour, (messages.get(hour) ?? 0) + 1);
|
|
113
|
+
// A tab separator cannot occur in a token, and JIDs do not contain one
|
|
114
|
+
// either, so splitting the key back apart is unambiguous.
|
|
115
|
+
for (const term of row.text.toLowerCase().match(WORD_TOKEN_RE) ?? []) {
|
|
116
|
+
const key = `${term}\t${row.chat_jid}\t${hour}`;
|
|
117
|
+
terms.set(key, (terms.get(key) ?? 0) + 1);
|
|
118
|
+
}
|
|
119
|
+
for (const glyph of row.text.match(EMOJI_REGEX) ?? []) {
|
|
120
|
+
const key = `${glyph}\t${row.person_jid}\t${row.from_me}\t${hour}`;
|
|
121
|
+
emoji.set(key, (emoji.get(key) ?? 0) + 1);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const insertTerm = target.prepare('INSERT INTO term_bucket (term, chat_jid, hour, count) VALUES (?, ?, ?, ?)');
|
|
125
|
+
const insertEmoji = target.prepare('INSERT INTO emoji_bucket (emoji, person_jid, from_me, hour, count) VALUES (?, ?, ?, ?, ?)');
|
|
126
|
+
const insertMessages = target.prepare('INSERT INTO message_bucket (hour, messages) VALUES (?, ?)');
|
|
127
|
+
target.transaction(() => {
|
|
128
|
+
for (const [key, count] of terms) {
|
|
129
|
+
const [term, chatJid, hour] = key.split('\t');
|
|
130
|
+
insertTerm.run(term, chatJid, Number(hour), count);
|
|
131
|
+
}
|
|
132
|
+
for (const [key, count] of emoji) {
|
|
133
|
+
const [glyph, personJid, fromMe, hour] = key.split('\t');
|
|
134
|
+
insertEmoji.run(glyph, personJid, Number(fromMe), Number(hour), count);
|
|
135
|
+
}
|
|
136
|
+
for (const [hour, count] of messages) {
|
|
137
|
+
insertMessages.run(hour, count);
|
|
138
|
+
}
|
|
139
|
+
})();
|
|
140
|
+
return rows.length;
|
|
141
|
+
}
|
|
142
|
+
function writeMeta(db, fingerprint, messages) {
|
|
143
|
+
const set = db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)');
|
|
144
|
+
set.run('fingerprint', fingerprint);
|
|
145
|
+
set.run('messages', String(messages));
|
|
146
|
+
set.run('built_at', new Date().toISOString());
|
|
147
|
+
}
|
|
148
|
+
function readMeta(db, key) {
|
|
149
|
+
try {
|
|
150
|
+
const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
|
|
151
|
+
return row?.value ?? null;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
// No meta table means an aborted build; treat it as absent.
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* The rollup for the current archive, building it if it is missing or stale.
|
|
160
|
+
*
|
|
161
|
+
* Returns null when the counts are unavailable for any reason, which is the
|
|
162
|
+
* caller's signal to fall back to scanning messages directly.
|
|
163
|
+
*/
|
|
164
|
+
export function getRollups(source = getDb()) {
|
|
165
|
+
// Derived from the handle we were given, never from ambient path state. An
|
|
166
|
+
// earlier version resolved the configured archive path instead, so any caller
|
|
167
|
+
// holding a different database — every test using an in-memory one — was
|
|
168
|
+
// silently served counts from somebody else's archive.
|
|
169
|
+
const primaryDb = source.name;
|
|
170
|
+
if (!primaryDb || primaryDb === ':memory:' || primaryDb === '')
|
|
171
|
+
return null;
|
|
172
|
+
let fingerprint;
|
|
173
|
+
try {
|
|
174
|
+
fingerprint = archiveFingerprint(primaryDb);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
if (cached?.fingerprint === fingerprint)
|
|
180
|
+
return cached.handle;
|
|
181
|
+
if (failedFingerprint === fingerprint)
|
|
182
|
+
return null;
|
|
183
|
+
closeRollups();
|
|
184
|
+
try {
|
|
185
|
+
const db = new Database(rollupPath(primaryDb));
|
|
186
|
+
db.pragma('journal_mode = WAL');
|
|
187
|
+
let messages;
|
|
188
|
+
if (readMeta(db, 'fingerprint') === fingerprint) {
|
|
189
|
+
messages = Number(readMeta(db, 'messages') ?? 0);
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
messages = buildRollups(source, db);
|
|
193
|
+
writeMeta(db, fingerprint, messages);
|
|
194
|
+
}
|
|
195
|
+
const handle = { db, messages };
|
|
196
|
+
cached = { fingerprint, handle };
|
|
197
|
+
return handle;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
// Read-only directory, no disk space, a corrupt sidecar — none of these
|
|
201
|
+
// should cost more than the exactness they were buying.
|
|
202
|
+
failedFingerprint = fingerprint;
|
|
203
|
+
cached = null;
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Build (or rebuild) the sidecar for an archive at a known path.
|
|
209
|
+
*
|
|
210
|
+
* The runtime decides a rollup is current by comparing fingerprints, so the
|
|
211
|
+
* prebuild has to stamp the fingerprint the same way — hence one function
|
|
212
|
+
* rather than two that have to agree.
|
|
213
|
+
*/
|
|
214
|
+
export function rebuildRollupsFor(primaryDb) {
|
|
215
|
+
const file = rollupPath(primaryDb);
|
|
216
|
+
const source = new Database(primaryDb, { fileMustExist: true, readonly: true });
|
|
217
|
+
const target = new Database(file);
|
|
218
|
+
try {
|
|
219
|
+
target.pragma('journal_mode = WAL');
|
|
220
|
+
const messages = buildRollups(source, target);
|
|
221
|
+
writeMeta(target, archiveFingerprint(primaryDb), messages);
|
|
222
|
+
return { file, messages };
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
target.close();
|
|
226
|
+
source.close();
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
/** The first hour bucket a period covers; buckets are whole hours of unix time. */
|
|
230
|
+
export function hourBucket(since) {
|
|
231
|
+
return Math.floor(since / HOUR_SECONDS);
|
|
232
|
+
}
|
|
233
|
+
/** Messages the rollup holds at or after `since`, for honest coverage reporting. */
|
|
234
|
+
export function messagesSince(handle, since) {
|
|
235
|
+
const row = handle.db
|
|
236
|
+
.prepare('SELECT COALESCE(SUM(messages), 0) AS total FROM message_bucket WHERE hour >= ?')
|
|
237
|
+
.get(hourBucket(since));
|
|
238
|
+
return row.total;
|
|
239
|
+
}
|
|
240
|
+
export function closeRollups() {
|
|
241
|
+
try {
|
|
242
|
+
cached?.handle.db.close();
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
// Already closed, or never opened.
|
|
246
|
+
}
|
|
247
|
+
cached = null;
|
|
248
|
+
}
|
|
249
|
+
/** Called after an import, when the sidecar describes an archive that is gone. */
|
|
250
|
+
export function invalidateRollups() {
|
|
251
|
+
closeRollups();
|
|
252
|
+
failedFingerprint = null;
|
|
253
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { parsePeriod, sinceTimestamp } from './query.js';
|
|
2
|
+
import { firstQueryString, resolveStatsTimeZone, tsModifierSql } from './timezone.js';
|
|
3
|
+
/** Bad filter input, reported as a 400 rather than silently widening the results. */
|
|
4
|
+
export class SearchFilterError extends Error {
|
|
5
|
+
code = 'VALIDATION_ERROR';
|
|
6
|
+
}
|
|
7
|
+
/** Longer than any JID or media type; only here so a junk URL fails clearly. */
|
|
8
|
+
const MAX_VALUE_LENGTH = 256;
|
|
9
|
+
const SECONDS_PER_DAY = 86_400;
|
|
10
|
+
function parseText(value, name) {
|
|
11
|
+
const text = firstQueryString(value);
|
|
12
|
+
if (text && text.length > MAX_VALUE_LENGTH) {
|
|
13
|
+
throw new SearchFilterError(`${name} is too long.`);
|
|
14
|
+
}
|
|
15
|
+
return text;
|
|
16
|
+
}
|
|
17
|
+
/** A real calendar date as YYYY-MM-DD, returned as UTC-midnight epoch seconds alongside. */
|
|
18
|
+
function parseDay(value, name) {
|
|
19
|
+
const day = firstQueryString(value);
|
|
20
|
+
if (!day)
|
|
21
|
+
return undefined;
|
|
22
|
+
const epochMs = /^\d{4}-\d{2}-\d{2}$/.test(day) ? Date.parse(`${day}T00:00:00Z`) : NaN;
|
|
23
|
+
// Date.parse rejects 2023-13-01 but rolls 2023-02-30 over to March, hence the round trip.
|
|
24
|
+
if (Number.isNaN(epochMs) || new Date(epochMs).toISOString().slice(0, 10) !== day) {
|
|
25
|
+
throw new SearchFilterError(`${name} must be a date in YYYY-MM-DD form.`);
|
|
26
|
+
}
|
|
27
|
+
return { day, epoch: epochMs / 1000 };
|
|
28
|
+
}
|
|
29
|
+
function parseIntegerInRange(value, name, max) {
|
|
30
|
+
const raw = firstQueryString(value);
|
|
31
|
+
if (!raw)
|
|
32
|
+
return undefined;
|
|
33
|
+
const parsed = /^\d{1,2}$/.test(raw) ? Number(raw) : NaN;
|
|
34
|
+
if (Number.isNaN(parsed) || parsed > max) {
|
|
35
|
+
throw new SearchFilterError(`${name} must be a whole number from 0 to ${max}.`);
|
|
36
|
+
}
|
|
37
|
+
return parsed;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Every spelling of one person in `messages`: the archive files them under a
|
|
41
|
+
* phone JID and an opaque `@lid`, and the reports merge the two, so a drill-down
|
|
42
|
+
* from a report has to as well or its count will not match.
|
|
43
|
+
*/
|
|
44
|
+
function identityJids(db, jid) {
|
|
45
|
+
const rows = db
|
|
46
|
+
.prepare('SELECT jid, lid FROM contacts WHERE jid = @jid OR lid = @jid')
|
|
47
|
+
.all({ jid });
|
|
48
|
+
const jids = new Set([jid]);
|
|
49
|
+
for (const row of rows) {
|
|
50
|
+
jids.add(row.jid);
|
|
51
|
+
if (row.lid)
|
|
52
|
+
jids.add(row.lid);
|
|
53
|
+
}
|
|
54
|
+
return JSON.stringify([...jids]);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Translate search filters into SQL, using the same expressions as the stats
|
|
58
|
+
* routes so a chart's bucket and the messages listed behind it stay in step:
|
|
59
|
+
*
|
|
60
|
+
* - `sender` is who wrote the message (`me` for your own).
|
|
61
|
+
* - `contact` is the Top Contacts definition, both directions: what they sent
|
|
62
|
+
* anywhere plus what you sent in your direct chat with them.
|
|
63
|
+
* - `period` is the dashboard's rolling window, not a calendar range.
|
|
64
|
+
* - `from`, `to`, `hour` and `weekday` read the viewer's calendar via `timeZone`.
|
|
65
|
+
*/
|
|
66
|
+
export function parseSearchFilters(params, db) {
|
|
67
|
+
const conditions = [];
|
|
68
|
+
const bindings = {};
|
|
69
|
+
const localTs = tsModifierSql(resolveStatsTimeZone(params.timeZone), 'messages.ts');
|
|
70
|
+
const chat = parseText(params.chat, 'chat');
|
|
71
|
+
if (chat) {
|
|
72
|
+
conditions.push('messages.chat_jid = @chat');
|
|
73
|
+
bindings.chat = chat;
|
|
74
|
+
}
|
|
75
|
+
const sender = parseText(params.sender, 'sender');
|
|
76
|
+
if (sender === 'me') {
|
|
77
|
+
conditions.push('messages.from_me = 1');
|
|
78
|
+
}
|
|
79
|
+
else if (sender) {
|
|
80
|
+
conditions.push(`
|
|
81
|
+
messages.from_me = 0
|
|
82
|
+
AND COALESCE(messages.sender_jid, messages.chat_jid) IN (SELECT value FROM json_each(@senderJids))
|
|
83
|
+
`);
|
|
84
|
+
bindings.senderJids = identityJids(db, sender);
|
|
85
|
+
}
|
|
86
|
+
const contact = parseText(params.contact, 'contact');
|
|
87
|
+
if (contact) {
|
|
88
|
+
conditions.push(`
|
|
89
|
+
CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END
|
|
90
|
+
IN (SELECT value FROM json_each(@contactJids))
|
|
91
|
+
`);
|
|
92
|
+
bindings.contactJids = identityJids(db, contact);
|
|
93
|
+
}
|
|
94
|
+
const mediaType = parseText(params.mediaType, 'mediaType');
|
|
95
|
+
if (mediaType === 'any') {
|
|
96
|
+
conditions.push("messages.media_type IS NOT NULL AND messages.media_type <> ''");
|
|
97
|
+
}
|
|
98
|
+
else if (mediaType) {
|
|
99
|
+
conditions.push('messages.media_type = @mediaType');
|
|
100
|
+
bindings.mediaType = mediaType;
|
|
101
|
+
}
|
|
102
|
+
const rawPeriod = firstQueryString(params.period);
|
|
103
|
+
if (rawPeriod) {
|
|
104
|
+
const period = parsePeriod(rawPeriod);
|
|
105
|
+
if (period !== rawPeriod) {
|
|
106
|
+
throw new SearchFilterError('period must be day, week, month, year or all.');
|
|
107
|
+
}
|
|
108
|
+
if (period !== 'all') {
|
|
109
|
+
conditions.push('messages.ts >= @since');
|
|
110
|
+
bindings.since = sinceTimestamp(period);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const from = parseDay(params.from, 'from');
|
|
114
|
+
const to = parseDay(params.to, 'to');
|
|
115
|
+
if (from && to && from.day > to.day) {
|
|
116
|
+
throw new SearchFilterError('from must not be after to.');
|
|
117
|
+
}
|
|
118
|
+
// The raw `ts` bound is what lets these use the index; no zone is a full day
|
|
119
|
+
// from UTC plus a margin, and the `date()` test settles the exact edge.
|
|
120
|
+
if (from) {
|
|
121
|
+
conditions.push(`messages.ts >= @fromEpoch AND date(${localTs}) >= @from`);
|
|
122
|
+
bindings.from = from.day;
|
|
123
|
+
bindings.fromEpoch = from.epoch - SECONDS_PER_DAY;
|
|
124
|
+
}
|
|
125
|
+
if (to) {
|
|
126
|
+
conditions.push(`messages.ts < @toEpoch AND date(${localTs}) <= @to`);
|
|
127
|
+
bindings.to = to.day;
|
|
128
|
+
bindings.toEpoch = to.epoch + 2 * SECONDS_PER_DAY;
|
|
129
|
+
}
|
|
130
|
+
const hour = parseIntegerInRange(params.hour, 'hour', 23);
|
|
131
|
+
if (hour !== undefined) {
|
|
132
|
+
conditions.push(`CAST(strftime('%H', ${localTs}) AS INTEGER) = @hour`);
|
|
133
|
+
bindings.hour = hour;
|
|
134
|
+
}
|
|
135
|
+
const weekday = parseIntegerInRange(params.weekday, 'weekday', 6);
|
|
136
|
+
if (weekday !== undefined) {
|
|
137
|
+
conditions.push(`CAST(strftime('%w', ${localTs}) AS INTEGER) = @weekday`);
|
|
138
|
+
bindings.weekday = weekday;
|
|
139
|
+
}
|
|
140
|
+
return { conditions, bindings };
|
|
141
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cleanDisplayNameSql, contactDisplayNameSql,
|
|
1
|
+
import { cleanDisplayNameSql, contactDisplayNameSql, contactLeftJoins } from './displayName.js';
|
|
2
2
|
export const MESSAGE_SEARCH_FTS_TABLE = 'messages_fts';
|
|
3
3
|
export function hasMessageSearchIndex(db) {
|
|
4
4
|
const row = db.prepare(`
|
|
@@ -51,7 +51,7 @@ export function createSearchIndexes(db) {
|
|
|
51
51
|
COALESCE(${contactDisplayNameSql('sender_contacts')}, '')
|
|
52
52
|
FROM messages
|
|
53
53
|
LEFT JOIN chats ON chats.jid = messages.chat_jid
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
${contactLeftJoins('chat_contacts', 'messages.chat_jid')}
|
|
55
|
+
${contactLeftJoins('sender_contacts', 'messages.sender_jid')};
|
|
56
56
|
`);
|
|
57
57
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which roster entries are the archive's owner.
|
|
3
|
+
*
|
|
4
|
+
* Messages I send carry `from_me = 1` and, in a group, the *group's* JID as the
|
|
5
|
+
* sender — my own JID is never written down. So every group roster lists me as
|
|
6
|
+
* a member who has never spoken, and a lurker report proudly named the person
|
|
7
|
+
* reading it as the quietest member of all 100 of their groups.
|
|
8
|
+
*
|
|
9
|
+
* There is no self column to read, but there is a structural giveaway: I am a
|
|
10
|
+
* member of every group I post in, and nobody else is. Taking the participants
|
|
11
|
+
* present in *all* of them is decisive in practice — on a real archive the two
|
|
12
|
+
* hits (a phone JID and its `@lid` twin, both mine) covered 100 of 100 groups
|
|
13
|
+
* while the next-most-common member reached 33.
|
|
14
|
+
*
|
|
15
|
+
* Below MIN_GROUPS the intersection means little: in two shared groups a spouse
|
|
16
|
+
* is indistinguishable from the account owner, so the answer is "unknown" and
|
|
17
|
+
* callers fall back to counting me as an ordinary member.
|
|
18
|
+
*/
|
|
19
|
+
const MIN_GROUPS = 3;
|
|
20
|
+
export function resolveSelfJids(db) {
|
|
21
|
+
const total = db.prepare(`
|
|
22
|
+
SELECT COUNT(DISTINCT messages.chat_jid) AS count
|
|
23
|
+
FROM messages
|
|
24
|
+
JOIN chats ON chats.jid = messages.chat_jid
|
|
25
|
+
WHERE messages.from_me = 1 AND chats.kind = 'group'
|
|
26
|
+
`).get().count;
|
|
27
|
+
if (total < MIN_GROUPS)
|
|
28
|
+
return new Set();
|
|
29
|
+
const rows = db.prepare(`
|
|
30
|
+
SELECT group_participants.user_jid AS jid
|
|
31
|
+
FROM group_participants
|
|
32
|
+
WHERE group_participants.group_jid IN (
|
|
33
|
+
SELECT DISTINCT messages.chat_jid
|
|
34
|
+
FROM messages
|
|
35
|
+
JOIN chats ON chats.jid = messages.chat_jid
|
|
36
|
+
WHERE messages.from_me = 1 AND chats.kind = 'group'
|
|
37
|
+
)
|
|
38
|
+
GROUP BY group_participants.user_jid
|
|
39
|
+
HAVING COUNT(DISTINCT group_participants.group_jid) = @total
|
|
40
|
+
`).all({ total });
|
|
41
|
+
return new Set(rows.map((row) => row.jid));
|
|
42
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import { getResolvedPaths } from '../runtimePaths.js';
|
|
4
|
+
/**
|
|
5
|
+
* Stats responses are pure functions of (archive contents, query params) and the
|
|
6
|
+
* archive is opened read-only, so a result stays valid until the DB file changes.
|
|
7
|
+
*
|
|
8
|
+
* Two things can invalidate an entry:
|
|
9
|
+
* - the archive fingerprint (path + mtime + size) changes — a re-sync, or the
|
|
10
|
+
* user pointing the dashboard at a different DB via Settings;
|
|
11
|
+
* - the time bucket rolls over — period windows like "last week" are relative
|
|
12
|
+
* to now, so entries expire after STATS_CACHE_TTL_MS.
|
|
13
|
+
*
|
|
14
|
+
* Both live in the key prefix, so a change drops every stale entry at once.
|
|
15
|
+
*/
|
|
16
|
+
export const STATS_CACHE_TTL_MS = 5 * 60 * 1000;
|
|
17
|
+
const MAX_ENTRIES = 100;
|
|
18
|
+
const cache = new Map();
|
|
19
|
+
let currentPrefix = '';
|
|
20
|
+
function archiveFingerprint() {
|
|
21
|
+
const { primaryDb } = getResolvedPaths();
|
|
22
|
+
try {
|
|
23
|
+
const stat = fs.statSync(primaryDb);
|
|
24
|
+
return `${primaryDb}:${stat.mtimeMs}:${stat.size}`;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return `${primaryDb}:unavailable`;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Stable, collision-free encoding of the query params a route was called with. */
|
|
31
|
+
function serializeParams(params) {
|
|
32
|
+
const entries = Object.entries(params)
|
|
33
|
+
.filter(([, value]) => value !== undefined)
|
|
34
|
+
.map(([key, value]) => [key, JSON.stringify(value) ?? 'null'])
|
|
35
|
+
.sort(([a], [b]) => a.localeCompare(b));
|
|
36
|
+
return JSON.stringify(entries);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Build the cache key for a stats request, dropping the cache first if the
|
|
40
|
+
* archive or the time bucket has moved on since the last call.
|
|
41
|
+
*/
|
|
42
|
+
export function statsCacheKey(route, params, now = Date.now()) {
|
|
43
|
+
const prefix = `${archiveFingerprint()}|${Math.floor(now / STATS_CACHE_TTL_MS)}`;
|
|
44
|
+
if (prefix !== currentPrefix) {
|
|
45
|
+
cache.clear();
|
|
46
|
+
currentPrefix = prefix;
|
|
47
|
+
}
|
|
48
|
+
return `${prefix}|${route}|${serializeParams(params)}`;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Strong validator derived from the key. The key already pins the archive state
|
|
52
|
+
* and the params, so a matching If-None-Match means the client holds the current
|
|
53
|
+
* value — we can answer 304 without opening the database at all.
|
|
54
|
+
*/
|
|
55
|
+
export function etagFor(key) {
|
|
56
|
+
return `"${createHash('sha1').update(key).digest('base64url')}"`;
|
|
57
|
+
}
|
|
58
|
+
/** Memoize `compute` under `key`. Thrown errors are never cached. */
|
|
59
|
+
export function getCached(key, compute) {
|
|
60
|
+
if (cache.has(key)) {
|
|
61
|
+
const hit = cache.get(key);
|
|
62
|
+
// Re-insert to refresh LRU position.
|
|
63
|
+
cache.delete(key);
|
|
64
|
+
cache.set(key, hit);
|
|
65
|
+
return hit;
|
|
66
|
+
}
|
|
67
|
+
const value = compute();
|
|
68
|
+
cache.set(key, value);
|
|
69
|
+
while (cache.size > MAX_ENTRIES) {
|
|
70
|
+
const oldest = cache.keys().next().value;
|
|
71
|
+
if (oldest === undefined)
|
|
72
|
+
break;
|
|
73
|
+
cache.delete(oldest);
|
|
74
|
+
}
|
|
75
|
+
return value;
|
|
76
|
+
}
|
|
77
|
+
export function clearStatsCache() {
|
|
78
|
+
cache.clear();
|
|
79
|
+
currentPrefix = '';
|
|
80
|
+
}
|
|
81
|
+
export function statsCacheSize() {
|
|
82
|
+
return cache.size;
|
|
83
|
+
}
|
|
@@ -24,15 +24,130 @@ export function resolveStatsTimeZone(raw) {
|
|
|
24
24
|
return 'UTC';
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
|
+
const formattersCache = new Map();
|
|
28
|
+
function getFormatter(timeZone) {
|
|
29
|
+
let formatter = formattersCache.get(timeZone);
|
|
30
|
+
if (!formatter) {
|
|
31
|
+
formatter = new Intl.DateTimeFormat('en-GB', {
|
|
32
|
+
timeZone,
|
|
33
|
+
hour: 'numeric',
|
|
34
|
+
hourCycle: 'h23',
|
|
35
|
+
});
|
|
36
|
+
formattersCache.set(timeZone, formatter);
|
|
37
|
+
}
|
|
38
|
+
return formatter;
|
|
39
|
+
}
|
|
27
40
|
/** Calendar hour 0–23 for Unix epoch seconds in the given IANA time zone. */
|
|
28
41
|
export function hourInTimezone(epochSeconds, timeZone) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
42
|
+
if (timeZone === 'UTC') {
|
|
43
|
+
return new Date(epochSeconds * 1000).getUTCHours();
|
|
44
|
+
}
|
|
45
|
+
const formatter = getFormatter(timeZone);
|
|
46
|
+
const parts = formatter.formatToParts(new Date(epochSeconds * 1000));
|
|
34
47
|
const value = Number(parts.find((p) => p.type === 'hour')?.value);
|
|
35
48
|
if (!Number.isFinite(value))
|
|
36
49
|
return 0;
|
|
37
50
|
return Math.min(23, Math.max(0, value));
|
|
38
51
|
}
|
|
52
|
+
const weekdayHourFormattersCache = new Map();
|
|
53
|
+
const WEEKDAY_INDEX = { Sun: 0, Mon: 1, Tue: 2, Wed: 3, Thu: 4, Fri: 5, Sat: 6 };
|
|
54
|
+
/**
|
|
55
|
+
* Weekday (0 = Sunday) and hour 0–23 for Unix epoch seconds in `timeZone`.
|
|
56
|
+
*
|
|
57
|
+
* One formatter call yields both, which matters for the 168-cell rhythm grid:
|
|
58
|
+
* it reads every message in the period, and two Intl round-trips per row
|
|
59
|
+
* doubled the cost for no gain.
|
|
60
|
+
*/
|
|
61
|
+
export function weekdayHourInTimezone(epochSeconds, timeZone) {
|
|
62
|
+
const date = new Date(epochSeconds * 1000);
|
|
63
|
+
if (timeZone === 'UTC') {
|
|
64
|
+
return { weekday: date.getUTCDay(), hour: date.getUTCHours() };
|
|
65
|
+
}
|
|
66
|
+
let formatter = weekdayHourFormattersCache.get(timeZone);
|
|
67
|
+
if (!formatter) {
|
|
68
|
+
formatter = new Intl.DateTimeFormat('en-US', { timeZone, weekday: 'short', hour: 'numeric', hourCycle: 'h23' });
|
|
69
|
+
weekdayHourFormattersCache.set(timeZone, formatter);
|
|
70
|
+
}
|
|
71
|
+
const parts = formatter.formatToParts(date);
|
|
72
|
+
const weekday = WEEKDAY_INDEX[parts.find((p) => p.type === 'weekday')?.value ?? ''] ?? 0;
|
|
73
|
+
const hour = Number(parts.find((p) => p.type === 'hour')?.value);
|
|
74
|
+
return { weekday, hour: Number.isFinite(hour) ? Math.min(23, Math.max(0, hour)) : 0 };
|
|
75
|
+
}
|
|
76
|
+
const dateFormattersCache = new Map();
|
|
77
|
+
function getDateFormatter(timeZone) {
|
|
78
|
+
let formatter = dateFormattersCache.get(timeZone);
|
|
79
|
+
if (!formatter) {
|
|
80
|
+
// en-CA renders as YYYY-MM-DD, which is the shape every bucket key uses.
|
|
81
|
+
formatter = new Intl.DateTimeFormat('en-CA', {
|
|
82
|
+
timeZone,
|
|
83
|
+
year: 'numeric',
|
|
84
|
+
month: '2-digit',
|
|
85
|
+
day: '2-digit',
|
|
86
|
+
});
|
|
87
|
+
dateFormattersCache.set(timeZone, formatter);
|
|
88
|
+
}
|
|
89
|
+
return formatter;
|
|
90
|
+
}
|
|
91
|
+
/** Calendar date as YYYY-MM-DD for Unix epoch seconds in the given IANA time zone. */
|
|
92
|
+
export function dateInTimezone(epochSeconds, timeZone) {
|
|
93
|
+
if (timeZone === 'UTC') {
|
|
94
|
+
return new Date(epochSeconds * 1000).toISOString().slice(0, 10);
|
|
95
|
+
}
|
|
96
|
+
return getDateFormatter(timeZone).format(new Date(epochSeconds * 1000));
|
|
97
|
+
}
|
|
98
|
+
/** Calendar month as YYYY-MM for Unix epoch seconds in the given IANA time zone. */
|
|
99
|
+
export function monthInTimezone(epochSeconds, timeZone) {
|
|
100
|
+
return dateInTimezone(epochSeconds, timeZone).slice(0, 7);
|
|
101
|
+
}
|
|
102
|
+
/** Zone's UTC offset in minutes at a given instant (positive east of Greenwich). */
|
|
103
|
+
export function timeZoneOffsetMinutes(timeZone, at = new Date()) {
|
|
104
|
+
if (timeZone === 'UTC')
|
|
105
|
+
return 0;
|
|
106
|
+
const parts = new Intl.DateTimeFormat('en-US', {
|
|
107
|
+
timeZone,
|
|
108
|
+
hourCycle: 'h23',
|
|
109
|
+
year: 'numeric',
|
|
110
|
+
month: '2-digit',
|
|
111
|
+
day: '2-digit',
|
|
112
|
+
hour: '2-digit',
|
|
113
|
+
minute: '2-digit',
|
|
114
|
+
second: '2-digit',
|
|
115
|
+
}).formatToParts(at);
|
|
116
|
+
const field = (type) => Number(parts.find((p) => p.type === type)?.value);
|
|
117
|
+
const asUtc = Date.UTC(field('year'), field('month') - 1, field('day'), field('hour'), field('minute'), field('second'));
|
|
118
|
+
if (!Number.isFinite(asUtc))
|
|
119
|
+
return 0;
|
|
120
|
+
// Sub-second drift between the two readings is irrelevant at minute resolution.
|
|
121
|
+
return Math.round((asUtc - Math.floor(at.getTime() / 1000) * 1000) / 60_000);
|
|
122
|
+
}
|
|
123
|
+
/** IANA zone the API process itself is running in. */
|
|
124
|
+
export const SERVER_TIME_ZONE = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
125
|
+
/**
|
|
126
|
+
* SQLite modifiers that move a `ts` column into `timeZone`'s calendar.
|
|
127
|
+
*
|
|
128
|
+
* SQLite ships no IANA database, so only two conversions are exact: UTC, and
|
|
129
|
+
* the server's own zone via `'localtime'` — which *is* DST-correct because
|
|
130
|
+
* SQLite defers to the C library. This dashboard is served from the same
|
|
131
|
+
* machine as the browser, so the requested zone is the server zone in practice
|
|
132
|
+
* and that branch is the one that runs.
|
|
133
|
+
*
|
|
134
|
+
* Any other zone falls back to the zone's offset *right now*, applied uniformly.
|
|
135
|
+
* That misfiles messages sent within `offset` hours of midnight on historical
|
|
136
|
+
* dates whose DST state differs from today's — a far smaller error than the
|
|
137
|
+
* blanket UTC bucketing this replaces.
|
|
138
|
+
*/
|
|
139
|
+
export function timeZoneModifiers(timeZone) {
|
|
140
|
+
if (timeZone === 'UTC')
|
|
141
|
+
return [];
|
|
142
|
+
if (timeZone === SERVER_TIME_ZONE)
|
|
143
|
+
return ["'localtime'"];
|
|
144
|
+
const offset = timeZoneOffsetMinutes(timeZone);
|
|
145
|
+
if (offset === 0)
|
|
146
|
+
return [];
|
|
147
|
+
// Numeric, so it can never carry anything injectable into the SQL text.
|
|
148
|
+
return [`'${offset > 0 ? '+' : '-'}${Math.abs(offset)} minutes'`];
|
|
149
|
+
}
|
|
150
|
+
/** `date(...)`/`strftime(...)` argument list for a timestamp column in `timeZone`. */
|
|
151
|
+
export function tsModifierSql(timeZone, column = 'ts') {
|
|
152
|
+
return [column, "'unixepoch'", ...timeZoneModifiers(timeZone)].join(', ');
|
|
153
|
+
}
|