wacrawl-dashboard 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +220 -89
- package/apps/api/dist/demo/demoArchive.js +371 -0
- package/apps/api/dist/demo/demoCast.js +138 -0
- package/apps/api/dist/demo/demoMedia.js +108 -0
- package/apps/api/dist/index.js +109 -29
- package/apps/api/dist/lib/archiveSchema.js +57 -0
- package/apps/api/dist/lib/archiveSync.js +151 -0
- package/apps/api/dist/lib/chatKind.js +41 -0
- package/apps/api/dist/lib/displayName.js +37 -3
- package/apps/api/dist/lib/mediaFsPath.js +27 -0
- package/apps/api/dist/lib/percentile.js +23 -0
- package/apps/api/dist/lib/query.js +33 -7
- package/apps/api/dist/lib/rollupDb.js +253 -0
- package/apps/api/dist/lib/searchFilters.js +141 -0
- package/apps/api/dist/lib/searchIndex.js +3 -3
- package/apps/api/dist/lib/selfIdentity.js +42 -0
- package/apps/api/dist/lib/statsCache.js +83 -0
- package/apps/api/dist/lib/timezone.js +120 -5
- package/apps/api/dist/lib/urlExtract.js +111 -0
- package/apps/api/dist/lib/wordCloudUsefulStopWords.js +36 -3
- package/apps/api/dist/pathsResolve.js +17 -3
- package/apps/api/dist/routes/data.js +168 -78
- package/apps/api/dist/routes/settings.js +3 -0
- package/apps/api/dist/routes/stats/profiles.js +522 -0
- package/apps/api/dist/routes/stats/reports.js +555 -0
- package/apps/api/dist/routes/stats/shared.js +75 -0
- package/apps/api/dist/routes/stats.js +486 -97
- package/apps/api/dist/routes/sync.js +60 -0
- package/apps/api/dist/scripts/generateDemoArchive.js +24 -0
- package/apps/api/package.json +31 -0
- package/apps/web/dist/apple-touch-icon.png +0 -0
- package/apps/web/dist/assets/charts-BMGh5n-d.js +36 -0
- package/apps/web/dist/assets/icons-BdElrSaU.js +1 -0
- package/apps/web/dist/assets/index-Bb8Zj42D.css +1 -0
- package/apps/web/dist/assets/index-wrxENU1e.js +9 -0
- package/apps/web/dist/assets/react-D3RL6ob4.js +3 -0
- package/apps/web/dist/favicon-16x16.png +0 -0
- package/apps/web/dist/favicon-32x32.png +0 -0
- package/apps/web/dist/favicon-512x512.png +0 -0
- package/apps/web/dist/favicon.ico +0 -0
- package/apps/web/dist/favicon.svg +59 -0
- package/apps/web/dist/index.html +11 -2
- package/bin/wacrawl-dashboard.js +64 -15
- package/package.json +25 -4
- package/apps/web/dist/assets/index-1fGu93mF.css +0 -1
- package/apps/web/dist/assets/index-Dfsoi6vG.js +0 -43
package/apps/api/dist/index.js
CHANGED
|
@@ -3,27 +3,103 @@ import 'dotenv/config';
|
|
|
3
3
|
import express from 'express';
|
|
4
4
|
import { ensurePrimaryDatabase } from './db.js';
|
|
5
5
|
import { isMediaRootAccessible } from './lib/mediaFsPath.js';
|
|
6
|
+
import { resolvePathsFromEnv } from './pathsResolve.js';
|
|
6
7
|
import { dataRouter } from './routes/data.js';
|
|
7
8
|
import { settingsRouter } from './routes/settings.js';
|
|
8
9
|
import { statsRouter } from './routes/stats.js';
|
|
10
|
+
import { syncRouter } from './routes/sync.js';
|
|
9
11
|
import { getResolvedPaths, pathsMiddleware } from './runtimePaths.js';
|
|
10
12
|
const PORT = Number(process.env.PORT ?? 3001);
|
|
11
13
|
const HOST = '127.0.0.1';
|
|
12
|
-
const ALLOWED_ORIGINS = new Set([
|
|
13
|
-
'http://localhost:5173',
|
|
14
|
-
'http://127.0.0.1:5173',
|
|
15
|
-
'http://localhost:4173',
|
|
16
|
-
'http://127.0.0.1:4173',
|
|
17
|
-
// Also allow same-origin requests when API serves the web frontend directly
|
|
18
|
-
`http://localhost:${PORT}`,
|
|
19
|
-
`http://127.0.0.1:${PORT}`,
|
|
20
|
-
]);
|
|
21
14
|
const ALLOWED_HOSTS = new Set([
|
|
22
15
|
`localhost:${PORT}`,
|
|
23
16
|
`127.0.0.1:${PORT}`,
|
|
24
17
|
]);
|
|
18
|
+
const LOOPBACK_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]', '::1']);
|
|
19
|
+
/**
|
|
20
|
+
* Accept any origin served from this machine. Development only: see createApp.
|
|
21
|
+
*
|
|
22
|
+
* Pinning the allowlist to ports 5173/4173 was brittle: Vite silently moves to
|
|
23
|
+
* 5174 when 5173 is already taken, and from then on every request failed with an
|
|
24
|
+
* opaque 500 that surfaced in the UI as "Cannot connect to the WaCrawl API".
|
|
25
|
+
*/
|
|
26
|
+
export function isLoopbackOrigin(origin) {
|
|
27
|
+
try {
|
|
28
|
+
const { hostname, protocol } = new URL(origin);
|
|
29
|
+
if (protocol !== 'http:' && protocol !== 'https:')
|
|
30
|
+
return false;
|
|
31
|
+
return LOOPBACK_HOSTNAMES.has(hostname);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** The dashboard this server is hosting, under either name for loopback. */
|
|
38
|
+
function isOwnOrigin(origin) {
|
|
39
|
+
try {
|
|
40
|
+
const { host, protocol } = new URL(origin);
|
|
41
|
+
return protocol === 'http:' && ALLOWED_HOSTS.has(host);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export const errorHandler = (error, _req, res, next) => {
|
|
48
|
+
// Once the response is on the wire there is no way to replace it with a JSON
|
|
49
|
+
// error — writing anyway throws ERR_HTTP_HEADERS_SENT. Hand back to Express so
|
|
50
|
+
// its final handler destroys the socket instead.
|
|
51
|
+
if (res.headersSent) {
|
|
52
|
+
next(error);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (error?.code === 'FORBIDDEN_ORIGIN') {
|
|
56
|
+
// Was an opaque 500 before, which the UI reported as "Cannot connect to the
|
|
57
|
+
// WaCrawl API" with no hint that the origin was the problem.
|
|
58
|
+
res.status(403).json({
|
|
59
|
+
error: {
|
|
60
|
+
code: 'FORBIDDEN_ORIGIN',
|
|
61
|
+
message: `Blocked request from origin ${error.origin ?? 'unknown'}. Only the dashboard itself may call this API.`,
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (error?.code === 'CHAT_NOT_FOUND') {
|
|
67
|
+
res.status(404).json({
|
|
68
|
+
error: { code: 'CHAT_NOT_FOUND', message: error.message ?? 'No such chat.' },
|
|
69
|
+
});
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (error?.code === 'NOT_A_GROUP') {
|
|
73
|
+
res.status(400).json({
|
|
74
|
+
error: { code: 'NOT_A_GROUP', message: error.message ?? 'That chat is not a group.' },
|
|
75
|
+
});
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (error?.code === 'VALIDATION_ERROR') {
|
|
79
|
+
res.status(400).json({
|
|
80
|
+
error: { code: 'VALIDATION_ERROR', message: error.message ?? 'Invalid request.' },
|
|
81
|
+
});
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
const code = error?.code === 'SQLITE_CANTOPEN' ? 'DATABASE_UNAVAILABLE' : 'INTERNAL_ERROR';
|
|
85
|
+
const message = code === 'DATABASE_UNAVAILABLE'
|
|
86
|
+
? 'Cannot open the WaCrawl database. Check Settings → Archive paths, WACRAWL_DB in .env, or run wacrawl sync.'
|
|
87
|
+
: 'Unexpected server error.';
|
|
88
|
+
res.status(code === 'DATABASE_UNAVAILABLE' ? 503 : 500).json({
|
|
89
|
+
error: {
|
|
90
|
+
code,
|
|
91
|
+
message,
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
};
|
|
25
95
|
export function createApp() {
|
|
26
96
|
const app = express();
|
|
97
|
+
// Production static file serving — set WACRAWL_STATIC_DIR to the web dist folder
|
|
98
|
+
const staticDir = process.env.WACRAWL_STATIC_DIR;
|
|
99
|
+
// A site on another local port (MAMP, any dev server) runs code the archive
|
|
100
|
+
// owner did not write, so it must not read the archive. Only development needs
|
|
101
|
+
// the wider rule, because the UI then comes from Vite on a port that can move.
|
|
102
|
+
const isAllowedOrigin = staticDir ? isOwnOrigin : isLoopbackOrigin;
|
|
27
103
|
app.disable('x-powered-by');
|
|
28
104
|
app.use(express.json({ limit: '64kb' }));
|
|
29
105
|
app.use((req, res, next) => {
|
|
@@ -41,11 +117,16 @@ export function createApp() {
|
|
|
41
117
|
});
|
|
42
118
|
app.use(cors({
|
|
43
119
|
origin(origin, callback) {
|
|
44
|
-
|
|
120
|
+
// No Origin header: same-origin, or a non-browser client such as curl or
|
|
121
|
+
// an <img> tag pointing at /api/media/file.
|
|
122
|
+
if (!origin || isAllowedOrigin(origin)) {
|
|
45
123
|
callback(null, true);
|
|
46
124
|
return;
|
|
47
125
|
}
|
|
48
|
-
callback(new Error('Origin is not allowed by CORS.')
|
|
126
|
+
callback(Object.assign(new Error('Origin is not allowed by CORS.'), {
|
|
127
|
+
code: 'FORBIDDEN_ORIGIN',
|
|
128
|
+
origin,
|
|
129
|
+
}));
|
|
49
130
|
},
|
|
50
131
|
}));
|
|
51
132
|
app.use(pathsMiddleware);
|
|
@@ -57,11 +138,14 @@ export function createApp() {
|
|
|
57
138
|
primaryDbReadable = true;
|
|
58
139
|
}
|
|
59
140
|
catch {
|
|
60
|
-
|
|
141
|
+
// Left false — /api/health reports the archive as unreachable.
|
|
61
142
|
}
|
|
62
143
|
const mediaStatus = isMediaRootAccessible();
|
|
63
144
|
res.json({
|
|
64
145
|
ok: primaryDbReadable,
|
|
146
|
+
// Settings can repoint a running demo at another archive, and a badge
|
|
147
|
+
// calling real messages "demo data" would be worse than no badge.
|
|
148
|
+
demo: process.env.WACRAWL_DEMO === '1' && paths.primaryDb === resolvePathsFromEnv().primaryDb,
|
|
65
149
|
dbPath: paths.primaryDb,
|
|
66
150
|
paths,
|
|
67
151
|
mediaAccessible: mediaStatus.accessible,
|
|
@@ -70,6 +154,7 @@ export function createApp() {
|
|
|
70
154
|
});
|
|
71
155
|
app.use('/api/settings', settingsRouter);
|
|
72
156
|
app.use('/api/stats', statsRouter);
|
|
157
|
+
app.use('/api/sync', syncRouter);
|
|
73
158
|
app.use('/api', dataRouter);
|
|
74
159
|
// JSON 404 only for /api routes; non-API routes fall through to static serving below
|
|
75
160
|
app.use('/api', (_req, res) => {
|
|
@@ -80,8 +165,6 @@ export function createApp() {
|
|
|
80
165
|
},
|
|
81
166
|
});
|
|
82
167
|
});
|
|
83
|
-
// Production static file serving — set WACRAWL_STATIC_DIR to the web dist folder
|
|
84
|
-
const staticDir = process.env.WACRAWL_STATIC_DIR;
|
|
85
168
|
if (staticDir) {
|
|
86
169
|
app.use(express.static(staticDir));
|
|
87
170
|
// SPA fallback: unmatched non-API GET requests return index.html
|
|
@@ -93,23 +176,20 @@ export function createApp() {
|
|
|
93
176
|
res.sendFile('index.html', { root: staticDir });
|
|
94
177
|
});
|
|
95
178
|
}
|
|
96
|
-
const errorHandler = (error, _req, res, _next) => {
|
|
97
|
-
const code = error?.code === 'SQLITE_CANTOPEN' ? 'DATABASE_UNAVAILABLE' : 'INTERNAL_ERROR';
|
|
98
|
-
const message = code === 'DATABASE_UNAVAILABLE'
|
|
99
|
-
? 'Cannot open the WaCrawl database. Check Settings → Archive paths, WACRAWL_DB in .env, or run wacrawl sync.'
|
|
100
|
-
: 'Unexpected server error.';
|
|
101
|
-
res.status(code === 'DATABASE_UNAVAILABLE' ? 503 : 500).json({
|
|
102
|
-
error: {
|
|
103
|
-
code,
|
|
104
|
-
message,
|
|
105
|
-
},
|
|
106
|
-
});
|
|
107
|
-
};
|
|
108
179
|
app.use(errorHandler);
|
|
109
180
|
return app;
|
|
110
181
|
}
|
|
111
|
-
|
|
112
|
-
|
|
182
|
+
/** Express 5 passes listen errors, such as a taken port, to the listen callback. */
|
|
183
|
+
export function reportListenResult(error) {
|
|
184
|
+
if (!error) {
|
|
113
185
|
console.info(`WaCrawl API listening on http://${HOST}:${PORT}`);
|
|
114
|
-
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
console.error(error.code === 'EADDRINUSE'
|
|
189
|
+
? `Port ${PORT} is already in use. WaCrawl may already be running at http://${HOST}:${PORT}. Otherwise, choose another port with PORT=<port>.`
|
|
190
|
+
: `Could not start the WaCrawl API: ${error.message}`);
|
|
191
|
+
process.exitCode = 1;
|
|
192
|
+
}
|
|
193
|
+
if (process.env.NODE_ENV !== 'test') {
|
|
194
|
+
createApp().listen(PORT, HOST, reportListenResult);
|
|
115
195
|
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The tables every wacrawl archive has, and all the dashboard requires.
|
|
3
|
+
*
|
|
4
|
+
* Shared by the test fixtures and the demo archive so they cannot drift apart.
|
|
5
|
+
* Newer wacrawl versions add to this (`chats.archived`, `sync_state`,
|
|
6
|
+
* `messages_fts`); the queries probe for those, so they stay out of the baseline.
|
|
7
|
+
*/
|
|
8
|
+
export const ARCHIVE_SCHEMA_SQL = `
|
|
9
|
+
CREATE TABLE chats (
|
|
10
|
+
jid TEXT PRIMARY KEY,
|
|
11
|
+
kind TEXT NOT NULL,
|
|
12
|
+
name TEXT,
|
|
13
|
+
last_message_at INTEGER
|
|
14
|
+
);
|
|
15
|
+
CREATE TABLE contacts (
|
|
16
|
+
jid TEXT PRIMARY KEY,
|
|
17
|
+
phone TEXT,
|
|
18
|
+
full_name TEXT,
|
|
19
|
+
first_name TEXT,
|
|
20
|
+
last_name TEXT,
|
|
21
|
+
business_name TEXT,
|
|
22
|
+
username TEXT,
|
|
23
|
+
lid TEXT
|
|
24
|
+
);
|
|
25
|
+
CREATE TABLE groups (
|
|
26
|
+
jid TEXT PRIMARY KEY,
|
|
27
|
+
name TEXT,
|
|
28
|
+
owner_jid TEXT,
|
|
29
|
+
created_at INTEGER
|
|
30
|
+
);
|
|
31
|
+
CREATE TABLE group_participants (
|
|
32
|
+
group_jid TEXT NOT NULL,
|
|
33
|
+
user_jid TEXT NOT NULL,
|
|
34
|
+
contact_name TEXT,
|
|
35
|
+
first_name TEXT,
|
|
36
|
+
is_admin INTEGER NOT NULL DEFAULT 0,
|
|
37
|
+
is_active INTEGER NOT NULL DEFAULT 0,
|
|
38
|
+
PRIMARY KEY (group_jid, user_jid)
|
|
39
|
+
);
|
|
40
|
+
CREATE TABLE messages (
|
|
41
|
+
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
42
|
+
source_pk INTEGER NOT NULL UNIQUE,
|
|
43
|
+
chat_jid TEXT NOT NULL,
|
|
44
|
+
chat_name TEXT,
|
|
45
|
+
msg_id TEXT NOT NULL,
|
|
46
|
+
sender_jid TEXT,
|
|
47
|
+
sender_name TEXT,
|
|
48
|
+
ts INTEGER NOT NULL,
|
|
49
|
+
from_me INTEGER NOT NULL,
|
|
50
|
+
text TEXT,
|
|
51
|
+
raw_type INTEGER NOT NULL,
|
|
52
|
+
message_type TEXT,
|
|
53
|
+
media_type TEXT,
|
|
54
|
+
media_path TEXT,
|
|
55
|
+
media_size INTEGER
|
|
56
|
+
);
|
|
57
|
+
`;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { closeDb, getDb } from '../db.js';
|
|
3
|
+
import { clearStatsCache } from './statsCache.js';
|
|
4
|
+
/** Import can be genuinely slow on a first run; give up rather than leak a process. */
|
|
5
|
+
const SYNC_TIMEOUT_MS = 10 * 60 * 1000;
|
|
6
|
+
/** Enough of the CLI's output to diagnose a failure, without unbounded buffering. */
|
|
7
|
+
const MAX_OUTPUT_BYTES = 8 * 1024;
|
|
8
|
+
const MULTI_IDENTITY_PATTERN = /multiple whatsapp account identities/i;
|
|
9
|
+
const MULTI_IDENTITY_HINT = 'The WhatsApp source holds more than one account identity, so wacrawl refuses to import it automatically. '
|
|
10
|
+
+ 'Run `wacrawl sync --adopt-source` in a terminal to bind this archive to one identity and merge. '
|
|
11
|
+
+ 'The dashboard will not do that for you: it rewrites which account owns the archive.';
|
|
12
|
+
const MISSING_BINARY_HINT = 'Install the wacrawl CLI and make sure it is on PATH, or point WACRAWL_BIN at the executable.';
|
|
13
|
+
const idleJob = {
|
|
14
|
+
state: 'idle',
|
|
15
|
+
startedAt: null,
|
|
16
|
+
finishedAt: null,
|
|
17
|
+
error: null,
|
|
18
|
+
hint: null,
|
|
19
|
+
};
|
|
20
|
+
let job = idleJob;
|
|
21
|
+
let inFlight = null;
|
|
22
|
+
export function syncEnabled() {
|
|
23
|
+
return process.env.WACRAWL_DISABLE_SYNC !== '1';
|
|
24
|
+
}
|
|
25
|
+
export function syncJobStatus() {
|
|
26
|
+
return { ...job };
|
|
27
|
+
}
|
|
28
|
+
/** Test seam: drop job state between cases so each starts from idle. */
|
|
29
|
+
export function resetSyncJob() {
|
|
30
|
+
job = idleJob;
|
|
31
|
+
inFlight = null;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Read the CLI's own bookkeeping table. Older archives predate `sync_state` and
|
|
35
|
+
* a misconfigured path means no archive at all, so every failure degrades to
|
|
36
|
+
* "unknown" rather than breaking the status endpoint.
|
|
37
|
+
*/
|
|
38
|
+
export function readArchiveSyncInfo() {
|
|
39
|
+
try {
|
|
40
|
+
const db = getDb();
|
|
41
|
+
const present = db
|
|
42
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sync_state'")
|
|
43
|
+
.get();
|
|
44
|
+
if (!present) {
|
|
45
|
+
return { lastImportAt: null, sourcePath: null };
|
|
46
|
+
}
|
|
47
|
+
const rows = db.prepare('SELECT key, value FROM sync_state').all();
|
|
48
|
+
const byKey = new Map(rows.map((row) => [row.key, row.value]));
|
|
49
|
+
return {
|
|
50
|
+
lastImportAt: byKey.get('last_import_at') ?? null,
|
|
51
|
+
sourcePath: byKey.get('source_path') ?? null,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return { lastImportAt: null, sourcePath: null };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Start an import unless one is already running.
|
|
60
|
+
*
|
|
61
|
+
* `started: false` means the caller lost the race and should poll the returned
|
|
62
|
+
* job rather than treating it as an error.
|
|
63
|
+
*/
|
|
64
|
+
export function startSync(dbPath) {
|
|
65
|
+
if (inFlight) {
|
|
66
|
+
return { started: false, job: syncJobStatus() };
|
|
67
|
+
}
|
|
68
|
+
job = { state: 'running', startedAt: new Date().toISOString(), finishedAt: null, error: null, hint: null };
|
|
69
|
+
inFlight = runWacrawlSync(dbPath).then((outcome) => {
|
|
70
|
+
if (outcome.ok) {
|
|
71
|
+
// The import rewrote the file underneath a readonly handle, and every
|
|
72
|
+
// cached stat was computed from the old contents.
|
|
73
|
+
closeDb();
|
|
74
|
+
clearStatsCache();
|
|
75
|
+
}
|
|
76
|
+
job = {
|
|
77
|
+
state: outcome.ok ? 'succeeded' : 'failed',
|
|
78
|
+
startedAt: job.startedAt,
|
|
79
|
+
finishedAt: new Date().toISOString(),
|
|
80
|
+
error: outcome.ok ? null : outcome.error,
|
|
81
|
+
hint: outcome.ok ? null : outcome.hint,
|
|
82
|
+
};
|
|
83
|
+
}).finally(() => {
|
|
84
|
+
inFlight = null;
|
|
85
|
+
});
|
|
86
|
+
return { started: true, job: syncJobStatus() };
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Spawned without a shell and with a fixed argument list, so the configured
|
|
90
|
+
* archive path cannot turn into extra commands.
|
|
91
|
+
*/
|
|
92
|
+
function runWacrawlSync(dbPath) {
|
|
93
|
+
return new Promise((resolve) => {
|
|
94
|
+
const bin = process.env.WACRAWL_BIN ?? 'wacrawl';
|
|
95
|
+
let output = '';
|
|
96
|
+
let settled = false;
|
|
97
|
+
function settle(outcome) {
|
|
98
|
+
if (settled)
|
|
99
|
+
return;
|
|
100
|
+
settled = true;
|
|
101
|
+
resolve(outcome);
|
|
102
|
+
}
|
|
103
|
+
const child = spawn(bin, ['--db', dbPath, 'sync'], {
|
|
104
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
105
|
+
timeout: SYNC_TIMEOUT_MS,
|
|
106
|
+
});
|
|
107
|
+
function collect(chunk) {
|
|
108
|
+
if (output.length >= MAX_OUTPUT_BYTES)
|
|
109
|
+
return;
|
|
110
|
+
output += chunk.toString('utf8');
|
|
111
|
+
}
|
|
112
|
+
child.stdout?.on('data', collect);
|
|
113
|
+
child.stderr?.on('data', collect);
|
|
114
|
+
child.on('error', (error) => {
|
|
115
|
+
settle(error.code === 'ENOENT'
|
|
116
|
+
? { ok: false, error: `Cannot run "${bin}": the wacrawl CLI was not found.`, hint: MISSING_BINARY_HINT }
|
|
117
|
+
: { ok: false, error: `Cannot run "${bin}": ${error.message}`, hint: null });
|
|
118
|
+
});
|
|
119
|
+
child.on('close', (code, signal) => {
|
|
120
|
+
if (code === 0) {
|
|
121
|
+
settle({ ok: true, error: null, hint: null });
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
// node's `timeout` option kills the child rather than emitting an error.
|
|
125
|
+
if (signal) {
|
|
126
|
+
settle({
|
|
127
|
+
ok: false,
|
|
128
|
+
error: `wacrawl sync was terminated by ${signal} after ${Math.round(SYNC_TIMEOUT_MS / 60_000)} minutes.`,
|
|
129
|
+
hint: 'Run `wacrawl sync` in a terminal to see how far it gets.',
|
|
130
|
+
});
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
settle({
|
|
134
|
+
ok: false,
|
|
135
|
+
error: summarize(output) || `wacrawl sync exited with code ${code}.`,
|
|
136
|
+
hint: MULTI_IDENTITY_PATTERN.test(output) ? MULTI_IDENTITY_HINT : null,
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* The CLI reports progress on the way to a failure, so the last few non-empty
|
|
143
|
+
* lines carry the reason while the earlier ones are noise.
|
|
144
|
+
*/
|
|
145
|
+
function summarize(output) {
|
|
146
|
+
const lines = output
|
|
147
|
+
.split('\n')
|
|
148
|
+
.map((line) => line.trim())
|
|
149
|
+
.filter(Boolean);
|
|
150
|
+
return lines.slice(-3).join(' — ').slice(0, 500);
|
|
151
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Real archives label one-to-one chats `dm`; the fixtures here — and older
|
|
3
|
+
* wacrawl versions — say `direct`. Filtering on a single literal silently
|
|
4
|
+
* matched nothing on real data, which is how the Chats page ended up listing
|
|
5
|
+
* groups only, and why the dormancy report came back empty for every account.
|
|
6
|
+
*
|
|
7
|
+
* `status` is WhatsApp's own broadcast pseudo-chat. It is neither a person nor
|
|
8
|
+
* a group, so it belongs in neither bucket.
|
|
9
|
+
*/
|
|
10
|
+
export const DIRECT_CHAT_KINDS = ['dm', 'direct'];
|
|
11
|
+
export function parseChatKind(value, fallback = 'all') {
|
|
12
|
+
if (value === 'direct' || value === 'group' || value === 'all')
|
|
13
|
+
return value;
|
|
14
|
+
return fallback;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* SQL predicate for a chat-kind filter, as a bare condition with no leading
|
|
18
|
+
* `AND`/`WHERE` so callers can place it wherever it fits.
|
|
19
|
+
*
|
|
20
|
+
* The literals are compile-time constants rather than bound parameters because
|
|
21
|
+
* this composes into larger statements that are prepared once and reused.
|
|
22
|
+
*/
|
|
23
|
+
export function chatKindSql(kind, column = 'chats.kind') {
|
|
24
|
+
if (kind === 'group')
|
|
25
|
+
return `${column} = 'group'`;
|
|
26
|
+
if (kind === 'direct')
|
|
27
|
+
return `${column} IN ('dm', 'direct')`;
|
|
28
|
+
return `${column} IN ('dm', 'direct', 'group')`;
|
|
29
|
+
}
|
|
30
|
+
/** Older archives predate a table; a missing one should degrade, not throw. */
|
|
31
|
+
export function tableExists(db, name) {
|
|
32
|
+
const row = db
|
|
33
|
+
.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = @name")
|
|
34
|
+
.get({ name });
|
|
35
|
+
return row !== undefined;
|
|
36
|
+
}
|
|
37
|
+
/** Newer wacrawl archives add columns (e.g. `chats.archived`); older ones lack them. */
|
|
38
|
+
export function columnExists(db, table, column) {
|
|
39
|
+
const columns = db.prepare(`PRAGMA table_info(${JSON.stringify(table)})`).all();
|
|
40
|
+
return columns.some((row) => row.name === column);
|
|
41
|
+
}
|
|
@@ -11,7 +11,7 @@ export function cleanDisplayNameSql(expression) {
|
|
|
11
11
|
END
|
|
12
12
|
`;
|
|
13
13
|
}
|
|
14
|
-
export function
|
|
14
|
+
export function contactDisplayNameSingleSql(alias) {
|
|
15
15
|
return `
|
|
16
16
|
COALESCE(
|
|
17
17
|
${cleanDisplayNameSql(`${alias}.full_name`)},
|
|
@@ -23,6 +23,40 @@ export function contactDisplayNameSql(alias) {
|
|
|
23
23
|
)
|
|
24
24
|
`;
|
|
25
25
|
}
|
|
26
|
-
export function
|
|
27
|
-
|
|
26
|
+
export function contactDisplayNameSql(alias, isSplit = true) {
|
|
27
|
+
if (!isSplit) {
|
|
28
|
+
return contactDisplayNameSingleSql(alias);
|
|
29
|
+
}
|
|
30
|
+
return `
|
|
31
|
+
COALESCE(
|
|
32
|
+
${contactDisplayNameSingleSql(`${alias}_jid`)},
|
|
33
|
+
${contactDisplayNameSingleSql(`${alias}_lid`)}
|
|
34
|
+
)
|
|
35
|
+
`;
|
|
36
|
+
}
|
|
37
|
+
export function contactLeftJoins(alias, jidExpression) {
|
|
38
|
+
return `
|
|
39
|
+
LEFT JOIN contacts AS ${alias}_jid ON ${alias}_jid.jid = ${jidExpression}
|
|
40
|
+
LEFT JOIN contacts AS ${alias}_lid ON ${alias}_lid.lid = ${jidExpression}
|
|
41
|
+
`;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The archive identifies one person two ways: a phone JID and an opaque `@lid`.
|
|
45
|
+
* Both turn up as message senders in the same chat, so anything that groups by
|
|
46
|
+
* `sender_jid` reports one person as two — a group leaderboard listed the same
|
|
47
|
+
* contact twice, 865 messages and 77.
|
|
48
|
+
*
|
|
49
|
+
* The contacts table holds the pairing, so joining through it collapses both
|
|
50
|
+
* spellings onto the phone JID. Requires `contactLeftJoins(alias, expression)`
|
|
51
|
+
* on the same expression. Falls back to the raw value for people who are not in
|
|
52
|
+
* contacts at all, which is the best that can be done for them.
|
|
53
|
+
*/
|
|
54
|
+
export function canonicalJidSql(alias, expression) {
|
|
55
|
+
return `COALESCE(${alias}_lid.jid, ${alias}_jid.jid, ${expression})`;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* True when a JID identifies a person rather than a group or broadcast pseudo-chat.
|
|
59
|
+
*/
|
|
60
|
+
export function isPersonJidSql(expression) {
|
|
61
|
+
return `${expression} IS NOT NULL AND ${expression} <> '' AND ${expression} NOT LIKE '%@g.us' AND ${expression} NOT LIKE '%@status' AND ${expression} NOT LIKE '%@broadcast'`;
|
|
28
62
|
}
|
|
@@ -59,6 +59,33 @@ export function resolveArchiveMediaPath(storedPath) {
|
|
|
59
59
|
return fallback;
|
|
60
60
|
return resolved;
|
|
61
61
|
}
|
|
62
|
+
/** Types a browser only displays. SVG is left out: opened directly, it can run script. */
|
|
63
|
+
const INLINE_MEDIA_EXTENSIONS = new Set([
|
|
64
|
+
'.jpg', '.jpeg', '.png', '.gif', '.webp', '.heic', '.heif',
|
|
65
|
+
'.mp4', '.mov', '.webm', '.m4v', '.3gp',
|
|
66
|
+
'.ogg', '.oga', '.opus', '.mp3', '.m4a', '.aac', '.wav',
|
|
67
|
+
'.pdf',
|
|
68
|
+
]);
|
|
69
|
+
/**
|
|
70
|
+
* Attachments are written by whoever sent them but served from the same origin as
|
|
71
|
+
* the API, so an HTML file opened as a page could read the whole archive. Anything
|
|
72
|
+
* not on the list downloads instead. The sandbox gives a page an opaque origin as
|
|
73
|
+
* a second line of defence. <img> and <video> ignore both headers, so thumbnails
|
|
74
|
+
* of unlisted types still render.
|
|
75
|
+
*/
|
|
76
|
+
export function mediaResponseHeaders(filePath) {
|
|
77
|
+
const extension = path.extname(filePath).toLowerCase();
|
|
78
|
+
const headers = { 'X-Content-Type-Options': 'nosniff' };
|
|
79
|
+
// Browser PDF viewers refuse to load in a sandboxed document, and script inside
|
|
80
|
+
// a PDF has no access to the origin.
|
|
81
|
+
if (extension !== '.pdf') {
|
|
82
|
+
headers['Content-Security-Policy'] = 'sandbox';
|
|
83
|
+
}
|
|
84
|
+
if (!INLINE_MEDIA_EXTENSIONS.has(extension)) {
|
|
85
|
+
headers['Content-Disposition'] = `attachment; filename*=UTF-8''${encodeURIComponent(path.basename(filePath))}`;
|
|
86
|
+
}
|
|
87
|
+
return headers;
|
|
88
|
+
}
|
|
62
89
|
/** Check whether the configured media root is readable. */
|
|
63
90
|
export function isMediaRootAccessible() {
|
|
64
91
|
const root = getArchiveMediaRoot();
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nearest-rank percentile over an ascending-sorted array.
|
|
3
|
+
*
|
|
4
|
+
* Reply latencies are heavily right-skewed — a single reply three weeks later
|
|
5
|
+
* drags a mean past anything the conversation actually felt like — so the
|
|
6
|
+
* report layer quotes p50/p90 instead of an average.
|
|
7
|
+
*/
|
|
8
|
+
export function percentile(sortedAscending, p) {
|
|
9
|
+
if (sortedAscending.length === 0)
|
|
10
|
+
return 0;
|
|
11
|
+
const rank = Math.ceil((p / 100) * sortedAscending.length);
|
|
12
|
+
const index = Math.min(sortedAscending.length - 1, Math.max(0, rank - 1));
|
|
13
|
+
return sortedAscending[index];
|
|
14
|
+
}
|
|
15
|
+
/** Arithmetic mean, 0 for an empty array. */
|
|
16
|
+
export function mean(values) {
|
|
17
|
+
if (values.length === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
let total = 0;
|
|
20
|
+
for (const value of values)
|
|
21
|
+
total += value;
|
|
22
|
+
return total / values.length;
|
|
23
|
+
}
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { getUnixTime } from 'date-fns';
|
|
2
|
+
import { timeZoneOffsetMinutes, tsModifierSql } from './timezone.js';
|
|
3
|
+
/**
|
|
4
|
+
* Describe a capped scan so the UI can say the numbers are partial.
|
|
5
|
+
*
|
|
6
|
+
* `scanned === limit` cannot distinguish "hit the cap" from "happens to have
|
|
7
|
+
* exactly that many rows"; erring towards claiming truncation is the safe
|
|
8
|
+
* direction, since the alternative is silently overstating coverage.
|
|
9
|
+
*/
|
|
10
|
+
export function scanCoverage(scanned, limit) {
|
|
11
|
+
return { scanned, limit, truncated: scanned >= limit };
|
|
12
|
+
}
|
|
2
13
|
const PERIOD_SECONDS = {
|
|
3
14
|
day: 86_400,
|
|
4
15
|
week: 604_800,
|
|
@@ -11,6 +22,11 @@ export function parsePeriod(value) {
|
|
|
11
22
|
}
|
|
12
23
|
return 'all';
|
|
13
24
|
}
|
|
25
|
+
/** `%value%` for a LIKE … ESCAPE '\\' test, or undefined when there is nothing to filter on. */
|
|
26
|
+
export function nameLikePattern(value) {
|
|
27
|
+
const text = typeof value === 'string' ? value.trim() : '';
|
|
28
|
+
return text ? `%${text.replace(/[\\%_]/g, (match) => `\\${match}`)}%` : undefined;
|
|
29
|
+
}
|
|
14
30
|
export function parseGranularity(value, fallback) {
|
|
15
31
|
if (value === 'day' || value === 'week' || value === 'month') {
|
|
16
32
|
return value;
|
|
@@ -51,18 +67,28 @@ export function unixSecondsToIso(seconds) {
|
|
|
51
67
|
}
|
|
52
68
|
return new Date(seconds * 1000).toISOString();
|
|
53
69
|
}
|
|
54
|
-
export function bucketDateSql(granularity) {
|
|
70
|
+
export function bucketDateSql(granularity, timeZone = 'UTC') {
|
|
71
|
+
const ts = tsModifierSql(timeZone);
|
|
55
72
|
if (granularity === 'month') {
|
|
56
|
-
return
|
|
73
|
+
return `strftime('%Y-%m-01', ${ts})`;
|
|
57
74
|
}
|
|
58
75
|
if (granularity === 'week') {
|
|
59
|
-
return
|
|
76
|
+
return `date(${ts}, 'weekday 1', '-7 days')`;
|
|
60
77
|
}
|
|
61
|
-
return
|
|
78
|
+
return `date(${ts})`;
|
|
62
79
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Unix bounds of a calendar year *as the viewer experiences it*.
|
|
82
|
+
*
|
|
83
|
+
* The offset is read at each boundary rather than once, so a year that starts
|
|
84
|
+
* and ends on opposite sides of a DST transition still lines up with the
|
|
85
|
+
* timezone-aware bucketing in `bucketDateSql`.
|
|
86
|
+
*/
|
|
87
|
+
export function yearBounds(year, timeZone = 'UTC') {
|
|
88
|
+
const utcStart = new Date(Date.UTC(year, 0, 1));
|
|
89
|
+
const utcEnd = new Date(Date.UTC(year + 1, 0, 1));
|
|
90
|
+
const start = getUnixTime(utcStart) - timeZoneOffsetMinutes(timeZone, utcStart) * 60;
|
|
91
|
+
const end = getUnixTime(utcEnd) - timeZoneOffsetMinutes(timeZone, utcEnd) * 60;
|
|
66
92
|
return { start, end };
|
|
67
93
|
}
|
|
68
94
|
export function normalizeBucketDate(date, granularity) {
|