tacilytics 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -22
- package/auth.js +49 -15
- package/datacenter.js +232 -0
- package/package.json +2 -1
- package/public/account.js +19 -5
- package/public/app.js +5 -2
- package/public/i18n.js +7 -1
- package/public/index.html +7 -5
- package/server.js +119 -26
package/README.md
CHANGED
|
@@ -14,7 +14,11 @@
|
|
|
14
14
|

|
|
15
15
|

|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
**[▶ Live demo](https://demo.tacilytics.com)** · **[Website](https://tacilytics.com)** · **[Hosted cloud](https://cloud.tacilytics.com)** · **[Quick start](#quick-start)**
|
|
18
|
+
|
|
19
|
+
<a href="https://demo.tacilytics.com"><img src="docs/dashboard.png" width="820" alt="Tacilytics dashboard — open the live demo" /></a>
|
|
20
|
+
|
|
21
|
+
<sub>The demo shows a fictional shop with generated data. No real visitors.</sub>
|
|
18
22
|
|
|
19
23
|
</div>
|
|
20
24
|
|
|
@@ -29,7 +33,7 @@
|
|
|
29
33
|
- 🚦 **Campaigns & funnels.** Full UTM (`source` / `medium` / `campaign`), entry & exit pages, **goal conversion rates** and **custom event properties**.
|
|
30
34
|
- ⏱️ **Engagement time** — average time-on-page, measured privately (visible seconds only, no extra identifier).
|
|
31
35
|
- 🤖 **Bots & crawlers dropped** automatically, so your numbers stay honest.
|
|
32
|
-
- ⚡ **~
|
|
36
|
+
- ⚡ **~5 KB tracker** (gzip, 14 KB raw, includes the strict mode, goals and revenue), SPA-aware (tracks `pushState` / `popstate` route changes), optional **DNT / Global Privacy Control** honoring.
|
|
33
37
|
- 🗓️ **Data minimisation built in** — optional auto-purge of raw events after a retention window you choose.
|
|
34
38
|
- 🎯 **Custom events & goals**, **live visitors**, **trends**, **custom date ranges**, **CSV export**.
|
|
35
39
|
- 🛰️ **Agent-ready:** a clean `/api/v1` REST API (with segmentation params) + a `tacilytics` CLI (with `--json`).
|
|
@@ -47,6 +51,16 @@ npx tacilytics serve # http://127.0.0.1:3920 — data/ and .env live in thi
|
|
|
47
51
|
Or install it globally: `npm i -g tacilytics`, then `tacilytics serve`.
|
|
48
52
|
Set your own `WEB_PASSWORD`, `SESSION_SECRET`, `INGEST_SALT` and `API_KEY` in a `.env` file in that folder (see [`.env.example`](.env.example)).
|
|
49
53
|
|
|
54
|
+
**With Docker:**
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
git clone https://github.com/Open-Cosmic-Software/tacilytics.git && cd tacilytics
|
|
58
|
+
cp .env.example .env # set WEB_PASSWORD, SESSION_SECRET, INGEST_SALT, API_KEY
|
|
59
|
+
docker compose up -d # http://127.0.0.1:3920 — data lives in the tacilytics-data volume
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The container listens on `0.0.0.0` inside, compose publishes it on `127.0.0.1` only. Outside Docker, set `HOST` to change the bind address.
|
|
63
|
+
|
|
50
64
|
**From source:**
|
|
51
65
|
|
|
52
66
|
```bash
|
|
@@ -136,7 +150,7 @@ Server mode runs no script and reads or stores nothing on the visitor's device.
|
|
|
136
150
|
|
|
137
151
|
```http
|
|
138
152
|
POST /api/v1/hit
|
|
139
|
-
Authorization: Bearer tk_… # a
|
|
153
|
+
Authorization: Bearer tk_… # a "server mode (count only)" key (Settings → API keys)
|
|
140
154
|
Content-Type: application/json
|
|
141
155
|
|
|
142
156
|
{ "domain": "example.com", "path": "/shop?utm_source=news", "ip": "203.0.113.7",
|
|
@@ -144,21 +158,33 @@ Content-Type: application/json
|
|
|
144
158
|
"purpose": "<Sec-Purpose or Purpose header>" }
|
|
145
159
|
```
|
|
146
160
|
|
|
147
|
-
Node / Express, after your static files and before your pages:
|
|
161
|
+
Node / Express, after your static files and before your pages. It reports only page loads that ended with `200` (no 404s, no redirects) and only for clients that asked for HTML:
|
|
148
162
|
|
|
149
163
|
```js
|
|
150
164
|
app.use((req, res, next) => {
|
|
151
|
-
if (req.method
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
165
|
+
if (req.method !== 'GET' || req.accepts(['html', 'json']) !== 'html') return next();
|
|
166
|
+
res.on('finish', () => {
|
|
167
|
+
if (res.statusCode !== 200) return;
|
|
168
|
+
fetch('https://analytics.example.com/api/v1/hit', {
|
|
169
|
+
method: 'POST', headers: { Authorization: 'Bearer ' + process.env.TACILYTICS_KEY, 'Content-Type': 'application/json' },
|
|
170
|
+
body: JSON.stringify({ domain: 'example.com', path: req.originalUrl, ip: req.ip, ua: req.get('user-agent'),
|
|
171
|
+
referrer: req.get('referer'), lang: req.get('accept-language'), purpose: req.get('sec-purpose') || req.get('purpose') }),
|
|
172
|
+
}).then((r) => { if (r.status === 429) console.warn('tacilytics: rate limited'); }).catch(() => {});
|
|
173
|
+
});
|
|
156
174
|
next();
|
|
157
175
|
});
|
|
158
176
|
```
|
|
159
177
|
|
|
160
|
-
|
|
161
|
-
|
|
178
|
+
**Which key goes on the web server.** Web servers are where sites get broken into, so the key there should be able to do as little as possible:
|
|
179
|
+
|
|
180
|
+
- Create a **server mode (count only)** key under Settings → API keys (admin or owner) and tick the site(s) it reports for. It can call `POST /api/v1/hit` for those sites and nothing else (`403` everywhere else, reads included), and it never reaches more than the account that created it.
|
|
181
|
+
- **Never** put the `API_KEY` from `.env` on a web server. It is the owner key for every site on the instance.
|
|
182
|
+
- Don't use a read + write key either. It still works on `/api/v1/hit` (as in 2.4.0), but a stolen one can also create public share links, point alert webhooks at any URL, upload or delete imports, and change notes, funnels and the time zone of its sites.
|
|
183
|
+
- If several sites run on different servers, give each server its own key, so one break-in can be revoked without touching the others.
|
|
184
|
+
- Server-mode keys arrived in 2.4.1. Before downgrading to 2.4.0, delete them: 2.4.0 does not know the scope and would treat them as ordinary read-only keys.
|
|
185
|
+
|
|
186
|
+
- **The visitor IP** is only accepted here, with a server-mode or write key, never on the public `/api/collect`. It is hashed like a tracker hit (daily salt) and never stored. Behind a proxy or load balancer, send the real visitor IP (e.g. `req.ip` with Express `trust proxy` set right), not your proxy's. Without an IP the hit is refused (`400`), because all visitors would fall onto one hash.
|
|
187
|
+
- **Filtered on our side:** bots and monitors (same list as the tracker), requests without a user agent, prefetch / prerender (`purpose`), and `dnt: true` / `gpc: true` if you choose to forward those headers. A key can only report for its own sites. The rate limit (6,000 page views a minute) applies per key, so sites on one shared-hosting IP don't block each other; above it the API answers `429` and the hit is not counted.
|
|
162
188
|
- **Page caches and CDNs:** if a cache plugin or a CDN (e.g. Cloudflare) answers the request, your server code never runs and the visit is not reported. Report from somewhere every request passes, or exclude HTML from the cache.
|
|
163
189
|
- **What you give up compared with the script:** no time on page, no scroll depth, no click / outbound / download events, no country, and no `?tacilytics=off`. Server-side goals work: add `"name": "Purchase", "props": { "value": 49.9 }`.
|
|
164
190
|
- **Data protection:** the IP and user agent are personal data while they are hashed. That needs a legal basis (typically legitimate interest, GDPR Art. 6(1)(f)) and a sentence in your privacy policy. If someone else runs Tacilytics for you (for example Tacilytics Cloud), your server passes visitor IPs to them, so you need a data processing agreement (DPA). Server mode does not decide whether you need consent for the rest of your website. This is not legal advice.
|
|
@@ -273,7 +299,7 @@ curl -H "Authorization: Bearer $TACILYTICS_API_KEY" \
|
|
|
273
299
|
| `POST /api/v1/sites` | add a site `{ domain }` |
|
|
274
300
|
| `GET /api/v1/stats?site=&period=` | aggregated stats — `today \| 7d \| 30d \| 90d \| 12mo`, or `from=YYYY-MM-DD&to=YYYY-MM-DD` |
|
|
275
301
|
| `GET /api/v1/realtime?site=` | online count, 30-min pulse + recent events |
|
|
276
|
-
| `POST /api/v1/hit` | server mode: report a page view from your web server (
|
|
302
|
+
| `POST /api/v1/hit` | server mode: report a page view from your web server (server-mode key), see [Server mode](#server-mode-no-script-at-all) |
|
|
277
303
|
|
|
278
304
|
Add any of `country, source, page, browser, os, device, lang, medium, campaign, goal` to `/stats` to segment, e.g. `…/stats?site=example.com&period=30d&country=DE&medium=email`.
|
|
279
305
|
|
|
@@ -309,6 +335,28 @@ devices. You get the 95 % that matters (how many, which pages, from where) witho
|
|
|
309
335
|
the same approach as Plausible/Fathom. Whether you still need consent depends on your
|
|
310
336
|
jurisdiction and setup; strict mode (`data-strict`) reduces the device reads further.
|
|
311
337
|
|
|
338
|
+
**Sources per visit.** Like GA, Plausible and Matomo, the source belongs to the visit: an internal
|
|
339
|
+
click or a reload inherits the source of the same visitor's previous page view within 30 minutes,
|
|
340
|
+
instead of showing up as *Direct*. This happens on the server (daily visitor hash), so strict mode
|
|
341
|
+
works the same and nothing extra is read on the device.
|
|
342
|
+
|
|
343
|
+
**Bots and scrapers.** Known crawler, AI-scraper and HTTP-library user agents are dropped. Scrapers
|
|
344
|
+
that pretend to be a normal browser usually run in a cloud: by default, visits from cloud and hosting
|
|
345
|
+
networks are ignored too (list: [X4BNet/lists_vpn](https://github.com/X4BNet/lists_vpn) *datacenter*,
|
|
346
|
+
MIT, downloaded once a day — no visitor data leaves your server). The IP is checked on arrival and
|
|
347
|
+
never stored. iCloud Private Relay runs on the same cloud networks, so Apple's published egress list is
|
|
348
|
+
downloaded too and always let through; without it the filter stays off (with a fixed local list via
|
|
349
|
+
`TACILYTICS_DATACENTER_FILE` it runs anyway — set `TACILYTICS_RELAY_FILE` too, the server warns at start).
|
|
350
|
+
A list that looks wrong (an IPv4 range shorter than /8 or IPv6 shorter than /24, more than 25 % of IPv4
|
|
351
|
+
or 0.1 % of global IPv6, a known residential address inside) is refused, and a failed download of either
|
|
352
|
+
list is retried with backoff. The site settings show how many hits were ignored today and grey the
|
|
353
|
+
switch out while the filter is not running. Honest trade-off: commercial VPNs, Opera's built-in VPN and
|
|
354
|
+
cloud desktops (Windows 365, AVD) are not counted either — for B2B sites consider switching it off.
|
|
355
|
+
If more than half of a site's hits in an hour are dropped (or all of at least 5 on a small site), the
|
|
356
|
+
server log warns (usually a proxy setup: check `TRUST_PROXY`). Switch it off per site in the site
|
|
357
|
+
settings (clock button), or for the whole server
|
|
358
|
+
with `TACILYTICS_DATACENTER=off`.
|
|
359
|
+
|
|
312
360
|
## Light mode & German
|
|
313
361
|
|
|
314
362
|
Settings → Security → **Appearance**: *Theme* (System / Light / Dark) and *Language* (Automatic / English / Deutsch), saved per browser. *System* follows the device's light/dark setting live; *Automatic* picks German when the browser is German. Your data (paths, sources, event names) is never translated.
|
|
@@ -360,6 +408,9 @@ The server-wide `API_KEY` from `.env` keeps owner rights for agents and the CLI
|
|
|
360
408
|
| `SMTP_URL` / `REPORT_FROM` | — | Mail server + sender for the weekly report |
|
|
361
409
|
| `REPORT_HOUR_UTC` | `6` | Monday hour (UTC) the weekly report goes out; it covers the last 7 full days in each site's time zone |
|
|
362
410
|
| `AUDIT_RETENTION_DAYS` | `365` | How long the team activity log is kept (min. 30) |
|
|
411
|
+
| `TACILYTICS_DATACENTER` | on | `off` disables the cloud / hosting network filter server-wide (no list download) |
|
|
412
|
+
| `TACILYTICS_DATACENTER_FILE` | — | Use local range lists instead of the daily download (comma-separated paths, CIDR per line), e.g. for offline installs |
|
|
413
|
+
| `TACILYTICS_RELAY_FILE` | — | Local copy of Apple's iCloud Private Relay egress CSV (with a local datacenter list) |
|
|
363
414
|
| `DATA_RETENTION_DAYS` | `0` | `0` = keep forever; `>0` purges raw events older than N days, daily (data minimisation) |
|
|
364
415
|
|
|
365
416
|
## Privacy & security
|
|
@@ -380,17 +431,28 @@ The server-wide `API_KEY` from `.env` keeps owner rights for agents and the CLI
|
|
|
380
431
|
- All visitor-supplied values are HTML-escaped before they reach the DOM.
|
|
381
432
|
- Run behind TLS; keep `.env` at `chmod 600`.
|
|
382
433
|
|
|
383
|
-
##
|
|
434
|
+
## How it compares
|
|
384
435
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
|
388
|
-
|
|
389
|
-
|
|
|
390
|
-
|
|
|
391
|
-
|
|
|
392
|
-
|
|
|
393
|
-
|
|
|
436
|
+
An honest overview. Plausible and Umami are good projects; pick what fits you.
|
|
437
|
+
|
|
438
|
+
| | Tacilytics | Plausible CE | Umami | Google Analytics 4 |
|
|
439
|
+
|---|---|---|---|---|
|
|
440
|
+
| Cookies | ❌ none | ❌ none | ❌ none | ✅ yes |
|
|
441
|
+
| IPs leave your server | ❌ never | ❌ | ❌ | ✅ sent to Google |
|
|
442
|
+
| No script on the device at all | ✅ server mode | ❌ | ❌ | ❌ |
|
|
443
|
+
| Strict mode (no device reads) | ✅ | ❌ | ❌ | ❌ |
|
|
444
|
+
| Mobile app SDK | ✅ Flutter | ❌ | ❌ | ✅ |
|
|
445
|
+
| Runs on | Node + SQLite file | Elixir + Postgres + ClickHouse | Node + Postgres/MySQL | Google |
|
|
446
|
+
| Tracker size (gzip) | ~5 KB | ~1 KB | ~2 KB | ~45 KB+ |
|
|
447
|
+
| Built for big traffic (millions/day) | ⚠️ not yet, SQLite | ✅ | ✅ | ✅ |
|
|
448
|
+
| Self-hosted, you own the data | ✅ | ✅ | ✅ | ❌ |
|
|
449
|
+
| Licence | MIT | AGPL | MIT | proprietary |
|
|
450
|
+
|
|
451
|
+
*As of October 2026, to the best of our knowledge. Spot a mistake? Please [open an issue](https://github.com/Open-Cosmic-Software/tacilytics/issues).*
|
|
452
|
+
|
|
453
|
+
## Contributing
|
|
454
|
+
|
|
455
|
+
Issues, ideas and pull requests are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). Security reports go [privately](SECURITY.md).
|
|
394
456
|
|
|
395
457
|
## Credits
|
|
396
458
|
|
|
@@ -398,6 +460,8 @@ The server-wide `API_KEY` from `.env` keeps owner rights for agents and the CLI
|
|
|
398
460
|
- Brand marks: [Simple Icons](https://simpleicons.org) (CC0). Brand names and logos are trademarks of their owners; see `public/assets/DISCLAIMER-simple-icons.md`. Windows and Edge have no free official mark, so a neutral icon is shown.
|
|
399
461
|
- Flags: [flag-icons](https://flagicons.lipis.dev) (MIT)
|
|
400
462
|
- Font: [Inter](https://rsms.me/inter/) (SIL OFL 1.1), self-hosted
|
|
463
|
+
- Datacenter ranges: [X4BNet/lists_vpn](https://github.com/X4BNet/lists_vpn) (MIT), downloaded at runtime, not bundled
|
|
464
|
+
- iCloud Private Relay egress ranges: published by Apple (`mask-api.icloud.com`), downloaded at runtime as an allowlist
|
|
401
465
|
|
|
402
466
|
## License
|
|
403
467
|
|
package/auth.js
CHANGED
|
@@ -180,6 +180,11 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
180
180
|
);
|
|
181
181
|
`);
|
|
182
182
|
|
|
183
|
+
// 2.4.1: key scopes. '' = the old behaviour (read-only or the account's own rights); 'hit' = server mode only,
|
|
184
|
+
// usable for POST /api/v1/hit on the listed sites and nothing else. Existing keys keep scope '' (unchanged).
|
|
185
|
+
for (const [col, decl] of [['scope', "TEXT NOT NULL DEFAULT ''"], ['sites', 'TEXT']]) {
|
|
186
|
+
if (!db.prepare('PRAGMA table_info(api_keys)').all().some((c) => c.name === col)) db.exec(`ALTER TABLE api_keys ADD COLUMN ${col} ${decl}`);
|
|
187
|
+
}
|
|
183
188
|
const q = {
|
|
184
189
|
userCount: db.prepare('SELECT COUNT(*) n FROM users'),
|
|
185
190
|
ownerCount: db.prepare("SELECT COUNT(*) n FROM users WHERE role = 'owner'"),
|
|
@@ -204,13 +209,14 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
204
209
|
// after the response is sent. Handlers can name the action/target via res.locals.audit;
|
|
205
210
|
// otherwise method + route are recorded. New endpoints are covered without remembering to.
|
|
206
211
|
// Not logged: tracking beacons, sign-in steps, WebAuthn option requests (no state change).
|
|
207
|
-
const AUDIT_SKIP = new Set(['/api/collect', '/auth/login', '/auth/login/totp', '/auth/passkey/options', '/auth/passkey/verify', '/auth/logout', '/api/me/passkeys/options', '/api/me/totp/setup']);
|
|
212
|
+
const AUDIT_SKIP = new Set(['/api/collect', '/auth/login', '/auth/login/totp', '/auth/passkey/options', '/auth/passkey/verify', '/auth/logout', '/api/me/passkeys/options', '/api/me/totp/setup', '/api/v1/hit']);
|
|
208
213
|
const AUDIT_DAYS = Math.max(30, parseInt(process.env.AUDIT_RETENTION_DAYS, 10) || 365);
|
|
209
214
|
const insAudit = db.prepare('INSERT INTO audit_log (ts, actor, action, target, ip) VALUES (?, ?, ?, ?, ?)');
|
|
210
215
|
function actorOf(p) {
|
|
211
216
|
if (!p) return null;
|
|
212
217
|
if (p.kind === 'user') return p.user.username;
|
|
213
218
|
if (p.kind === 'userkey') return p.user.username + ' (API key)';
|
|
219
|
+
if (p.kind === 'hitkey') return p.user.username + ' (server-mode key)';
|
|
214
220
|
if (p.kind === 'apikey') return 'API key';
|
|
215
221
|
if (p.kind === 'legacy') return 'passphrase';
|
|
216
222
|
return null;
|
|
@@ -220,17 +226,25 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
220
226
|
res.on('finish', () => {
|
|
221
227
|
if (res.statusCode < 200 || res.statusCode >= 300) return;
|
|
222
228
|
try {
|
|
223
|
-
|
|
229
|
+
// one row per named action (a single save can change several things, e.g. time zone + filter)
|
|
230
|
+
const list = res.locals.audits && res.locals.audits.length ? res.locals.audits : [{}];
|
|
224
231
|
const route = req.route ? req.route.path : req.path;
|
|
225
|
-
const
|
|
226
|
-
const
|
|
227
|
-
|
|
232
|
+
const ts = Date.now();
|
|
233
|
+
for (const a of list) {
|
|
234
|
+
const action = a.action || `${req.method} ${route}`;
|
|
235
|
+
const actor = a.actor !== undefined ? a.actor : actorOf(req.principal);
|
|
236
|
+
insAudit.run(ts, actor, action, a.target == null ? null : String(a.target).slice(0, 200), clientIp(req));
|
|
237
|
+
}
|
|
228
238
|
} catch (e) { console.error('audit log:', e.message); }
|
|
229
239
|
});
|
|
230
240
|
next();
|
|
231
241
|
});
|
|
232
242
|
setInterval(() => db.prepare('DELETE FROM audit_log WHERE ts < ?').run(Date.now() - AUDIT_DAYS * 864e5), 6 * 3600e3).unref();
|
|
233
|
-
const audit = (res, action, target, actor) => {
|
|
243
|
+
const audit = (res, action, target, actor) => {
|
|
244
|
+
const a = { action, target, ...(actor !== undefined ? { actor } : {}) };
|
|
245
|
+
res.locals.audit = a;
|
|
246
|
+
(res.locals.audits || (res.locals.audits = [])).push(a);
|
|
247
|
+
};
|
|
234
248
|
if (hasUsers() && !process.env.TACILYTICS_ORIGIN) {
|
|
235
249
|
console.warn('⚠️ TACILYTICS_ORIGIN is not set. Passkeys use the request host, so they break if the dashboard is reached under another host/port or behind a proxy without X-Forwarded-Proto. Set it to the public dashboard URL.');
|
|
236
250
|
}
|
|
@@ -256,8 +270,14 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
256
270
|
const u = k && q.userById.get(k.user_id);
|
|
257
271
|
if (!u) return null;
|
|
258
272
|
if (!k.last_used || Date.now() - k.last_used > 60000) db.prepare('UPDATE api_keys SET last_used = ? WHERE id = ?').run(Date.now(), k.id);
|
|
273
|
+
const userSites = u.sites ? JSON.parse(u.sites) : null;
|
|
274
|
+
if (k.scope === 'hit') {
|
|
275
|
+
// never more than the account itself may see today (an account limited later narrows its old keys too)
|
|
276
|
+
const ks = JSON.parse(k.sites || '[]').filter((id) => userSites == null || userSites.includes(id));
|
|
277
|
+
return { kind: 'hitkey', role: 'viewer', sites: ks, user: u, keyId: k.id };
|
|
278
|
+
}
|
|
259
279
|
const role = k.read_only ? 'viewer' : u.role;
|
|
260
|
-
return { kind: 'userkey', role, sites:
|
|
280
|
+
return { kind: 'userkey', role, sites: userSites, user: u, keyId: k.id };
|
|
261
281
|
}
|
|
262
282
|
return null; // a wrong key is a failed sign-in, not an anonymous request
|
|
263
283
|
}
|
|
@@ -278,6 +298,8 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
278
298
|
function auth(req, res, next) {
|
|
279
299
|
const p = principal(req);
|
|
280
300
|
if (!p) return res.status(401).json({ error: 'Not authenticated' });
|
|
301
|
+
// a server-mode key reports page views and does nothing else — not even read stats
|
|
302
|
+
if (p.kind === 'hitkey' && !(req.method === 'POST' && req.path === '/api/v1/hit')) return res.status(403).json({ error: 'This key can only report page views (POST /api/v1/hit)' });
|
|
281
303
|
req.principal = p;
|
|
282
304
|
next();
|
|
283
305
|
}
|
|
@@ -498,7 +520,7 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
498
520
|
res.json(out);
|
|
499
521
|
});
|
|
500
522
|
const userOnly = (req, res, next) => {
|
|
501
|
-
if (req.principal.kind === 'userkey') return res.status(403).json({ error: 'API keys cannot change account settings — sign in' });
|
|
523
|
+
if (req.principal.kind === 'userkey' || req.principal.kind === 'hitkey') return res.status(403).json({ error: 'API keys cannot change account settings — sign in' });
|
|
502
524
|
return req.principal.user ? next() : res.status(400).json({ error: 'Create an account first' });
|
|
503
525
|
};
|
|
504
526
|
async function needPassword(req, res) {
|
|
@@ -599,7 +621,7 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
599
621
|
});
|
|
600
622
|
|
|
601
623
|
app.get('/api/audit', requireRole('owner'), (req, res) => {
|
|
602
|
-
if (req.principal.kind === 'userkey') return res.status(403).json({ error: 'API keys cannot read the audit log — sign in' });
|
|
624
|
+
if (req.principal.kind === 'userkey' || req.principal.kind === 'hitkey') return res.status(403).json({ error: 'API keys cannot read the audit log — sign in' });
|
|
603
625
|
const before = Number(req.query.before) || Date.now() + 1;
|
|
604
626
|
const limit = Math.min(500, Math.max(1, Number(req.query.limit) || 100));
|
|
605
627
|
res.json(db.prepare('SELECT id, ts, actor, action, target, ip FROM audit_log WHERE ts < ? ORDER BY ts DESC, id DESC LIMIT ?').all(before, limit));
|
|
@@ -607,21 +629,33 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
607
629
|
|
|
608
630
|
// ── personal API keys ──
|
|
609
631
|
app.get('/api/me/keys', auth, userOnly, (req, res) => {
|
|
610
|
-
res.json(db.prepare('SELECT id, prefix, name, read_only, created, last_used FROM api_keys WHERE user_id = ? ORDER BY created').all(req.principal.user.id)
|
|
611
|
-
.map((k) => ({ ...k, read_only: !!k.read_only })));
|
|
632
|
+
res.json(db.prepare('SELECT id, prefix, name, read_only, scope, sites, created, last_used FROM api_keys WHERE user_id = ? ORDER BY created').all(req.principal.user.id)
|
|
633
|
+
.map((k) => ({ ...k, read_only: !!k.read_only, scope: k.scope || '', sites: k.sites ? JSON.parse(k.sites) : null })));
|
|
612
634
|
});
|
|
613
635
|
app.post('/api/me/keys', auth, userOnly, (req, res) => {
|
|
614
636
|
const b = req.body || {};
|
|
615
637
|
const n = db.prepare('SELECT COUNT(*) n FROM api_keys WHERE user_id = ?').get(req.principal.user.id).n;
|
|
616
638
|
if (n >= 20) return res.status(400).json({ error: 'At most 20 keys per account' });
|
|
617
|
-
// a viewer can only ever get a read-only key
|
|
618
|
-
const readOnly = req.principal.role === 'viewer' ? true : b.readOnly !== false;
|
|
619
639
|
const key = 'tk_' + crypto.randomBytes(24).toString('base64url');
|
|
620
640
|
const name = String(b.name || '').trim().slice(0, 60) || 'API key';
|
|
641
|
+
if (b.scope === 'hit') {
|
|
642
|
+
// server mode: bound to explicit sites, never more than the account may see. Admin or owner only: reporting
|
|
643
|
+
// page views is writing data (it can add visitors), and a viewer may not write — same rule as /api/v1/hit in 2.4.0.
|
|
644
|
+
if (req.principal.role === 'viewer') return res.status(403).json({ error: 'Viewers cannot create server-mode keys' });
|
|
645
|
+
if (!Array.isArray(b.sites) || !b.sites.length) return res.status(400).json({ error: 'Pick the site(s) this server reports for' });
|
|
646
|
+
const ids = [...new Set(b.sites.map(Number))];
|
|
647
|
+
if (ids.some((id) => !allSiteIds().includes(id) || !siteAllowed(req, id))) return res.status(403).json({ error: 'You have no access to one of these sites' });
|
|
648
|
+
const info = db.prepare("INSERT INTO api_keys (key_hash, prefix, user_id, name, read_only, scope, sites, created) VALUES (?, ?, ?, ?, 1, 'hit', ?, ?)")
|
|
649
|
+
.run(sha256(key), key.slice(0, 10), req.principal.user.id, name, JSON.stringify(ids), Date.now());
|
|
650
|
+
audit(res, 'me.apikey.add', `${name} (${key.slice(0, 10)}…, server mode: sites ${ids.join(',')})`);
|
|
651
|
+
return res.json({ id: info.lastInsertRowid, key, name, read_only: true, scope: 'hit', sites: ids });
|
|
652
|
+
}
|
|
653
|
+
// a viewer can only ever get a read-only key
|
|
654
|
+
const readOnly = req.principal.role === 'viewer' ? true : b.readOnly !== false;
|
|
621
655
|
const info = db.prepare('INSERT INTO api_keys (key_hash, prefix, user_id, name, read_only, created) VALUES (?, ?, ?, ?, ?, ?)')
|
|
622
656
|
.run(sha256(key), key.slice(0, 10), req.principal.user.id, name, readOnly ? 1 : 0, Date.now());
|
|
623
657
|
audit(res, 'me.apikey.add', `${name} (${key.slice(0, 10)}…, ${readOnly ? 'read-only' : 'read + write'})`);
|
|
624
|
-
res.json({ id: info.lastInsertRowid, key, name, read_only: readOnly }); // the key is shown exactly once
|
|
658
|
+
res.json({ id: info.lastInsertRowid, key, name, read_only: readOnly, scope: '' }); // the key is shown exactly once
|
|
625
659
|
});
|
|
626
660
|
app.delete('/api/me/keys/:id', auth, userOnly, (req, res) => {
|
|
627
661
|
const k = db.prepare('SELECT name, prefix FROM api_keys WHERE id = ? AND user_id = ?').get(Number(req.params.id), req.principal.user.id);
|
|
@@ -633,7 +667,7 @@ function setupAuth({ app, db, WEB_PASSWORD, API_KEY, rateLimited, clientIp, site
|
|
|
633
667
|
// ── team (owner) ───────────────────────────────────────────────────────────
|
|
634
668
|
// team management needs a real sign-in (or the server API_KEY): a personal key that leaks must not
|
|
635
669
|
// be able to invite, promote or remove people
|
|
636
|
-
const ownerOnly = (req, res, next) => requireRole('owner')(req, res, () => (req.principal.kind === 'userkey'
|
|
670
|
+
const ownerOnly = (req, res, next) => requireRole('owner')(req, res, () => (req.principal.kind === 'userkey' || req.principal.kind === 'hitkey'
|
|
637
671
|
? res.status(403).json({ error: 'API keys cannot manage the team — sign in' }) : next()));
|
|
638
672
|
|
|
639
673
|
// first account: only from a legacy (WEB_PASSWORD) or API-key session, only while there are none
|
package/datacenter.js
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Datacenter IP check (2.5). Scrapers that pretend to be a normal Chrome on a Mac pass every
|
|
3
|
+
// user-agent filter, but people do not browse from cloud servers. Like Plausible, the visitor IP is
|
|
4
|
+
// checked once on arrival against public lists of hosting / cloud networks and never stored.
|
|
5
|
+
//
|
|
6
|
+
// List: X4BNet/lists_vpn "datacenter" output (MIT, https://github.com/X4BNet/lists_vpn), refreshed once
|
|
7
|
+
// a day into the data directory. Nothing about visitors leaves the server: we only download the list.
|
|
8
|
+
// If no list is available (offline install, first start without network) the check is off and every
|
|
9
|
+
// visit counts — losing real visitors is worse than counting a few bots.
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const net = require('net');
|
|
12
|
+
|
|
13
|
+
const DEFAULT_URLS = [
|
|
14
|
+
'https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/datacenter/ipv4.txt',
|
|
15
|
+
'https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/datacenter/ipv6.txt',
|
|
16
|
+
];
|
|
17
|
+
// iCloud Private Relay runs on Akamai / Fastly / Cloudflare networks that the datacenter list contains.
|
|
18
|
+
// Those are real Safari visitors, so Apple's published egress ranges are an allowlist checked first.
|
|
19
|
+
const RELAY_URL = 'https://mask-api.icloud.com/egress-ip-ranges.csv';
|
|
20
|
+
|
|
21
|
+
// sanity limits: a broken or hostile list must never be able to drop everyone
|
|
22
|
+
const MIN_BITS_V4 = 8;
|
|
23
|
+
const MIN_BITS_V6 = 24; // the real list's largest IPv6 prefix is /24
|
|
24
|
+
const MAX_V4_SHARE = 0.25; // today ~9 % of the IPv4 space
|
|
25
|
+
const MAX_V6_SHARE = 0.001; // of global unicast 2000::/3, today ~12 ppm
|
|
26
|
+
// residential / ISP addresses that must never be inside a datacenter list (Telekom, Vodafone, Comcast, Telekom v6)
|
|
27
|
+
// IPv6: Telekom, Vodafone, T-Mobile, 1&1, O2
|
|
28
|
+
const CANARIES = ['79.192.0.1', '84.150.3.3', '91.64.1.1', '73.0.0.1',
|
|
29
|
+
'2003::1', '2a02:8108::1', '2a01:598::1', '2001:16b8::1', '2a02:3030::1'];
|
|
30
|
+
|
|
31
|
+
function v4ToInt(ip) {
|
|
32
|
+
const p = ip.split('.');
|
|
33
|
+
return ((+p[0] << 24) >>> 0) + (+p[1] << 16) + (+p[2] << 8) + +p[3];
|
|
34
|
+
}
|
|
35
|
+
function v6ToBig(ip) {
|
|
36
|
+
// expand "::" and an embedded IPv4 tail, then read 8 groups of 16 bits
|
|
37
|
+
let s = ip.toLowerCase();
|
|
38
|
+
const v4 = s.match(/(\d+\.\d+\.\d+\.\d+)$/);
|
|
39
|
+
if (v4) { const n = v4ToInt(v4[1]); s = s.slice(0, -v4[1].length) + ((n >>> 16).toString(16)) + ':' + ((n & 0xffff).toString(16)); }
|
|
40
|
+
const [head, tail] = s.split('::');
|
|
41
|
+
const h = head ? head.split(':') : [];
|
|
42
|
+
const t = tail !== undefined ? (tail ? tail.split(':') : []) : [];
|
|
43
|
+
const groups = tail !== undefined ? [...h, ...Array(8 - h.length - t.length).fill('0'), ...t] : h;
|
|
44
|
+
let x = 0n;
|
|
45
|
+
for (const g of groups) x = (x << 16n) + BigInt(parseInt(g || '0', 16));
|
|
46
|
+
return x;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// parse "a.b.c.d/nn" and "x::/nn" lines into sorted, merged [start, end] ranges
|
|
50
|
+
function parseList(text, { minV4 = MIN_BITS_V4, minV6 = MIN_BITS_V6 } = {}) {
|
|
51
|
+
const v4 = []; const v6 = [];
|
|
52
|
+
for (const raw of String(text).split('\n')) {
|
|
53
|
+
const line = raw.split('#')[0].trim();
|
|
54
|
+
if (!line) continue;
|
|
55
|
+
const [addrRaw, bitsS, extra] = line.split(',')[0].trim().split('/');
|
|
56
|
+
const addr = addrRaw.trim();
|
|
57
|
+
if (extra !== undefined) continue;
|
|
58
|
+
if (bitsS !== undefined && !/^\d{1,3}$/.test(bitsS)) continue; // "1.2.3.4/" must not become /0
|
|
59
|
+
const kind = net.isIP(addr);
|
|
60
|
+
if (kind === 4) {
|
|
61
|
+
const bits = bitsS === undefined ? 32 : +bitsS;
|
|
62
|
+
if (!(bits >= minV4 && bits <= 32)) continue;
|
|
63
|
+
const size = 2 ** (32 - bits);
|
|
64
|
+
const start = Math.floor(v4ToInt(addr) / size) * size;
|
|
65
|
+
v4.push([start, start + size - 1]);
|
|
66
|
+
} else if (kind === 6) {
|
|
67
|
+
const bits = bitsS === undefined ? 128 : +bitsS;
|
|
68
|
+
if (!(bits >= minV6 && bits <= 128)) continue;
|
|
69
|
+
const host = 128n - BigInt(bits);
|
|
70
|
+
const start = (v6ToBig(addr) >> host) << host;
|
|
71
|
+
v6.push([start, start + (1n << host) - 1n]);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const merge = (arr) => {
|
|
75
|
+
arr.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
76
|
+
const out = [];
|
|
77
|
+
for (const r of arr) {
|
|
78
|
+
const last = out[out.length - 1];
|
|
79
|
+
if (last && r[0] <= last[1] + (typeof r[0] === 'bigint' ? 1n : 1)) { if (r[1] > last[1]) last[1] = r[1]; } else out.push([r[0], r[1]]);
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
};
|
|
83
|
+
return { v4: merge(v4), v6: merge(v6) };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function inRanges(ranges, x) {
|
|
87
|
+
let lo = 0; let hi = ranges.length - 1;
|
|
88
|
+
while (lo <= hi) {
|
|
89
|
+
const mid = (lo + hi) >> 1;
|
|
90
|
+
if (x < ranges[mid][0]) hi = mid - 1;
|
|
91
|
+
else if (x > ranges[mid][1]) lo = mid + 1;
|
|
92
|
+
else return true;
|
|
93
|
+
}
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function v4Share(l) { let n = 0; for (const [a, b] of l.v4) n += b - a + 1; return n / 2 ** 32; }
|
|
98
|
+
// share of global unicast 2000::/3 (2^125 addresses), counted only for the part inside it
|
|
99
|
+
function v6Share(l) {
|
|
100
|
+
const lo = 0x2n << 124n; const hi = lo + (1n << 125n) - 1n;
|
|
101
|
+
let n = 0n;
|
|
102
|
+
for (const [a, b] of l.v6) { const s = a > lo ? a : lo; const e = b < hi ? b : hi; if (e >= s) n += e - s + 1n; }
|
|
103
|
+
return Number((n * 1000000n) >> 125n) / 1e6;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Reject a whole list that looks wrong instead of trusting it. Returns an error text or null.
|
|
107
|
+
function validate(l) {
|
|
108
|
+
if (!l.v4.length && !l.v6.length) return 'no ranges';
|
|
109
|
+
const share = v4Share(l);
|
|
110
|
+
if (share > MAX_V4_SHARE) return `covers ${(share * 100).toFixed(1)} % of IPv4 (limit ${MAX_V4_SHARE * 100} %)`;
|
|
111
|
+
const share6 = v6Share(l);
|
|
112
|
+
if (share6 > MAX_V6_SHARE) return `covers ${(share6 * 100).toFixed(3)} % of global IPv6 (limit ${MAX_V6_SHARE * 100} %)`;
|
|
113
|
+
for (const c of CANARIES) if (lookup(l, c)) return 'contains residential address ' + c;
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
function lookup(l, ip) {
|
|
117
|
+
const kind = net.isIP(ip);
|
|
118
|
+
if (kind === 4) return inRanges(l.v4, v4ToInt(ip));
|
|
119
|
+
if (kind === 6) return inRanges(l.v6, v6ToBig(ip));
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
// Apple relay CSV: "prefix,country,region,city,". Relay prefixes are small, no minimum length.
|
|
123
|
+
// One pass over the ~12 MB file: keep only the prefix column and count the well-formed lines.
|
|
124
|
+
function parseRelay(text) {
|
|
125
|
+
const t = String(text);
|
|
126
|
+
const out = [];
|
|
127
|
+
let lines = 0;
|
|
128
|
+
let i = 0;
|
|
129
|
+
let comma = -1; // reused across lines: a file without commas must not rescan to the end for every line
|
|
130
|
+
if (t.charCodeAt(0) === 0xfeff) i = 1; // byte-order mark
|
|
131
|
+
while (i < t.length) {
|
|
132
|
+
let nl = t.indexOf('\n', i); if (nl < 0) nl = t.length;
|
|
133
|
+
if (comma < i && comma !== -2) { comma = t.indexOf(',', i); if (comma < 0) comma = -2; }
|
|
134
|
+
if (comma > i && comma < nl) {
|
|
135
|
+
const prefix = t.slice(i, comma);
|
|
136
|
+
if (/^[0-9a-fA-F:.]+\/\d{1,3}$/.test(prefix)) { out.push(prefix); lines++; }
|
|
137
|
+
}
|
|
138
|
+
i = nl + 1;
|
|
139
|
+
}
|
|
140
|
+
const r = parseList(out.join('\n'), { minV4: 0, minV6: 0 });
|
|
141
|
+
r.lines = lines;
|
|
142
|
+
return r;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function createDatacenterCheck({ dir, mode = process.env.TACILYTICS_DATACENTER, files = process.env.TACILYTICS_DATACENTER_FILE,
|
|
146
|
+
relayFile = process.env.TACILYTICS_RELAY_FILE, log = console } = {}) {
|
|
147
|
+
let list = { v4: [], v6: [] };
|
|
148
|
+
let relay = { v4: [], v6: [] };
|
|
149
|
+
let loaded = false;
|
|
150
|
+
let loadedAt = 0;
|
|
151
|
+
let retryMs = 5 * 60000;
|
|
152
|
+
const off = String(mode || '').toLowerCase() === 'off';
|
|
153
|
+
const P = require('path');
|
|
154
|
+
const cache = dir ? P.join(dir, '.datacenter-ranges.txt') : null;
|
|
155
|
+
const relayCache = dir ? P.join(dir, '.relay-ranges.csv') : null;
|
|
156
|
+
const say = (k, m) => { if (log[k]) log[k]('datacenter check: ' + m); };
|
|
157
|
+
|
|
158
|
+
function use(text, from) {
|
|
159
|
+
const l = parseList(text);
|
|
160
|
+
const bad = validate(l);
|
|
161
|
+
if (bad) { say('warn', `list from ${from} rejected (${bad})` + (loaded ? ', keeping the last list' : ', filter stays off')); return false; }
|
|
162
|
+
list = l; loaded = true; loadedAt = Date.now();
|
|
163
|
+
say('info', `${l.v4.length} IPv4 + ${l.v6.length} IPv6 ranges (${from}, ${(v4Share(l) * 100).toFixed(1)} % of IPv4)`);
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
function useRelay(text, from) {
|
|
167
|
+
const r = parseRelay(text);
|
|
168
|
+
if (r.lines < 100) { say('warn', `relay list from ${from} too short, ignored`); return false; }
|
|
169
|
+
relay = { v4: r.v4, v6: r.v6 };
|
|
170
|
+
say('info', `${r.v4.length} IPv4 + ${r.v6.length} IPv6 iCloud Private Relay ranges allowed (${from})`);
|
|
171
|
+
return true;
|
|
172
|
+
}
|
|
173
|
+
const writeCache = (f, text) => { if (!f) return; const tmp = f + '.tmp'; fs.writeFileSync(tmp, text, { mode: 0o600 }); fs.renameSync(tmp, f); };
|
|
174
|
+
async function get(u) {
|
|
175
|
+
const r = await fetch(u, { signal: AbortSignal.timeout(30000) });
|
|
176
|
+
if (!r.ok) throw new Error(u + ' → HTTP ' + r.status);
|
|
177
|
+
return r.text();
|
|
178
|
+
}
|
|
179
|
+
let timer = null;
|
|
180
|
+
function schedule(ms) { if (timer) clearTimeout(timer); timer = setTimeout(refresh, ms); if (timer.unref) timer.unref(); }
|
|
181
|
+
async function refresh() {
|
|
182
|
+
if (off) return;
|
|
183
|
+
let ok = true;
|
|
184
|
+
try {
|
|
185
|
+
const text = (await Promise.all(DEFAULT_URLS.map(get))).join('\n');
|
|
186
|
+
if (use(text, 'download')) writeCache(cache, text); else ok = false;
|
|
187
|
+
} catch (e) { ok = false; say('warn', 'refresh failed (' + e.message + ')' + (loaded ? ', keeping the last list' : ', filter stays off until a list loads')); }
|
|
188
|
+
try { const t = await get(RELAY_URL); if (useRelay(t, 'download')) writeCache(relayCache, t); else ok = false; } catch (e) { ok = false; say('warn', 'relay list refresh failed (' + e.message + ')'); }
|
|
189
|
+
// without the relay allowlist the filter is off, so a missing relay list is a failure too
|
|
190
|
+
if (!relay.v4.length && !relay.v6.length) ok = false;
|
|
191
|
+
// retry with backoff while something failed, otherwise once a day
|
|
192
|
+
if (ok) { retryMs = 5 * 60000; schedule(24 * 3600 * 1000); } else { schedule(retryMs); retryMs = Math.min(retryMs * 4, 6 * 3600 * 1000); }
|
|
193
|
+
}
|
|
194
|
+
function start() {
|
|
195
|
+
if (off) { say('info', 'off (TACILYTICS_DATACENTER=off)'); return; }
|
|
196
|
+
if (relayFile) { try { useRelay(fs.readFileSync(relayFile, 'utf8'), 'file'); } catch (e) { say('warn', 'relay file: ' + e.message); } }
|
|
197
|
+
if (files) {
|
|
198
|
+
// fixed local list (offline installs, tests): no download
|
|
199
|
+
if (!relayFile) say('warn', 'TACILYTICS_DATACENTER_FILE without TACILYTICS_RELAY_FILE: iCloud Private Relay visitors inside the list will be dropped');
|
|
200
|
+
try { use(String(files).split(',').map((f) => fs.readFileSync(f.trim(), 'utf8')).join('\n'), 'file'); } catch (e) { say('warn', 'list file: ' + e.message); }
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
let fresh = false;
|
|
204
|
+
let relayOk = false;
|
|
205
|
+
if (relayCache && fs.existsSync(relayCache)) { try { relayOk = useRelay(fs.readFileSync(relayCache, 'utf8'), 'cache'); } catch {} }
|
|
206
|
+
if (cache && fs.existsSync(cache)) {
|
|
207
|
+
try {
|
|
208
|
+
// clamp: a cache file with a future mtime must not give a negative age (timer overflow)
|
|
209
|
+
const age = Math.max(0, Date.now() - fs.statSync(cache).mtimeMs);
|
|
210
|
+
if (use(fs.readFileSync(cache, 'utf8'), 'cache')) { loadedAt = Date.now() - age; fresh = relayOk && age < 24 * 3600 * 1000; }
|
|
211
|
+
if (age > 7 * 24 * 3600 * 1000) say('warn', `cached list is ${Math.round(age / 86400000)} days old`);
|
|
212
|
+
} catch {}
|
|
213
|
+
}
|
|
214
|
+
if (fresh) schedule(Math.max(1000, 24 * 3600 * 1000 - (Date.now() - loadedAt))); else refresh();
|
|
215
|
+
}
|
|
216
|
+
function isDatacenter(ip) {
|
|
217
|
+
if (!loaded) return false;
|
|
218
|
+
let s = String(ip || '').split('%')[0].toLowerCase();
|
|
219
|
+
if (s.startsWith('::ffff:') && net.isIPv4(s.slice(7))) s = s.slice(7);
|
|
220
|
+
if (!net.isIP(s)) return false;
|
|
221
|
+
// without Apple's relay allowlist the datacenter list would drop iCloud Private Relay users
|
|
222
|
+
// (Akamai / Fastly egress): then the filter stays off. Only a fixed local list may run without it.
|
|
223
|
+
if (!relay.v4.length && !relay.v6.length && !files) return false;
|
|
224
|
+
if (lookup(relay, s)) return false; // iCloud Private Relay = a real person
|
|
225
|
+
return lookup(list, s);
|
|
226
|
+
}
|
|
227
|
+
// active = the filter really drops something right now (list loaded and, unless a fixed file is used, the relay allowlist too)
|
|
228
|
+
const active = () => !off && loaded && (!!files || relay.v4.length > 0 || relay.v6.length > 0);
|
|
229
|
+
return { start, refresh, isDatacenter, get loaded() { return loaded; }, get loadedAt() { return loadedAt; }, get active() { return active(); }, get off() { return off; } };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
module.exports = { createDatacenterCheck, _test: { parseList, parseRelay, validate, inRanges, v4ToInt, v6ToBig, v6Share, MIN_BITS_V6 } };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tacilytics",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "Privacy-first, cookieless analytics for websites and apps. Self-hosted: one Node process, one SQLite file.",
|
|
5
5
|
"main": "server.js",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -49,6 +49,7 @@
|
|
|
49
49
|
"server.js",
|
|
50
50
|
"cli.js",
|
|
51
51
|
"day-salt.js",
|
|
52
|
+
"datacenter.js",
|
|
52
53
|
"auth.js",
|
|
53
54
|
"importers.js",
|
|
54
55
|
"report.js",
|
package/public/account.js
CHANGED
|
@@ -277,11 +277,13 @@
|
|
|
277
277
|
'<p class="muted">Every Monday morning, for each of your sites: key figures, a day-by-day chart, highlights, top pages, sources, countries, devices, goals, shop numbers and your notes — compared with the week before.</p>' +
|
|
278
278
|
'<div id="repBox"><p class="muted small">Loading…</p></div></div>' +
|
|
279
279
|
'<div class="set-card"><h3>' + ico('i-code') + 'API keys</h3>' +
|
|
280
|
-
'<p class="muted">For scripts, agents and the CLI (<code>TACILYTICS_API_KEY</code>). A key acts as you, sees only your sites, and cannot touch the team or your security settings
|
|
280
|
+
'<p class="muted">For scripts, agents and the CLI (<code>TACILYTICS_API_KEY</code>). A key acts as you, sees only your sites, and cannot touch the team or your security settings.</p>' +
|
|
281
|
+
'<p class="muted small">For server mode, create a “server mode (count only)” key: it can only report page views for the sites you tick, nothing else. Never put a read + write key on a web server.</p>' +
|
|
281
282
|
'<div id="keyList"><p class="muted small">Loading…</p></div>' +
|
|
282
283
|
'<div class="set-inline"><input id="keyName" type="text" placeholder="Name, e.g. reporting script" maxlength="60">' +
|
|
283
|
-
|
|
284
|
-
'<button class="btn" id="keyAdd">Create key</button></div
|
|
284
|
+
'<select id="keyKind" style="flex:0 0 auto;width:auto"><option value="ro">read-only</option>' + (me.role !== 'viewer' ? '<option value="rw">read + write</option><option value="hit">server mode (count only)</option>' : '') + '</select>' +
|
|
285
|
+
'<button class="btn" id="keyAdd">Create key</button></div>' +
|
|
286
|
+
'<div id="keySites" class="hide small muted" style="margin-top:8px"></div><div id="keyOut"></div></div>' +
|
|
285
287
|
'<div class="set-card"><h3>' + ico('i-shield') + 'Password</h3>' +
|
|
286
288
|
'<div class="set-inline"><input id="pwOld" type="password" placeholder="Current password" autocomplete="current-password"><input id="pwNew" type="password" placeholder="New password (10+)" autocomplete="new-password"><button class="btn ghost" id="pwSave">Change</button></div>' +
|
|
287
289
|
'<p class="muted small">Changing it signs you out on all other devices.</p></div>' +
|
|
@@ -343,7 +345,7 @@
|
|
|
343
345
|
function loadKeys() {
|
|
344
346
|
call('GET', '/api/me/keys').then(function (keys) {
|
|
345
347
|
$('keyList').innerHTML = keys.length ? keys.map(function (k) {
|
|
346
|
-
return '<div class="set-row"><span>' + ico('i-code') + esc(k.name) + ' <code class="muted small">' + esc(k.prefix) + '…</code>' + (k.read_only ? ' <span class="pill">read-only</span>' : ' <span class="pill">read + write</span>') + '</span>' +
|
|
348
|
+
return '<div class="set-row"><span>' + ico('i-code') + esc(k.name) + ' <code class="muted small">' + esc(k.prefix) + '…</code>' + (k.scope === 'hit' ? ' <span class="pill"><span>server mode</span> · <span data-no-i18n>' + esc(siteNames(k.sites)) + '</span></span>' : k.read_only ? ' <span class="pill">read-only</span>' : ' <span class="pill">read + write</span>') + '</span>' +
|
|
347
349
|
'<span class="muted small">' + (k.last_used ? 'used ' + date(k.last_used) : 'never used') + '</span><button class="btn ghost danger mini" data-delkey="' + k.id + '">Revoke</button></div>';
|
|
348
350
|
}).join('') : '<p class="muted small">No keys yet.</p>';
|
|
349
351
|
$('keyList').querySelectorAll('[data-delkey]').forEach(function (b) {
|
|
@@ -352,9 +354,21 @@
|
|
|
352
354
|
}).catch(function (e) { setErr(e.message); });
|
|
353
355
|
}
|
|
354
356
|
loadKeys();
|
|
357
|
+
var siteList = (window.Taci && window.Taci.sites && window.Taci.sites()) || [];
|
|
358
|
+
$('keyKind').onchange = function () {
|
|
359
|
+
var hit = this.value === 'hit';
|
|
360
|
+
$('keySites').classList.toggle('hide', !hit);
|
|
361
|
+
if (hit && !$('keySites').innerHTML) $('keySites').innerHTML = 'Reports for: ' + siteList.map(function (st) {
|
|
362
|
+
return '<label style="display:inline-flex;gap:4px;align-items:center;margin-right:10px"><input type="checkbox" class="keySite" value="' + st.id + '" style="width:auto;margin:0"' + (siteList.length === 1 ? ' checked' : '') + '>' + esc(st.domain) + '</label>';
|
|
363
|
+
}).join('');
|
|
364
|
+
};
|
|
355
365
|
$('keyAdd').onclick = function () {
|
|
356
366
|
setErr('');
|
|
357
|
-
|
|
367
|
+
var kind = $('keyKind').value;
|
|
368
|
+
var body = { name: $('keyName').value.trim() };
|
|
369
|
+
if (kind === 'hit') { body.scope = 'hit'; body.sites = Array.prototype.map.call(document.querySelectorAll('.keySite:checked'), function (c) { return Number(c.value); }); }
|
|
370
|
+
else body.readOnly = kind !== 'rw';
|
|
371
|
+
call('POST', '/api/me/keys', body).then(function (k) {
|
|
358
372
|
$('keyName').value = '';
|
|
359
373
|
$('keyOut').innerHTML = '<p class="muted small" style="margin:10px 0 4px">Copy it now — it is shown only once.</p><div class="snippet">' + esc(k.key) + '</div><button class="btn ghost mini" id="keyCopy">Copy key</button>';
|
|
360
374
|
$('keyCopy').onclick = function () { navigator.clipboard.writeText(k.key); this.textContent = 'Copied ✓'; };
|
package/public/app.js
CHANGED
|
@@ -145,13 +145,16 @@
|
|
|
145
145
|
function showTz() { var s = currentSite(); $('tzName').textContent = s ? String(s.timezone || 'UTC').replace(/_/g, ' ') : 'UTC'; $('tzBtn').classList.toggle('hide', !s); }
|
|
146
146
|
$('tzBtn').onclick = function () {
|
|
147
147
|
var s = currentSite(); if (!s) return;
|
|
148
|
-
$('tzSite').textContent = s.domain; $('tzErr').textContent = ''; fillTz($('tzSelect'), s.timezone || 'UTC');
|
|
148
|
+
$('tzSite').textContent = s.domain; $('tzErr').textContent = ''; fillTz($('tzSelect'), s.timezone || 'UTC'); $('dcFilter').checked = s.dc_filter !== 0; $('dcFilter').disabled = s.dc_active === false;
|
|
149
|
+
$('dcFilter').parentNode.style.opacity = s.dc_active === false ? '0.55' : '';
|
|
150
|
+
$('dcInfo').textContent = s.dc_active === false ? 'The filter is not running on this server right now (switched off, or the network lists have not loaded yet). Every visit is counted.'
|
|
151
|
+
: s.dc_dropped_today ? s.dc_dropped_today + ' of ' + s.dc_seen_today + ' hits from cloud networks ignored today (UTC).' : '';
|
|
149
152
|
$('tzModal').classList.remove('hide'); $('tzSelect').focus();
|
|
150
153
|
};
|
|
151
154
|
$('cancelTz').onclick = function () { closeModal($('tzModal'), { force: true }); };
|
|
152
155
|
$('saveTz').onclick = function () {
|
|
153
156
|
var s = currentSite(); if (!s) return;
|
|
154
|
-
api('/api/sites/' + s.id, { method: 'PATCH', body: JSON.stringify({ timezone: $('tzSelect').value }) }).then(function (r) {
|
|
157
|
+
api('/api/sites/' + s.id, { method: 'PATCH', body: JSON.stringify($('dcFilter').disabled ? { timezone: $('tzSelect').value } : { timezone: $('tzSelect').value, dcFilter: $('dcFilter').checked }) }).then(function (r) {
|
|
155
158
|
if (r.error) { $('tzErr').textContent = r.error; return; }
|
|
156
159
|
closeModal($('tzModal'), { force: true });
|
|
157
160
|
loadSites().then(load);
|
package/public/i18n.js
CHANGED
|
@@ -131,16 +131,21 @@
|
|
|
131
131
|
'Tacilytics · open source ·': 'Tacilytics · Open Source ·',
|
|
132
132
|
'· open source ·': '· Open Source ·',
|
|
133
133
|
'Discard your changes?': 'Änderungen verwerfen?',
|
|
134
|
-
'Time zone': 'Zeitzone',
|
|
134
|
+
'Time zone': 'Zeitzone', 'Site settings': 'Website-Einstellungen',
|
|
135
|
+
'The filter is not running on this server right now (switched off, or the network lists have not loaded yet). Every visit is counted.': 'Der Filter läuft auf diesem Server gerade nicht (ausgeschaltet oder die Netzlisten sind noch nicht geladen). Jeder Besuch wird gezählt.',
|
|
135
136
|
'Time zone (days start at midnight here)': 'Zeitzone (hier beginnt der Tag um Mitternacht)',
|
|
136
137
|
'Time zone of this site — days start at midnight here': 'Zeitzone dieser Website, hier beginnt der Tag um Mitternacht',
|
|
137
138
|
'Days, hours, the weekly report and alerts follow this time zone for': 'Tage, Stunden, Wochenbericht und Alarme richten sich nach dieser Zeitzone für',
|
|
139
|
+
'Ignore visits from cloud and hosting networks (scrapers). The IP is checked on arrival and never stored. iCloud Private Relay is let through; commercial VPNs and cloud desktops are not counted.': 'Besuche aus Cloud- und Hosting-Netzen ignorieren (Scraper). Die IP wird beim Eintreffen geprüft und nie gespeichert. iCloud Private Relay wird durchgelassen, kommerzielle VPNs und Cloud-Desktops werden nicht gezählt.',
|
|
138
140
|
'Stored data does not change — switching back and forth is safe.': 'Gespeicherte Daten ändern sich nicht, Hin- und Herwechseln ist gefahrlos.',
|
|
139
141
|
'Unique visitors per day are an approximation outside UTC: the anonymous visitor ID changes at midnight UTC.': 'Eindeutige Besucher pro Tag sind außerhalb von UTC eine Näherung: Die anonyme Besucher-ID wechselt um Mitternacht UTC.',
|
|
140
142
|
'Unknown time zone': 'Unbekannte Zeitzone',
|
|
141
143
|
'Direct / none': 'Direkt / keine', 'Android app': 'Android-App', 'iOS app': 'iOS-App', 'Web': 'Web',
|
|
142
144
|
'For scripts, agents and the CLI (': 'Für Skripte, Agents und die CLI (',
|
|
143
145
|
'). A key acts as you, sees only your sites, and cannot touch the team or your security settings. Read-only unless you untick it.': '). Ein Key handelt als du, sieht nur deine Seiten und kann weder das Team noch deine Sicherheitseinstellungen ändern. Nur lesen, außer du nimmst den Haken raus.',
|
|
146
|
+
'). A key acts as you, sees only your sites, and cannot touch the team or your security settings.': '). Ein Key handelt als du, sieht nur deine Seiten und kann weder das Team noch deine Sicherheitseinstellungen ändern.',
|
|
147
|
+
'For server mode, create a “server mode (count only)” key: it can only report page views for the sites you tick, nothing else. Never put a read + write key on a web server.': 'Für den Server-Modus legst du einen Key „Server-Modus (nur zählen)“ an: Er kann nur Seitenaufrufe für die angehakten Seiten melden, sonst nichts. Leg nie einen Key mit Schreibrechten auf einen Webserver.',
|
|
148
|
+
'server mode (count only)': 'Server-Modus (nur zählen)', 'server mode': 'Server-Modus', 'Reports for:': 'Meldet für:',
|
|
144
149
|
'Signed in as': 'Angemeldet als',
|
|
145
150
|
'Viewer — read only': 'Viewer: nur lesen', 'Admin — sites, share links and funnels (all or picked sites)': 'Admin: Seiten, Share-Links und Funnels (alle oder ausgewählte Seiten)', 'Owner — everything, including the team': 'Owner: alles, auch das Team',
|
|
146
151
|
'In Plausible:': 'In Plausible:', 'Site settings → Imports & Exports → Export data': 'Website-Einstellungen → Imports & Exports → Export data',
|
|
@@ -163,6 +168,7 @@
|
|
|
163
168
|
[/^Days from (.+) on already have own data — imported numbers count only before that day\.$/, 'Ab $1 gibt es schon eigene Daten. Importierte Zahlen zählen nur vor diesem Tag.'],
|
|
164
169
|
[/^Left the site \(([\d.,]+k?)\)$/, 'Seite verlassen ($1)'],
|
|
165
170
|
[/^(\d+)% deep$/, '$1 % tief'],
|
|
171
|
+
[/^(\d+) of (\d+) hits from cloud networks ignored today \(UTC\)\.$/, '$1 von $2 Aufrufen aus Cloud-Netzen heute ignoriert (UTC).'],
|
|
166
172
|
[/^(.+) — ([\d.,]+k?) measured views$/, '$1 · $2 gemessene Aufrufe'],
|
|
167
173
|
[/^· ([\d.,]+k?) sessions, top paths, 30-min session window$/, '· $1 Sitzungen, häufigste Wege, Sitzungsfenster 30 Min.'],
|
|
168
174
|
[/^([\d.,]+) searches · ([\d.,]+)% without results$/, '$1 Suchen · $2 % ohne Treffer'],
|
package/public/index.html
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
<meta name="theme-color" content="#071114">
|
|
10
10
|
<link rel="preload" href="/assets/fonts/inter-latin-wght-normal.woff2" as="font" type="font/woff2" crossorigin>
|
|
11
11
|
<script src="/theme.js?v=1"></script>
|
|
12
|
-
<link rel="stylesheet" href="/style.css?v=
|
|
12
|
+
<link rel="stylesheet" href="/style.css?v=2026102201">
|
|
13
13
|
</head>
|
|
14
14
|
<body>
|
|
15
15
|
|
|
@@ -221,12 +221,14 @@
|
|
|
221
221
|
<div class="modal">
|
|
222
222
|
<button type="button" class="modal-x" data-close aria-label="Close" title="Close"><svg class="ico ui" aria-hidden="true"><use href="/assets/icons.svg?v=2#i-x"></use></svg></button>
|
|
223
223
|
<span class="logo em"><svg class="ico big" aria-hidden="true"><use href="/assets/icons.svg?v=2#i-timer"></use></svg></span>
|
|
224
|
-
<h2>
|
|
224
|
+
<h2>Site settings</h2>
|
|
225
225
|
<p><span>Days, hours, the weekly report and alerts follow this time zone for</span> <b id="tzSite"></b></p>
|
|
226
226
|
<p>Stored data does not change — switching back and forth is safe.</p>
|
|
227
227
|
<p class="muted small">Unique visitors per day are an approximation outside UTC: the anonymous visitor ID changes at midnight UTC.</p>
|
|
228
228
|
<div id="tzErr" class="err"></div>
|
|
229
229
|
<select id="tzSelect" class="tz-select"></select>
|
|
230
|
+
<label class="check" style="display:flex;gap:8px;align-items:flex-start;margin:14px 0 4px;text-align:left"><input type="checkbox" id="dcFilter"> <span>Ignore visits from cloud and hosting networks (scrapers). The IP is checked on arrival and never stored. iCloud Private Relay is let through; commercial VPNs and cloud desktops are not counted.</span></label>
|
|
231
|
+
<p id="dcInfo" class="muted small" style="margin:0 0 10px;text-align:left"></p>
|
|
230
232
|
<button class="btn" id="saveTz">Save</button>
|
|
231
233
|
<button class="btn ghost" id="cancelTz" style="margin-top:10px">Cancel</button>
|
|
232
234
|
</div>
|
|
@@ -389,8 +391,8 @@
|
|
|
389
391
|
</div>
|
|
390
392
|
|
|
391
393
|
<div class="tip" id="tip"></div>
|
|
392
|
-
<script src="/i18n.js?v=2.2.
|
|
393
|
-
<script src="/account.js?v=2.5.
|
|
394
|
-
<script src="/app.js?v=2.7.
|
|
394
|
+
<script src="/i18n.js?v=2.2.6"></script>
|
|
395
|
+
<script src="/account.js?v=2.5.9"></script>
|
|
396
|
+
<script src="/app.js?v=2.7.3"></script>
|
|
395
397
|
</body>
|
|
396
398
|
</html>
|
package/server.js
CHANGED
|
@@ -24,6 +24,8 @@ for (const dir of [...new Set([process.cwd(), __dirname])]) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
const PORT = parseInt(process.env.PORT) || 3920;
|
|
27
|
+
// Bind address. Default loopback (put TLS in front); the Docker image sets HOST=0.0.0.0 inside the container.
|
|
28
|
+
const HOST = process.env.HOST || '127.0.0.1';
|
|
27
29
|
const WEB_PASSWORD = process.env.WEB_PASSWORD || 'tacilytics';
|
|
28
30
|
const API_KEY = process.env.API_KEY || ''; // for agents / CLI (Bearer token)
|
|
29
31
|
const SESSION_SECRET = process.env.SESSION_SECRET || crypto.randomBytes(16).toString('hex');
|
|
@@ -145,11 +147,45 @@ const norm = (d) => String(d || '').toLowerCase().replace(/^www\./, '').replace(
|
|
|
145
147
|
if (!db.prepare('PRAGMA table_info(sites)').all().some((c) => c.name === 'timezone')) db.exec("ALTER TABLE sites ADD COLUMN timezone TEXT NOT NULL DEFAULT 'UTC'");
|
|
146
148
|
// paused sites collect nothing (hosting: trial over / subscription ended). Only an unlimited admin or owner can pause.
|
|
147
149
|
if (!db.prepare('PRAGMA table_info(sites)').all().some((c) => c.name === 'paused')) db.exec('ALTER TABLE sites ADD COLUMN paused INTEGER NOT NULL DEFAULT 0');
|
|
150
|
+
// datacenter filter (2.5): drop visits from cloud / hosting networks (IP checked on arrival, never stored). On by default.
|
|
151
|
+
if (!db.prepare('PRAGMA table_info(sites)').all().some((c) => c.name === 'dc_filter')) db.exec('ALTER TABLE sites ADD COLUMN dc_filter INTEGER NOT NULL DEFAULT 1');
|
|
148
152
|
const getSiteByDomain = db.prepare('SELECT * FROM sites WHERE domain = ?');
|
|
149
153
|
const insSite = db.prepare("INSERT OR IGNORE INTO sites (domain, name, created, timezone) VALUES (?, ?, ?, COALESCE(?, 'UTC'))");
|
|
150
154
|
const allSites = db.prepare('SELECT * FROM sites ORDER BY domain');
|
|
151
155
|
const insEvent = db.prepare(`INSERT INTO events (site_id, ts, day, path, source, referrer, country, tz, browser, os, device, lang, name, utm_medium, utm_campaign, dur, props, vh, platform, app_version)
|
|
152
156
|
VALUES (@site_id, @ts, @day, @path, @source, @referrer, @country, @tz, @browser, @os, @device, @lang, @name, @utm_medium, @utm_campaign, @dur, @props, @vh, @platform, @app_version)`);
|
|
157
|
+
// visit source (2.5): like GA/Plausible/Matomo, the source belongs to the VISIT, not to each page view.
|
|
158
|
+
// A page view without an third-party referrer and without UTM (an internal click, a reload) inherits the
|
|
159
|
+
// source of the same visitor's previous page view within 30 minutes. Server-side only (visitor hash),
|
|
160
|
+
// so strict mode reads nothing new on the device.
|
|
161
|
+
const VISIT_GAP = 30 * 60000;
|
|
162
|
+
// The visitor hash is salted per UTC day, so a visit never spans two days: `day` is exact and lets
|
|
163
|
+
// SQLite use idx_events_vh instead of scanning every event of the site in the window.
|
|
164
|
+
const lastSource = db.prepare('SELECT source, utm_medium, utm_campaign FROM events INDEXED BY idx_events_vh WHERE site_id = ? AND vh = ? AND day = ? AND name IS NULL AND ts >= ? ORDER BY id DESC LIMIT 1');
|
|
165
|
+
// → { source, medium, campaign }. UTM medium / campaign travel with an inherited source; a page
|
|
166
|
+
// view with its own UTM keeps its own. A return from a payment provider keeps the visit's source
|
|
167
|
+
// (the raw referrer column still has the host).
|
|
168
|
+
function visitSource(siteId, vh, day, ts, utm, referrer, domain, medium, campaign) {
|
|
169
|
+
if (utm) return { source: utm, medium, campaign };
|
|
170
|
+
if (referrer) {
|
|
171
|
+
let h = '';
|
|
172
|
+
try { h = norm(new URL(referrer).hostname); } catch {}
|
|
173
|
+
if (h && h !== domain && !isPaymentHost(h)) return { source: h, medium, campaign };
|
|
174
|
+
if (h && h !== domain) {
|
|
175
|
+
const prev = lastSource.get(siteId, vh, day, ts - VISIT_GAP);
|
|
176
|
+
return prev ? { source: prev.source, medium: medium || prev.utm_medium, campaign: campaign || prev.utm_campaign } : { source: h, medium, campaign };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (medium || campaign) {
|
|
180
|
+
// internal link with its own utm_medium / campaign but no source: keep the visit's source, use the own medium (like GA)
|
|
181
|
+
let own = false;
|
|
182
|
+
try { own = !!referrer && norm(new URL(referrer).hostname) === domain; } catch {}
|
|
183
|
+
const prev = own ? lastSource.get(siteId, vh, day, ts - VISIT_GAP) : null;
|
|
184
|
+
return { source: (prev && prev.source) || 'Direct', medium, campaign };
|
|
185
|
+
}
|
|
186
|
+
const prev = lastSource.get(siteId, vh, day, ts - VISIT_GAP);
|
|
187
|
+
return prev ? { source: prev.source || 'Direct', medium: prev.utm_medium, campaign: prev.utm_campaign } : { source: 'Direct', medium: null, campaign: null };
|
|
188
|
+
}
|
|
153
189
|
// engagement: attach a visit duration (seconds) to the visitor's most recent pageview for that path.
|
|
154
190
|
const updDur = db.prepare(`UPDATE events SET dur = MAX(COALESCE(dur, 0), @dur), scroll = CASE WHEN @scroll IS NULL THEN scroll ELSE MAX(COALESCE(scroll, 0), @scroll) END WHERE id = (
|
|
155
191
|
SELECT id FROM events WHERE site_id = @site_id AND vh = @vh AND name IS NULL AND path = @path AND ts >= @since
|
|
@@ -216,9 +252,14 @@ function appPlatform(b) {
|
|
|
216
252
|
|
|
217
253
|
// ── UA parsing + bot filter ─────────────────────────────────────────────────
|
|
218
254
|
const BOT_RE = /bot|crawl|spider|slurp|bing|google|yandex|baidu|duckduck|facebookexternal|embedly|quora|pinterest|slackbot|telegrambot|whatsapp|twitter|discordbot|preview|monitor|uptime|pingdom|lighthouse|headless|phantom|curl|wget|python-requests|axios|node-fetch|go-http/i;
|
|
255
|
+
// more non-browser clients (2.5): AI / dataset scrapers and HTTP libraries without "bot" in the name
|
|
256
|
+
// Split so native apps can still use a plain HTTP client: apps (with `pf`) are only checked against
|
|
257
|
+
// BOT_RE + BOT_SCRAPER_RE, websites additionally against BOT_LIB_RE (React Native / OkHttp apps send `okhttp/4.x`).
|
|
258
|
+
const BOT_SCRAPER_RE = /third-partyagent|facebookcatalog|datasets|scrapy|ia_archiver|archive\.org|dataprovider|selenium|puppeteer|playwright/i;
|
|
259
|
+
const BOT_LIB_RE = /okhttp|java\/|libwww|httpx|aiohttp|httpclient|python-urllib|^mozilla\/5\.0$/i;
|
|
219
260
|
function parseUA(ua) {
|
|
220
261
|
ua = ua || '';
|
|
221
|
-
if (!ua || BOT_RE.test(ua)) return null; // drop bots/crawlers entirely
|
|
262
|
+
if (!ua || BOT_RE.test(ua) || BOT_SCRAPER_RE.test(ua) || BOT_LIB_RE.test(ua)) return null; // drop bots/crawlers entirely
|
|
222
263
|
let browser = 'Other', os = 'Other', device = 'Desktop';
|
|
223
264
|
if (/iPad/i.test(ua) || (/Tablet/i.test(ua) && !/Mobile/i.test(ua))) device = 'Tablet';
|
|
224
265
|
else if (/Mobi|Android|iPhone|iPod/i.test(ua)) device = 'Mobile';
|
|
@@ -250,6 +291,33 @@ const { createDaySalt, dayStr } = require('./day-salt.js');
|
|
|
250
291
|
// Daily salt: fresh random secret per UTC day in data/.day-salt (0600), never in the DB.
|
|
251
292
|
// Rules and the single-process note live in day-salt.js.
|
|
252
293
|
const SALT_FILE = path.join(path.dirname(DB_PATH), '.day-salt');
|
|
294
|
+
const dc = require('./datacenter.js').createDatacenterCheck({ dir: path.dirname(DB_PATH) });
|
|
295
|
+
dc.start();
|
|
296
|
+
// A wrong proxy setup (TRUST_PROXY too low, proxy on a cloud box, server mode forwarding its own IP)
|
|
297
|
+
// would make EVERY hit look like a datacenter and vanish. Count per site and hour (numbers only, no
|
|
298
|
+
// IPs) and warn loudly when most of a site's traffic is being dropped.
|
|
299
|
+
const dcStats = new Map();
|
|
300
|
+
// per UTC day, numbers only: shown in the dashboard ("N hits from cloud networks ignored today"), so a
|
|
301
|
+
// Cloud customer who never sees the server log still notices a 100 % drop. Lost on restart, that is fine.
|
|
302
|
+
const dcToday = new Map();
|
|
303
|
+
function dcDropped(site, dropped) {
|
|
304
|
+
const now = Date.now();
|
|
305
|
+
const hour = Math.floor(now / 3600000);
|
|
306
|
+
let st = dcStats.get(site.id);
|
|
307
|
+
if (!st || st.hour !== hour) { st = { hour, total: 0, dropped: 0, warned: false }; dcStats.set(site.id, st); }
|
|
308
|
+
st.total++; if (dropped) st.dropped++;
|
|
309
|
+
const day = dayStr(now);
|
|
310
|
+
let d = dcToday.get(site.id);
|
|
311
|
+
if (!d || d.day !== day) { d = { day, total: 0, dropped: 0 }; dcToday.set(site.id, d); }
|
|
312
|
+
d.total++; if (dropped) d.dropped++;
|
|
313
|
+
// big sites: most of the hour dropped. Small sites (a few hits an hour) never reach 20, so also warn
|
|
314
|
+
// when every one of at least 5 hits was dropped.
|
|
315
|
+
if (!st.warned && ((st.total >= 20 && st.dropped / st.total > 0.5) || (st.total >= 5 && st.dropped === st.total))) {
|
|
316
|
+
st.warned = true;
|
|
317
|
+
console.warn(`⚠️ datacenter filter: ${st.dropped} of ${st.total} hits for ${site.domain} this hour came from cloud / hosting networks and were not counted. If that is not a bot wave, check TRUST_PROXY (and the ip you send in server mode), or switch the filter off for this site.`);
|
|
318
|
+
}
|
|
319
|
+
return dropped;
|
|
320
|
+
}
|
|
253
321
|
const todaysSalt = createDaySalt(SALT_FILE);
|
|
254
322
|
// one-time cleanup of the short-lived DB table from 03934ed: zero the pages, flush the WAL
|
|
255
323
|
db.pragma('secure_delete = ON');
|
|
@@ -330,7 +398,7 @@ function collect(req, res) {
|
|
|
330
398
|
// A bot-looking UA is still dropped, so the bot filter cannot be bypassed by just adding `pf`.
|
|
331
399
|
const rawUa = req.headers['user-agent'] || '';
|
|
332
400
|
let ua = parseUA(rawUa);
|
|
333
|
-
if (platform && rawUa && !BOT_RE.test(rawUa)) {
|
|
401
|
+
if (platform && rawUa && !BOT_RE.test(rawUa) && !BOT_SCRAPER_RE.test(rawUa)) {
|
|
334
402
|
const dv = String(b.dv || '').toLowerCase();
|
|
335
403
|
ua = { browser: 'App', os: APP_PLATFORMS[platform], device: dv === 'tablet' ? 'Tablet' : dv === 'desktop' ? 'Desktop' : (platform === 'android' || platform === 'ios') ? 'Mobile' : 'Desktop' };
|
|
336
404
|
}
|
|
@@ -365,17 +433,16 @@ function collect(req, res) {
|
|
|
365
433
|
if (dur > 0 || sc != null) updDur.run({ site_id: site.id, vh, path: pth, dur, scroll: sc, since: ts - 6 * 3600000 });
|
|
366
434
|
return res.status(204).end();
|
|
367
435
|
}
|
|
436
|
+
// after the bot / DNT / internal / engagement exits: only hits that would really be counted enter the statistics.
|
|
437
|
+
// (Engagement beacons from a datacenter IP only update a row that never existed, so they need no check.)
|
|
438
|
+
if (site.dc_filter && dcDropped(site, dc.isDatacenter(ip))) return res.status(204).end(); // cloud / hosting network: not a person
|
|
368
439
|
|
|
369
440
|
// referrer → source host
|
|
370
441
|
let referrer = String(b.r || b.referrer || '').slice(0, 512);
|
|
371
|
-
let source = 'Direct';
|
|
372
442
|
const utm = String(b.s || b.utm || '').slice(0, 60);
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
377
|
-
const utmMedium = String(b.um || b.utm_medium || '').slice(0, 60).trim() || null;
|
|
378
|
-
const utmCampaign = String(b.uc || b.utm_campaign || '').slice(0, 80).trim() || null;
|
|
443
|
+
const vs = visitSource(site.id, vh, day, ts, utm, referrer, domain,
|
|
444
|
+
String(b.um || b.utm_medium || '').slice(0, 60).trim() || null, String(b.uc || b.utm_campaign || '').slice(0, 80).trim() || null);
|
|
445
|
+
const source = vs.source; const utmMedium = vs.medium; const utmCampaign = vs.campaign;
|
|
379
446
|
const tz = strict ? null : (String(b.tz || '').slice(0, 60) || null);
|
|
380
447
|
const lang = strict ? acceptLang(req.headers['accept-language']) : (String(b.l || b.lang || '').slice(0, 12).split(',')[0] || null);
|
|
381
448
|
const name = String(b.n || b.name || '').slice(0, 80).trim() || null; // custom event / goal
|
|
@@ -402,6 +469,7 @@ const { auth, requireRole, siteAllowed, audit, grantSite, isLimited } = setupAut
|
|
|
402
469
|
allSiteIds: () => db.prepare('SELECT id FROM sites').all().map((r) => r.id),
|
|
403
470
|
});
|
|
404
471
|
const adminOnly = requireRole('admin');
|
|
472
|
+
const ROLE_RANK = { viewer: 1, admin: 2, owner: 3 };
|
|
405
473
|
|
|
406
474
|
// share-token auth: read-only stats access scoped to exactly one site.
|
|
407
475
|
// Accepted as ?share=<token>; the handler must check siteOk(req, site).
|
|
@@ -420,7 +488,14 @@ const NO_SITE = { error: 'You have no access to this site' };
|
|
|
420
488
|
function newToken() { return crypto.randomBytes(24).toString('hex'); }
|
|
421
489
|
|
|
422
490
|
// ── sites management ─────────────────────────────────────────────────────────
|
|
423
|
-
|
|
491
|
+
// dc_active: the datacenter filter really runs on this server (list + relay allowlist loaded, not switched off).
|
|
492
|
+
// dc_dropped_today / dc_seen_today: hits checked and ignored today (UTC), numbers only.
|
|
493
|
+
function withDcInfo(st) {
|
|
494
|
+
const d = dcToday.get(st.id);
|
|
495
|
+
const fresh = d && d.day === dayStr(Date.now());
|
|
496
|
+
return { ...st, dc_active: dc.active, dc_dropped_today: fresh ? d.dropped : 0, dc_seen_today: fresh ? d.total : 0 };
|
|
497
|
+
}
|
|
498
|
+
const sitesListHandler = (req, res) => res.json(allSites.all().filter((st) => siteAllowed(req, st.id)).map(withDcInfo));
|
|
424
499
|
const sitesCreateHandler = (req, res) => {
|
|
425
500
|
// a site-limited account (a hosting customer) never creates sites itself: the operator does, because sites are what
|
|
426
501
|
// is billed and paused. Otherwise a paused customer could add a fresh, unpaused site and move the snippet (Kiro, 2.3.0).
|
|
@@ -460,20 +535,31 @@ app.patch('/api/sites/:id', adminOnly, (req, res) => {
|
|
|
460
535
|
const site = db.prepare('SELECT * FROM sites WHERE id = ?').get(id);
|
|
461
536
|
if (!site) return res.status(404).json({ error: 'Unknown site' });
|
|
462
537
|
const b = req.body || {};
|
|
538
|
+
// validate everything first, then write: a 400 must never leave a half-applied (and unaudited) change
|
|
539
|
+
if (b.paused !== undefined && isLimited(req)) return res.status(403).json({ error: 'Your role does not allow this' }); // operator's switch (hosting)
|
|
540
|
+
if (b.dcFilter !== undefined && typeof b.dcFilter !== 'boolean') return res.status(400).json({ error: 'dcFilter must be true or false' });
|
|
541
|
+
const setTz = b.timezone !== undefined || (b.paused === undefined && b.dcFilter === undefined);
|
|
542
|
+
const timezone = setTz ? TZ.validTz(String(b.timezone || '')) : null;
|
|
543
|
+
if (setTz && !timezone) return res.status(400).json({ error: 'Unknown time zone' });
|
|
544
|
+
// all writes in one transaction: either every change lands or none
|
|
545
|
+
db.transaction(() => {
|
|
463
546
|
if (b.paused !== undefined) {
|
|
464
|
-
// pausing is the operator's switch (hosting: trial or subscription ended), never the customer's
|
|
465
|
-
if (isLimited(req)) return res.status(403).json({ error: 'Your role does not allow this' });
|
|
466
547
|
const paused = b.paused ? 1 : 0;
|
|
467
548
|
db.prepare('UPDATE sites SET paused = ? WHERE id = ?').run(paused, id);
|
|
468
549
|
if (paused !== site.paused) audit(res, paused ? 'site.pause' : 'site.resume', site.domain);
|
|
469
550
|
}
|
|
470
|
-
if (b.
|
|
471
|
-
|
|
472
|
-
|
|
551
|
+
if (b.dcFilter !== undefined) {
|
|
552
|
+
// the site's own admin may switch it: it only decides which of their own visits are counted
|
|
553
|
+
const v = b.dcFilter ? 1 : 0;
|
|
554
|
+
db.prepare('UPDATE sites SET dc_filter = ? WHERE id = ?').run(v, id);
|
|
555
|
+
if (v !== site.dc_filter) audit(res, v ? 'site.dcfilter.on' : 'site.dcfilter.off', site.domain);
|
|
556
|
+
}
|
|
557
|
+
if (setTz) {
|
|
473
558
|
db.prepare('UPDATE sites SET timezone = ? WHERE id = ?').run(timezone, id);
|
|
474
559
|
if (timezone !== site.timezone) audit(res, 'site.timezone', site.domain + ': ' + site.timezone + ' → ' + timezone);
|
|
475
560
|
}
|
|
476
|
-
|
|
561
|
+
})();
|
|
562
|
+
res.json(withDcInfo(db.prepare('SELECT * FROM sites WHERE id = ?').get(id)));
|
|
477
563
|
});
|
|
478
564
|
app.delete('/api/sites/:id', adminOnly, (req, res) => {
|
|
479
565
|
const id = Number(req.params.id);
|
|
@@ -501,7 +587,7 @@ app.get('/api/v1', (_req, res) => res.json({
|
|
|
501
587
|
'GET /api/v1/stats?site=&period=': 'aggregated stats (period: today|7d|30d|90d|12mo, or from=YYYY-MM-DD&to=YYYY-MM-DD)',
|
|
502
588
|
'GET /api/v1/realtime?site=': 'online count, last-30-min pulse + recent events',
|
|
503
589
|
'GET /api/v1/usage?month=YYYY-MM': 'page views per site in a calendar month (UTC), owner / server key only',
|
|
504
|
-
'POST /api/v1/hit': 'server mode: report a page view from your web server { domain, path, ip, ua, referrer?, lang?, purpose?, name? } (
|
|
590
|
+
'POST /api/v1/hit': 'server mode: report a page view from your web server { domain, path, ip, ua, referrer?, lang?, purpose?, name? } (server-mode key)',
|
|
505
591
|
},
|
|
506
592
|
filters: 'add any of country,source,page,browser,os,device,lang,medium,campaign,goal to /stats to segment',
|
|
507
593
|
}));
|
|
@@ -524,9 +610,13 @@ app.post('/api/v1/sites', adminOnly, sitesCreateHandler);
|
|
|
524
610
|
// exactly like a tracker hit and never stored.
|
|
525
611
|
const net = require('net');
|
|
526
612
|
const PREFETCH_RE = /prefetch|prerender/i;
|
|
527
|
-
//
|
|
528
|
-
|
|
529
|
-
|
|
613
|
+
// Who may report: a server-mode key (scope 'hit', recommended, bound to its sites and good for nothing else), or —
|
|
614
|
+
// as in 2.4.0 — a key / sign-in with write access (admin role). A read-only key is a viewer and gets 403.
|
|
615
|
+
app.post('/api/v1/hit', auth, (req, res) => {
|
|
616
|
+
const p = req.principal;
|
|
617
|
+
if (p.kind !== 'hitkey' && ROLE_RANK[p.role] < ROLE_RANK.admin) return res.status(403).json({ error: 'Use a server-mode key (Settings → API keys)' });
|
|
618
|
+
// per key, not per web-server IP: many sites behind one shared-hosting IP must not starve each other
|
|
619
|
+
if (rateLimited('h:' + (p.keyId != null ? 'k' + p.keyId : p.kind + ':' + (p.user ? p.user.id : '')), 6000, 60000)) return res.status(429).json({ error: 'Too many hits' });
|
|
530
620
|
const b = req.body || {};
|
|
531
621
|
const domain = norm(b.domain || b.d);
|
|
532
622
|
const site = domain && getSiteByDomain.get(domain);
|
|
@@ -541,6 +631,9 @@ app.post('/api/v1/hit', adminOnly, (req, res) => {
|
|
|
541
631
|
const ua = parseUA(rawUa);
|
|
542
632
|
if (!ua) return res.status(204).end(); // bot, crawler, monitor or no user agent
|
|
543
633
|
if (b.dnt === true || b.dnt === '1' || b.gpc === true || b.gpc === '1') return res.status(204).end(); // forwarded DNT / GPC (opt-in on the sending side)
|
|
634
|
+
// after prefetch / bot / DNT: server mode forwards crawlers too (Googlebot sits in cloud ranges), they must not
|
|
635
|
+
// inflate the drop statistics and trigger false TRUST_PROXY warnings
|
|
636
|
+
if (site.dc_filter && dcDropped(site, dc.isDatacenter(ip))) return res.status(204).end(); // cloud / hosting network: not a person
|
|
544
637
|
|
|
545
638
|
const ts = Date.now();
|
|
546
639
|
const day = dayStr(ts);
|
|
@@ -556,17 +649,17 @@ app.post('/api/v1/hit', adminOnly, (req, res) => {
|
|
|
556
649
|
|
|
557
650
|
const referrer = String(b.referrer || b.r || '').slice(0, 512);
|
|
558
651
|
const q = url.searchParams;
|
|
559
|
-
let source = 'Direct';
|
|
560
652
|
const utm = String(q.get('utm_source') || q.get('ref') || '').slice(0, 60).trim();
|
|
561
|
-
|
|
562
|
-
|
|
653
|
+
const vs = visitSource(site.id, vh, day, ts, utm, referrer, domain,
|
|
654
|
+
String(q.get('utm_medium') || '').slice(0, 60).trim() || null, String(q.get('utm_campaign') || '').slice(0, 80).trim() || null);
|
|
655
|
+
const source = vs.source;
|
|
563
656
|
const name = String(b.name || b.n || '').slice(0, 80).trim() || null; // optional server-side goal (e.g. Purchase)
|
|
564
657
|
insEvent.run({
|
|
565
658
|
site_id: site.id, ts, day, path: pth, source, referrer: referrer || null,
|
|
566
659
|
country: null, tz: null, browser: ua.browser, os: ua.os, device: ua.device,
|
|
567
660
|
lang: acceptLang(b.lang), name,
|
|
568
|
-
utm_medium:
|
|
569
|
-
utm_campaign:
|
|
661
|
+
utm_medium: vs.medium,
|
|
662
|
+
utm_campaign: vs.campaign,
|
|
570
663
|
dur: null, props: name ? sanitizeProps(b.props) : null, vh, platform: null, app_version: null,
|
|
571
664
|
});
|
|
572
665
|
res.status(204).end();
|
|
@@ -1471,4 +1564,4 @@ if (process.env.TACILYTICS_TEST_HOOKS === '1') {
|
|
|
1471
1564
|
app.all(/^\/(api|auth)(\/|$)/, (_req, res) => res.status(404).json({ error: 'Not found' }));
|
|
1472
1565
|
app.get('*', sendIndex);
|
|
1473
1566
|
|
|
1474
|
-
app.listen(PORT,
|
|
1567
|
+
app.listen(PORT, HOST, () => console.log(`Tacilytics on http://${HOST}:${PORT}${RETENTION_DAYS ? ` · retention ${RETENTION_DAYS}d` : ''}${DIGEST_WEBHOOK_URL ? ` · digest ${DIGEST_HOUR_UTC}:00 UTC` : ''}`));
|