plausible2umami 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plausible2umami-0.1.0/.github/workflows/publish.yml +25 -0
- plausible2umami-0.1.0/.github/workflows/test.yml +20 -0
- plausible2umami-0.1.0/.gitignore +8 -0
- plausible2umami-0.1.0/LICENSE +21 -0
- plausible2umami-0.1.0/PKG-INFO +276 -0
- plausible2umami-0.1.0/README.md +249 -0
- plausible2umami-0.1.0/pyproject.toml +40 -0
- plausible2umami-0.1.0/src/plausible2umami/__init__.py +1 -0
- plausible2umami-0.1.0/src/plausible2umami/__main__.py +6 -0
- plausible2umami-0.1.0/src/plausible2umami/cli.py +214 -0
- plausible2umami-0.1.0/src/plausible2umami/extract.py +194 -0
- plausible2umami-0.1.0/src/plausible2umami/load.py +143 -0
- plausible2umami-0.1.0/src/plausible2umami/transform.py +250 -0
- plausible2umami-0.1.0/tests/test_transform.py +254 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
# Trusted publishing: PyPI verifies this workflow's identity through OIDC, so
|
|
4
|
+
# there is no API token to store or rotate. The matching publisher has to be
|
|
5
|
+
# configured on PyPI for owner einsz, repo plausible2umami, workflow
|
|
6
|
+
# publish.yml, with the environment field left blank.
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
release:
|
|
10
|
+
types: [published]
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
publish:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
permissions:
|
|
16
|
+
id-token: write
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.13"
|
|
22
|
+
- run: pip install build
|
|
23
|
+
- run: python -m build
|
|
24
|
+
- run: pip install twine && twine check dist/*
|
|
25
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
pytest:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python: ["3.10", "3.13"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python }}
|
|
19
|
+
- run: pip install -e '.[dev]'
|
|
20
|
+
- run: pytest -q
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 einsz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: plausible2umami
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Migrate self-hosted Plausible analytics into Umami v3, event by event
|
|
5
|
+
Project-URL: Homepage, https://github.com/einsz/plausible2umami
|
|
6
|
+
Project-URL: Issues, https://github.com/einsz/plausible2umami/issues
|
|
7
|
+
Author: einsz
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: analytics,clickhouse,migration,plausible,umami
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: System Administrators
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Internet :: Log Analysis
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Provides-Extra: all
|
|
18
|
+
Requires-Dist: clickhouse-connect>=0.7; extra == 'all'
|
|
19
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'all'
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
|
+
Provides-Extra: extract
|
|
23
|
+
Requires-Dist: clickhouse-connect>=0.7; extra == 'extract'
|
|
24
|
+
Provides-Extra: load
|
|
25
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'load'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# plausible2umami
|
|
29
|
+
|
|
30
|
+
Migrate **self-hosted** Plausible analytics into Umami v3, one event at a time.
|
|
31
|
+
|
|
32
|
+
Most Plausible-to-Umami tools read Plausible's CSV export. That export is daily
|
|
33
|
+
totals per dimension, not events, so those tools have to invent the sessions
|
|
34
|
+
that Umami needs: attributes drawn independently from daily distributions,
|
|
35
|
+
pageviews spread round-robin across synthetic visits. The daily chart ends up
|
|
36
|
+
about right and every individual row is fiction.
|
|
37
|
+
|
|
38
|
+
If you self-host Plausible you don't have to accept that. Your raw events are
|
|
39
|
+
sitting in ClickHouse with real timestamps, real session ids, referrers and
|
|
40
|
+
device data, one row per event. This tool reads those.
|
|
41
|
+
|
|
42
|
+
## What it needs
|
|
43
|
+
|
|
44
|
+
- Plausible Community Edition, self-hosted, with access to its ClickHouse
|
|
45
|
+
(the `events_v2` table)
|
|
46
|
+
- Umami **v3.x on PostgreSQL**. v3 dropped MySQL and MariaDB, and this tool
|
|
47
|
+
targets v3 only. The v3 images live on Docker Hub as
|
|
48
|
+
`umamisoftware/umami:3.3.1` rather than ghcr, and the `postgresql-` tag
|
|
49
|
+
prefix is gone now that Postgres is the only option
|
|
50
|
+
|
|
51
|
+
## Use
|
|
52
|
+
|
|
53
|
+
Create the target website in Umami's UI first, then copy its id out of the
|
|
54
|
+
tracking snippet.
|
|
55
|
+
|
|
56
|
+
Find the Plausible site id and see how much data there is:
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
docker compose exec plausible_events_db clickhouse-client \
|
|
60
|
+
-q "SELECT site_id, count() AS events, min(timestamp), max(timestamp)
|
|
61
|
+
FROM plausible_events_db.events_v2 GROUP BY site_id"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Extract, inspect, load:
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
pip install 'plausible2umami[all] @ git+https://github.com/einsz/plausible2umami'
|
|
68
|
+
|
|
69
|
+
plausible2umami extract --site-id 1 --out events.jsonl
|
|
70
|
+
plausible2umami load --in events.jsonl --website-id <uuid> --dry-run
|
|
71
|
+
plausible2umami load --in events.jsonl --website-id <uuid> \
|
|
72
|
+
--dsn postgres://umami:pass@localhost:5432/umami
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
If ClickHouse isn't reachable from where you're running this, which is common,
|
|
76
|
+
skip `extract` and produce the same file from inside the container:
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
plausible2umami query --site-id 1 # prints the SQL
|
|
80
|
+
|
|
81
|
+
docker exec <clickhouse-container> clickhouse-client \
|
|
82
|
+
-q "<that SQL> FORMAT JSONEachRow" | gzip > events.jsonl.gz
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The output format is identical either way, and `load` reads `.gz` directly.
|
|
86
|
+
Keep the query's `ORDER BY` if you write it out by hand: it groups each
|
|
87
|
+
visitor's events together and puts identical events next to each other, which
|
|
88
|
+
is what lets the loader stream a multi-gigabyte file in flat memory.
|
|
89
|
+
|
|
90
|
+
### Large sites: extract one day at a time
|
|
91
|
+
|
|
92
|
+
Do not run the single-query extract against a site with millions of events on a
|
|
93
|
+
small server. The `ORDER BY` makes ClickHouse sort the whole result, and
|
|
94
|
+
Plausible's own `low-resources.xml` caps threads while leaving sort memory
|
|
95
|
+
unbounded, with `max_bytes_before_external_sort` at its default of 0, meaning
|
|
96
|
+
it never spills to disk. On a 4 GB host with Plausible already resident, a
|
|
97
|
+
13.5 million row sort takes the machine into swap.
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
plausible2umami query --site-id 1 --by-day --out /tmp/site1 > extract.sh
|
|
101
|
+
sh extract.sh
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
That fetches a day per query, so no sort exceeds one day's rows, and it reads
|
|
105
|
+
only the relevant parts because `site_id` and the date lead the table's primary
|
|
106
|
+
key. One file per day means an interrupted run resumes rather than restarting,
|
|
107
|
+
and the parts are concatenated at the end without recompression, since joined
|
|
108
|
+
gzip members are valid gzip.
|
|
109
|
+
|
|
110
|
+
This is safe because Plausible's `user_id` is a hash whose salt rotates daily,
|
|
111
|
+
so every row for a given visitor falls inside one day. Chunking by day
|
|
112
|
+
preserves exactly the grouping the loader depends on.
|
|
113
|
+
|
|
114
|
+
### Scale, and what it costs in disk
|
|
115
|
+
|
|
116
|
+
Both phases stream, so memory stays flat regardless of size. Disk does not.
|
|
117
|
+
|
|
118
|
+
Measured on a real migration of 9,753,130 events spanning two and a half years,
|
|
119
|
+
loaded into a fresh Umami v3.3.1:
|
|
120
|
+
|
|
121
|
+
| | |
|
|
122
|
+
|---|---|
|
|
123
|
+
| Extract, gzipped JSONL | 127 MB |
|
|
124
|
+
| Load time | 33 minutes |
|
|
125
|
+
| Postgres afterwards | **9,179 MB** |
|
|
126
|
+
| of which row data | 1,258 MB |
|
|
127
|
+
| of which indexes | **7,539 MB** |
|
|
128
|
+
| The same events in ClickHouse | **127 MiB** |
|
|
129
|
+
|
|
130
|
+
Umami puts fifteen indexes on `website_event`, and they cost six times the data
|
|
131
|
+
they index. Your events are the cheap part.
|
|
132
|
+
|
|
133
|
+
Four of them index columns Plausible has no equivalent for, so they sit empty
|
|
134
|
+
after an import: `page_title`, `tag`, `url_query`, and `event_name` unless you
|
|
135
|
+
keep engagement events. Roughly 2 GB of index over nothing.
|
|
136
|
+
|
|
137
|
+
Decide how much history you want before you start. `--since` filters at load
|
|
138
|
+
time from an extract you already have, so changing your mind costs a flag
|
|
139
|
+
rather than another trip to ClickHouse.
|
|
140
|
+
|
|
141
|
+
## What actually maps
|
|
142
|
+
|
|
143
|
+
The identity mapping is the part worth understanding, and it is easy to get
|
|
144
|
+
backwards:
|
|
145
|
+
|
|
146
|
+
| Plausible | Umami | Meaning |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| `user_id` | `session.session_id` | the visitor |
|
|
149
|
+
| `session_id` | `website_event.visit_id` | the 30-minute visit |
|
|
150
|
+
|
|
151
|
+
Plausible's `user_id` is a salted hash that rotates daily, so an imported Umami
|
|
152
|
+
session is really a visitor-day. That is a property of Plausible's data model,
|
|
153
|
+
not something this tool discards.
|
|
154
|
+
|
|
155
|
+
### Copied directly
|
|
156
|
+
|
|
157
|
+
| Plausible `events_v2` | Umami |
|
|
158
|
+
|---|---|
|
|
159
|
+
| `timestamp` | `created_at` |
|
|
160
|
+
| `name` | `event_type` (1 pageview, 2 custom) and `event_name` |
|
|
161
|
+
| `pathname` | `url_path`, `url_query` |
|
|
162
|
+
| `hostname` | `hostname` |
|
|
163
|
+
| `referrer` | `referrer_domain`, `referrer_path`, `referrer_query` |
|
|
164
|
+
| `utm_source` `utm_medium` `utm_campaign` `utm_content` `utm_term` | same names |
|
|
165
|
+
| `browser` | `session.browser` |
|
|
166
|
+
| `operating_system` | `session.os` |
|
|
167
|
+
| `screen_size` | `session.device` |
|
|
168
|
+
| `country_code` | `session.country` |
|
|
169
|
+
| `subdivision1_code` | `session.region` |
|
|
170
|
+
| `city_name` | `session.city` |
|
|
171
|
+
|
|
172
|
+
Plausible's `screen_size` is a bucket ("Desktop", "Mobile"), which is Umami's
|
|
173
|
+
`device`. Umami's `screen` wants "1920x1080" and has no source here.
|
|
174
|
+
|
|
175
|
+
### Left null, because Plausible never collected it
|
|
176
|
+
|
|
177
|
+
`session.language`, `session.screen`, `session.distinct_id`, `page_title`, and
|
|
178
|
+
the web-vitals columns (`lcp`, `inp`, `cls`, `fcp`, `ttfb`).
|
|
179
|
+
|
|
180
|
+
City is mapped. The extract selects `city_name`, an ALIAS column that resolves `city_geoname_id` against Plausible's bundled
|
|
181
|
+
`location_data_dict`, so no GeoNames table has to be shipped.
|
|
182
|
+
|
|
183
|
+
Whether you get anything depends on your GeoIP database. Plausible only records
|
|
184
|
+
`city_geoname_id` when it has a **city-level** one; with country-level data the
|
|
185
|
+
id is 0 for every event and `session.city` ends up null.
|
|
186
|
+
On the instance this was developed against, all 5,102 test events had id 0, so
|
|
187
|
+
the mapping is verified only as far as the dictionary: `dictGet` resolves ids
|
|
188
|
+
to names correctly ("Berlin", "New York City"), but no live event
|
|
189
|
+
carrying a real id was ever migrated.
|
|
190
|
+
|
|
191
|
+
If your Plausible predates the `city_name` alias, remove it from the extract
|
|
192
|
+
query. The loader treats a missing `city_name` as null rather than failing.
|
|
193
|
+
|
|
194
|
+
### Dropped, no Umami equivalent
|
|
195
|
+
|
|
196
|
+
`scroll_depth`, `engagement_time`, the `revenue_*` fields, `click_id_param`,
|
|
197
|
+
`acquisition_channel`, and `browser_version` / `operating_system_version`.
|
|
198
|
+
|
|
199
|
+
### Filtered out by default
|
|
200
|
+
|
|
201
|
+
Plausible emits an internal `engagement` event alongside pageviews, carrying
|
|
202
|
+
`scroll_depth` and `engagement_time`. Umami has nowhere to put either, so
|
|
203
|
+
importing them adds rows and no information while filling Umami's custom-events
|
|
204
|
+
list with a single meaningless name.
|
|
205
|
+
|
|
206
|
+
They are dropped unless you pass `--include-engagement`. The volume is worth
|
|
207
|
+
knowing: on the site this was first tested against, 3,557 of 5,094 events were
|
|
208
|
+
engagement. Filtering them left pageviews, visitors and visits identical and
|
|
209
|
+
cut the row count by 70%.
|
|
210
|
+
|
|
211
|
+
### Not implemented
|
|
212
|
+
|
|
213
|
+
- **Custom event properties.** `meta.key` / `meta.value` would map to Umami's
|
|
214
|
+
`event_data` table. Custom event *names* are imported; their properties are
|
|
215
|
+
not.
|
|
216
|
+
|
|
217
|
+
Nothing is fabricated. Every field outside the tables above lands as null.
|
|
218
|
+
|
|
219
|
+
## Safety
|
|
220
|
+
|
|
221
|
+
Re-running is safe. Every id is a UUIDv5 derived from the source data and the
|
|
222
|
+
target website id, and every insert is `ON CONFLICT DO NOTHING`, so a load that
|
|
223
|
+
dies partway through can just be run again. Rows already written are left
|
|
224
|
+
alone.
|
|
225
|
+
|
|
226
|
+
- `--dry-run` transforms everything and writes nothing, printing counts, the
|
|
227
|
+
date range and a sample of what would go in
|
|
228
|
+
- `--limit N` loads a slice, to see how it looks before committing
|
|
229
|
+
- Loading into a website that already has events requires `--force`, so you
|
|
230
|
+
don't accidentally mix imported history into live traffic
|
|
231
|
+
- Values are truncated to Umami's column widths rather than failing a batch
|
|
232
|
+
two thirds of the way through
|
|
233
|
+
|
|
234
|
+
Take a database dump first anyway. This writes directly to Umami's tables.
|
|
235
|
+
|
|
236
|
+
## Development
|
|
237
|
+
|
|
238
|
+
```sh
|
|
239
|
+
python -m venv .venv && .venv/bin/pip install -e '.[dev,all]'
|
|
240
|
+
.venv/bin/python -m pytest
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
The transform is pure functions over dicts, so the tests need no database.
|
|
244
|
+
|
|
245
|
+
## Compatibility
|
|
246
|
+
|
|
247
|
+
| | Verified against |
|
|
248
|
+
|---|---|
|
|
249
|
+
| Plausible CE | v3.1.0 and v3.2.1, ClickHouse 24.12 |
|
|
250
|
+
| Umami | v3.3.1, PostgreSQL 16 |
|
|
251
|
+
|
|
252
|
+
Plausible 3.1.0 to 3.2.1 changes nothing here: `events_v2` is identical in
|
|
253
|
+
column names and types, and extracting the same 5,094 events under both
|
|
254
|
+
versions and loading each produced the same sessions, the same visits and the
|
|
255
|
+
same per-path pageview counts, with no differences at all.
|
|
256
|
+
|
|
257
|
+
Correctness was checked by loading 5,094 real events from a live Plausible
|
|
258
|
+
instance into a fresh Umami v3.3.1, then comparing Umami's own dashboard
|
|
259
|
+
aggregation against the raw ClickHouse rows:
|
|
260
|
+
|
|
261
|
+
| | Plausible raw | Umami dashboard |
|
|
262
|
+
|---|---|---|
|
|
263
|
+
| pageviews | 1537 | 1537 |
|
|
264
|
+
| visitors (distinct `user_id`) | 883 | 883 |
|
|
265
|
+
| visits (distinct `session_id`) | 1089 | 1089 |
|
|
266
|
+
| top page | `/` 1522 | `/` 1522 |
|
|
267
|
+
|
|
268
|
+
Zero events referenced a missing session, and a full second run of the same
|
|
269
|
+
file wrote no new rows.
|
|
270
|
+
|
|
271
|
+
Both projects change their schemas. If yours differ, the mapping tables above
|
|
272
|
+
tell you what to check.
|
|
273
|
+
|
|
274
|
+
## Licence
|
|
275
|
+
|
|
276
|
+
MIT.
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# plausible2umami
|
|
2
|
+
|
|
3
|
+
Migrate **self-hosted** Plausible analytics into Umami v3, one event at a time.
|
|
4
|
+
|
|
5
|
+
Most Plausible-to-Umami tools read Plausible's CSV export. That export is daily
|
|
6
|
+
totals per dimension, not events, so those tools have to invent the sessions
|
|
7
|
+
that Umami needs: attributes drawn independently from daily distributions,
|
|
8
|
+
pageviews spread round-robin across synthetic visits. The daily chart ends up
|
|
9
|
+
about right and every individual row is fiction.
|
|
10
|
+
|
|
11
|
+
If you self-host Plausible you don't have to accept that. Your raw events are
|
|
12
|
+
sitting in ClickHouse with real timestamps, real session ids, referrers and
|
|
13
|
+
device data, one row per event. This tool reads those.
|
|
14
|
+
|
|
15
|
+
## What it needs
|
|
16
|
+
|
|
17
|
+
- Plausible Community Edition, self-hosted, with access to its ClickHouse
|
|
18
|
+
(the `events_v2` table)
|
|
19
|
+
- Umami **v3.x on PostgreSQL**. v3 dropped MySQL and MariaDB, and this tool
|
|
20
|
+
targets v3 only. The v3 images live on Docker Hub as
|
|
21
|
+
`umamisoftware/umami:3.3.1` rather than ghcr, and the `postgresql-` tag
|
|
22
|
+
prefix is gone now that Postgres is the only option
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
Create the target website in Umami's UI first, then copy its id out of the
|
|
27
|
+
tracking snippet.
|
|
28
|
+
|
|
29
|
+
Find the Plausible site id and see how much data there is:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
docker compose exec plausible_events_db clickhouse-client \
|
|
33
|
+
-q "SELECT site_id, count() AS events, min(timestamp), max(timestamp)
|
|
34
|
+
FROM plausible_events_db.events_v2 GROUP BY site_id"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Extract, inspect, load:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
pip install 'plausible2umami[all] @ git+https://github.com/einsz/plausible2umami'
|
|
41
|
+
|
|
42
|
+
plausible2umami extract --site-id 1 --out events.jsonl
|
|
43
|
+
plausible2umami load --in events.jsonl --website-id <uuid> --dry-run
|
|
44
|
+
plausible2umami load --in events.jsonl --website-id <uuid> \
|
|
45
|
+
--dsn postgres://umami:pass@localhost:5432/umami
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
If ClickHouse isn't reachable from where you're running this, which is common,
|
|
49
|
+
skip `extract` and produce the same file from inside the container:
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
plausible2umami query --site-id 1 # prints the SQL
|
|
53
|
+
|
|
54
|
+
docker exec <clickhouse-container> clickhouse-client \
|
|
55
|
+
-q "<that SQL> FORMAT JSONEachRow" | gzip > events.jsonl.gz
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The output format is identical either way, and `load` reads `.gz` directly.
|
|
59
|
+
Keep the query's `ORDER BY` if you write it out by hand: it groups each
|
|
60
|
+
visitor's events together and puts identical events next to each other, which
|
|
61
|
+
is what lets the loader stream a multi-gigabyte file in flat memory.
|
|
62
|
+
|
|
63
|
+
### Large sites: extract one day at a time
|
|
64
|
+
|
|
65
|
+
Do not run the single-query extract against a site with millions of events on a
|
|
66
|
+
small server. The `ORDER BY` makes ClickHouse sort the whole result, and
|
|
67
|
+
Plausible's own `low-resources.xml` caps threads while leaving sort memory
|
|
68
|
+
unbounded, with `max_bytes_before_external_sort` at its default of 0, meaning
|
|
69
|
+
it never spills to disk. On a 4 GB host with Plausible already resident, a
|
|
70
|
+
13.5 million row sort takes the machine into swap.
|
|
71
|
+
|
|
72
|
+
```sh
|
|
73
|
+
plausible2umami query --site-id 1 --by-day --out /tmp/site1 > extract.sh
|
|
74
|
+
sh extract.sh
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
That fetches a day per query, so no sort exceeds one day's rows, and it reads
|
|
78
|
+
only the relevant parts because `site_id` and the date lead the table's primary
|
|
79
|
+
key. One file per day means an interrupted run resumes rather than restarting,
|
|
80
|
+
and the parts are concatenated at the end without recompression, since joined
|
|
81
|
+
gzip members are valid gzip.
|
|
82
|
+
|
|
83
|
+
This is safe because Plausible's `user_id` is a hash whose salt rotates daily,
|
|
84
|
+
so every row for a given visitor falls inside one day. Chunking by day
|
|
85
|
+
preserves exactly the grouping the loader depends on.
|
|
86
|
+
|
|
87
|
+
### Scale, and what it costs in disk
|
|
88
|
+
|
|
89
|
+
Both phases stream, so memory stays flat regardless of size. Disk does not.
|
|
90
|
+
|
|
91
|
+
Measured on a real migration of 9,753,130 events spanning two and a half years,
|
|
92
|
+
loaded into a fresh Umami v3.3.1:
|
|
93
|
+
|
|
94
|
+
| | |
|
|
95
|
+
|---|---|
|
|
96
|
+
| Extract, gzipped JSONL | 127 MB |
|
|
97
|
+
| Load time | 33 minutes |
|
|
98
|
+
| Postgres afterwards | **9,179 MB** |
|
|
99
|
+
| of which row data | 1,258 MB |
|
|
100
|
+
| of which indexes | **7,539 MB** |
|
|
101
|
+
| The same events in ClickHouse | **127 MiB** |
|
|
102
|
+
|
|
103
|
+
Umami puts fifteen indexes on `website_event`, and they cost six times the data
|
|
104
|
+
they index. Your events are the cheap part.
|
|
105
|
+
|
|
106
|
+
Four of them index columns Plausible has no equivalent for, so they sit empty
|
|
107
|
+
after an import: `page_title`, `tag`, `url_query`, and `event_name` unless you
|
|
108
|
+
keep engagement events. Roughly 2 GB of index over nothing.
|
|
109
|
+
|
|
110
|
+
Decide how much history you want before you start. `--since` filters at load
|
|
111
|
+
time from an extract you already have, so changing your mind costs a flag
|
|
112
|
+
rather than another trip to ClickHouse.
|
|
113
|
+
|
|
114
|
+
## What actually maps
|
|
115
|
+
|
|
116
|
+
The identity mapping is the part worth understanding, and it is easy to get
|
|
117
|
+
backwards:
|
|
118
|
+
|
|
119
|
+
| Plausible | Umami | Meaning |
|
|
120
|
+
|---|---|---|
|
|
121
|
+
| `user_id` | `session.session_id` | the visitor |
|
|
122
|
+
| `session_id` | `website_event.visit_id` | the 30-minute visit |
|
|
123
|
+
|
|
124
|
+
Plausible's `user_id` is a salted hash that rotates daily, so an imported Umami
|
|
125
|
+
session is really a visitor-day. That is a property of Plausible's data model,
|
|
126
|
+
not something this tool discards.
|
|
127
|
+
|
|
128
|
+
### Copied directly
|
|
129
|
+
|
|
130
|
+
| Plausible `events_v2` | Umami |
|
|
131
|
+
|---|---|
|
|
132
|
+
| `timestamp` | `created_at` |
|
|
133
|
+
| `name` | `event_type` (1 pageview, 2 custom) and `event_name` |
|
|
134
|
+
| `pathname` | `url_path`, `url_query` |
|
|
135
|
+
| `hostname` | `hostname` |
|
|
136
|
+
| `referrer` | `referrer_domain`, `referrer_path`, `referrer_query` |
|
|
137
|
+
| `utm_source` `utm_medium` `utm_campaign` `utm_content` `utm_term` | same names |
|
|
138
|
+
| `browser` | `session.browser` |
|
|
139
|
+
| `operating_system` | `session.os` |
|
|
140
|
+
| `screen_size` | `session.device` |
|
|
141
|
+
| `country_code` | `session.country` |
|
|
142
|
+
| `subdivision1_code` | `session.region` |
|
|
143
|
+
| `city_name` | `session.city` |
|
|
144
|
+
|
|
145
|
+
Plausible's `screen_size` is a bucket ("Desktop", "Mobile"), which is Umami's
|
|
146
|
+
`device`. Umami's `screen` wants "1920x1080" and has no source here.
|
|
147
|
+
|
|
148
|
+
### Left null, because Plausible never collected it
|
|
149
|
+
|
|
150
|
+
`session.language`, `session.screen`, `session.distinct_id`, `page_title`, and
|
|
151
|
+
the web-vitals columns (`lcp`, `inp`, `cls`, `fcp`, `ttfb`).
|
|
152
|
+
|
|
153
|
+
City is mapped. The extract selects `city_name`, an ALIAS column that resolves `city_geoname_id` against Plausible's bundled
|
|
154
|
+
`location_data_dict`, so no GeoNames table has to be shipped.
|
|
155
|
+
|
|
156
|
+
Whether you get anything depends on your GeoIP database. Plausible only records
|
|
157
|
+
`city_geoname_id` when it has a **city-level** one; with country-level data the
|
|
158
|
+
id is 0 for every event and `session.city` ends up null.
|
|
159
|
+
On the instance this was developed against, all 5,102 test events had id 0, so
|
|
160
|
+
the mapping is verified only as far as the dictionary: `dictGet` resolves ids
|
|
161
|
+
to names correctly ("Berlin", "New York City"), but no live event
|
|
162
|
+
carrying a real id was ever migrated.
|
|
163
|
+
|
|
164
|
+
If your Plausible predates the `city_name` alias, remove it from the extract
|
|
165
|
+
query. The loader treats a missing `city_name` as null rather than failing.
|
|
166
|
+
|
|
167
|
+
### Dropped, no Umami equivalent
|
|
168
|
+
|
|
169
|
+
`scroll_depth`, `engagement_time`, the `revenue_*` fields, `click_id_param`,
|
|
170
|
+
`acquisition_channel`, and `browser_version` / `operating_system_version`.
|
|
171
|
+
|
|
172
|
+
### Filtered out by default
|
|
173
|
+
|
|
174
|
+
Plausible emits an internal `engagement` event alongside pageviews, carrying
|
|
175
|
+
`scroll_depth` and `engagement_time`. Umami has nowhere to put either, so
|
|
176
|
+
importing them adds rows and no information while filling Umami's custom-events
|
|
177
|
+
list with a single meaningless name.
|
|
178
|
+
|
|
179
|
+
They are dropped unless you pass `--include-engagement`. The volume is worth
|
|
180
|
+
knowing: on the site this was first tested against, 3,557 of 5,094 events were
|
|
181
|
+
engagement. Filtering them left pageviews, visitors and visits identical and
|
|
182
|
+
cut the row count by 70%.
|
|
183
|
+
|
|
184
|
+
### Not implemented
|
|
185
|
+
|
|
186
|
+
- **Custom event properties.** `meta.key` / `meta.value` would map to Umami's
|
|
187
|
+
`event_data` table. Custom event *names* are imported; their properties are
|
|
188
|
+
not.
|
|
189
|
+
|
|
190
|
+
Nothing is fabricated. Every field outside the tables above lands as null.
|
|
191
|
+
|
|
192
|
+
## Safety
|
|
193
|
+
|
|
194
|
+
Re-running is safe. Every id is a UUIDv5 derived from the source data and the
|
|
195
|
+
target website id, and every insert is `ON CONFLICT DO NOTHING`, so a load that
|
|
196
|
+
dies partway through can just be run again. Rows already written are left
|
|
197
|
+
alone.
|
|
198
|
+
|
|
199
|
+
- `--dry-run` transforms everything and writes nothing, printing counts, the
|
|
200
|
+
date range and a sample of what would go in
|
|
201
|
+
- `--limit N` loads a slice, to see how it looks before committing
|
|
202
|
+
- Loading into a website that already has events requires `--force`, so you
|
|
203
|
+
don't accidentally mix imported history into live traffic
|
|
204
|
+
- Values are truncated to Umami's column widths rather than failing a batch
|
|
205
|
+
two thirds of the way through
|
|
206
|
+
|
|
207
|
+
Take a database dump first anyway. This writes directly to Umami's tables.
|
|
208
|
+
|
|
209
|
+
## Development
|
|
210
|
+
|
|
211
|
+
```sh
|
|
212
|
+
python -m venv .venv && .venv/bin/pip install -e '.[dev,all]'
|
|
213
|
+
.venv/bin/python -m pytest
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
The transform is pure functions over dicts, so the tests need no database.
|
|
217
|
+
|
|
218
|
+
## Compatibility
|
|
219
|
+
|
|
220
|
+
| | Verified against |
|
|
221
|
+
|---|---|
|
|
222
|
+
| Plausible CE | v3.1.0 and v3.2.1, ClickHouse 24.12 |
|
|
223
|
+
| Umami | v3.3.1, PostgreSQL 16 |
|
|
224
|
+
|
|
225
|
+
Plausible 3.1.0 to 3.2.1 changes nothing here: `events_v2` is identical in
|
|
226
|
+
column names and types, and extracting the same 5,094 events under both
|
|
227
|
+
versions and loading each produced the same sessions, the same visits and the
|
|
228
|
+
same per-path pageview counts, with no differences at all.
|
|
229
|
+
|
|
230
|
+
Correctness was checked by loading 5,094 real events from a live Plausible
|
|
231
|
+
instance into a fresh Umami v3.3.1, then comparing Umami's own dashboard
|
|
232
|
+
aggregation against the raw ClickHouse rows:
|
|
233
|
+
|
|
234
|
+
| | Plausible raw | Umami dashboard |
|
|
235
|
+
|---|---|---|
|
|
236
|
+
| pageviews | 1537 | 1537 |
|
|
237
|
+
| visitors (distinct `user_id`) | 883 | 883 |
|
|
238
|
+
| visits (distinct `session_id`) | 1089 | 1089 |
|
|
239
|
+
| top page | `/` 1522 | `/` 1522 |
|
|
240
|
+
|
|
241
|
+
Zero events referenced a missing session, and a full second run of the same
|
|
242
|
+
file wrote no new rows.
|
|
243
|
+
|
|
244
|
+
Both projects change their schemas. If yours differ, the mapping tables above
|
|
245
|
+
tell you what to check.
|
|
246
|
+
|
|
247
|
+
## Licence
|
|
248
|
+
|
|
249
|
+
MIT.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "plausible2umami"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Migrate self-hosted Plausible analytics into Umami v3, event by event"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "einsz" }]
|
|
13
|
+
keywords = ["plausible", "umami", "analytics", "migration", "clickhouse"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: System Administrators",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Internet :: Log Analysis",
|
|
20
|
+
]
|
|
21
|
+
dependencies = []
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/einsz/plausible2umami"
|
|
25
|
+
Issues = "https://github.com/einsz/plausible2umami/issues"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
extract = ["clickhouse-connect>=0.7"]
|
|
29
|
+
load = ["psycopg[binary]>=3.1"]
|
|
30
|
+
all = ["clickhouse-connect>=0.7", "psycopg[binary]>=3.1"]
|
|
31
|
+
dev = ["pytest>=8"]
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
plausible2umami = "plausible2umami.cli:main"
|
|
35
|
+
|
|
36
|
+
[tool.hatch.build.targets.wheel]
|
|
37
|
+
packages = ["src/plausible2umami"]
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|