tg-chat-dump 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tg_chat_dump-1.1.0/.gitignore +7 -0
- tg_chat_dump-1.1.0/LICENSE +22 -0
- tg_chat_dump-1.1.0/PKG-INFO +229 -0
- tg_chat_dump-1.1.0/README.md +205 -0
- tg_chat_dump-1.1.0/dump.py +6 -0
- tg_chat_dump-1.1.0/pyproject.toml +59 -0
- tg_chat_dump-1.1.0/tests/test_dump.py +330 -0
- tg_chat_dump-1.1.0/tgdump/__init__.py +1 -0
- tg_chat_dump-1.1.0/tgdump/accounts.py +64 -0
- tg_chat_dump-1.1.0/tgdump/cli.py +111 -0
- tg_chat_dump-1.1.0/tgdump/config.py +92 -0
- tg_chat_dump-1.1.0/tgdump/export.py +101 -0
- tg_chat_dump-1.1.0/tgdump/fetch.py +243 -0
- tg_chat_dump-1.1.0/tgdump/interactive.py +454 -0
- tg_chat_dump-1.1.0/tgdump/scope.py +78 -0
- tg_chat_dump-1.1.0/tgdump/store.py +201 -0
- tg_chat_dump-1.1.0/tgdump/util.py +26 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 renkagod
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tg-chat-dump
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Dump a whole Telegram chat or forum at 100k+ messages a minute into SQLite, JSONL and one folder per topic
|
|
5
|
+
Project-URL: Homepage, https://github.com/renkagod/tg-chat-dump
|
|
6
|
+
Project-URL: Issues, https://github.com/renkagod/tg-chat-dump/issues
|
|
7
|
+
Author: renkagod
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: archive,backup,dump,export,forum,takeout,telegram,telethon
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Communications :: Chat
|
|
17
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: prompt-toolkit>=3.0.53
|
|
20
|
+
Requires-Dist: python-dotenv>=1.0
|
|
21
|
+
Requires-Dist: python-socks[asyncio]>=2.4
|
|
22
|
+
Requires-Dist: telethon>=1.40
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# tg-chat-dump
|
|
26
|
+
|
|
27
|
+
[](pyproject.toml)
|
|
28
|
+
[](https://github.com/LonamiWebs/Telethon)
|
|
29
|
+
[](LICENSE)
|
|
30
|
+
|
|
31
|
+
<p align="center"><img src="assets/speed.svg" width="100%" alt="Over 100,000 messages per minute: ~115,000 with tg-chat-dump, ~29,000 with Telegram Desktop's export, ~6,000 through the regular API"></p>
|
|
32
|
+
|
|
33
|
+
<p align="center"><img src="assets/demo.gif" width="100%" alt="Interactive mode: find a chat with live suggestions, then dump 420,912 messages in 3m 48s"><br><sub>A 420,912-message group dumped in 3m 48s with two accounts (the download part is sped up 12×; account names are blurred).</sub></p>
|
|
34
|
+
|
|
35
|
+
Dump an entire Telegram chat into SQLite and a folder per forum topic at **over 100,000 messages a minute**. It runs Telegram's own data-export mode with parallel workers on several accounts, so a chat of several hundred thousand messages is done in minutes: about 4× faster than Telegram Desktop's own export, and a whole forum in one pass instead of topic by topic.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
out/1234567890_My_Chat/
|
|
39
|
+
├── 1_General/
|
|
40
|
+
│ ├── messages.jsonl
|
|
41
|
+
│ └── messages.txt
|
|
42
|
+
├── 12_Off-topic/
|
|
43
|
+
│ ├── messages.jsonl
|
|
44
|
+
│ └── messages.txt
|
|
45
|
+
└── 37_Announcements/
|
|
46
|
+
├── messages.jsonl
|
|
47
|
+
└── messages.txt
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Features
|
|
51
|
+
|
|
52
|
+
- Over 100,000 messages a minute with Telegram's export mode and two accounts: about 4× Telegram Desktop's export and 10× the regular API (see [Performance](#performance)).
|
|
53
|
+
- Run it without arguments for the interactive mode: add or remove accounts, find a chat as you type, watch a progress bar with ETA.
|
|
54
|
+
- A whole forum in one pass. Messages are sorted into topics as they arrive.
|
|
55
|
+
- The chat is split into message-id ranges that several workers download at once. Telegram limits each account separately, so every extra account adds about as much speed as the first.
|
|
56
|
+
- Progress is committed to SQLite every 100 messages. An interrupted run resumes where it stopped, and later runs fetch only new messages.
|
|
57
|
+
- Optional extras, each a toggle: reactions and views, poll results, formatting with hidden links, comments under channel posts, and Telegram's export mode (see [Extras](#extras)).
|
|
58
|
+
- Filters by date range, author or message type, such as links or documents.
|
|
59
|
+
- On `FLOOD_WAIT` the run sleeps and then continues on its own.
|
|
60
|
+
- SOCKS5 and HTTP proxies, for networks where Telegram is blocked.
|
|
61
|
+
- `messages.txt` for reading, `messages.jsonl` for scripts, and the SQLite database for queries.
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
1. Create an app at [my.telegram.org](https://my.telegram.org) → *API development tools* and note its `api_id` and `api_hash`.
|
|
66
|
+
2. Clone and install (with [uv](https://docs.astral.sh/uv/)):
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
git clone https://github.com/renkagod/tg-chat-dump.git
|
|
70
|
+
cd tg-chat-dump
|
|
71
|
+
uv sync
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Without uv: `pip install telethon python-dotenv "python-socks[asyncio]" prompt-toolkit`.
|
|
75
|
+
|
|
76
|
+
3. Run it:
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
uv run dump.py
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The first start asks for `api_id` and `api_hash` and saves them to `.env`. After that it works like this:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
Accounts:
|
|
86
|
+
1) Alice @alice
|
|
87
|
+
2) Bob
|
|
88
|
+
Output folder: D:\Telegram dumps
|
|
89
|
+
Extras: meta, polls
|
|
90
|
+
[a] add account [d N] remove [f] output folder [x] extras [Enter] continue >
|
|
91
|
+
Loading your chats… 312 found.
|
|
92
|
+
|
|
93
|
+
Search chat (name, @username or id; Tab completes, Enter lists all): forum
|
|
94
|
+
1) Forum Club @forumclub [forum] 2/2 accounts
|
|
95
|
+
2) Forum News @forumnews [channel] 1/2 accounts
|
|
96
|
+
Number, [p] search public chats, or Enter to search again > 1
|
|
97
|
+
|
|
98
|
+
Forum Club [forum] id -1001234567890
|
|
99
|
+
~152,310 messages
|
|
100
|
+
Dump it with 2 accounts, extras: meta, polls? [Y/n, f = filters]
|
|
101
|
+
[████████░░░░░░░░░░░░] 41% 62,104 saved +62,104 11,480/min ETA 7m 40s
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
- Matching chats pop up under the cursor as you type; Tab fills in the highlighted one and Enter picks it.
|
|
105
|
+
- If nothing in your chats matches the search, public groups and channels are searched too. Public chats can be dumped without joining.
|
|
106
|
+
- Every account that can see the chat is used.
|
|
107
|
+
- `x` toggles the extras, `f` sets the output folder. Both are saved. By default the output goes to `out/` next to `dump.py`.
|
|
108
|
+
- Answering `f` instead of `Y` asks for filters for this one dump.
|
|
109
|
+
- **Ctrl+C** stops the dump; the next run resumes it.
|
|
110
|
+
- If Telegram is blocked in your network, set `TG_PROXY` in `.env`, for example `TG_PROXY=socks5://127.0.0.1:1080`.
|
|
111
|
+
|
|
112
|
+
## Extras
|
|
113
|
+
|
|
114
|
+
All of them are off by default. Switch them on with `x` in the interactive mode, `--with` on the command line, or `TG_OPTIONS` in `.env`.
|
|
115
|
+
|
|
116
|
+
| Extra | Adds | Cost |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| `meta` | `reactions` (`{"👍": 12}`), `views`, `forwards`, `replies` | none, it is already in every message |
|
|
119
|
+
| `polls` | poll question, answers, votes; checklist items and which are done | none |
|
|
120
|
+
| `markdown` | `text_md`: the text as Markdown, with bold, links behind words, mentions | none |
|
|
121
|
+
| `comments` | for a channel: the comments under its posts, from the linked discussion group, in `comments/`, grouped by post | a second dump of the discussion group |
|
|
122
|
+
| `takeout` | runs the dump in Telegram's data-export mode, the one Telegram Desktop uses; it skips the usual rate limits, about 10× faster (see [Performance](#performance)) | the first time, Telegram asks you to allow the export from a phone logged in to that account, otherwise after 24 hours; until then the dump runs at normal speed |
|
|
123
|
+
|
|
124
|
+
Turning an extra on later does not touch messages that are already saved. A new run adds it to new messages only.
|
|
125
|
+
|
|
126
|
+
## Filters
|
|
127
|
+
|
|
128
|
+
```sh
|
|
129
|
+
uv run dump.py --chat @somegroup --since 2026-01-01 --until 2026-03-31 # a date range, inclusive
|
|
130
|
+
uv run dump.py --chat @somegroup --from @alice # one author
|
|
131
|
+
uv run dump.py --chat @somegroup --type links # links, docs, photos, videos, voice, pinned, ...
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
A filtered dump gets its own database and folder, for example `out/1234567890_My_Chat_since-2026-01-01_links/`, so it never mixes with the full dump. Filters can be combined. They work with whole-chat dumps, not with `--topics`.
|
|
135
|
+
|
|
136
|
+
## Command line
|
|
137
|
+
|
|
138
|
+
With arguments it runs without questions, for scripts and cron. Settings come from `.env` (see `.env.example`):
|
|
139
|
+
|
|
140
|
+
```sh
|
|
141
|
+
uv run dump.py --chat @somegroup # whole chat (or only new messages on repeat runs)
|
|
142
|
+
uv run dump.py --chat @somegroup --topics 12,37 # only these forum topics
|
|
143
|
+
uv run dump.py --chat @somegroup --with meta,polls # with extras
|
|
144
|
+
uv run dump.py --chat @somegroup --export-only # rebuild the folders from the database, no network
|
|
145
|
+
uv run dump.py --login acc2 # add another account (see below)
|
|
146
|
+
uv run dump.py --chat @somegroup --workers 2 # workers per account, default 3
|
|
147
|
+
uv run dump.py --chat @somegroup --out "D:\Telegram dumps" # output folder, remembered in .env
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### More accounts, more speed
|
|
151
|
+
|
|
152
|
+
Every extra account that is a member of the chat adds throughput. Add one with `a` in the interactive mode, or:
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
uv run dump.py --login acc2 # log in once, saved to data/acc2.session
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Each `data/*.session` file is picked up automatically. Accounts that are not logged in or are not members of the chat are skipped with a warning.
|
|
159
|
+
|
|
160
|
+
### Performance
|
|
161
|
+
|
|
162
|
+
Measured on a large forum supergroup (several hundred thousand messages):
|
|
163
|
+
|
|
164
|
+
| Accounts | Speed | Full dump |
|
|
165
|
+
|---|---|---|
|
|
166
|
+
| 1 | ~6,000–7,000 messages/min | ~1h 45m |
|
|
167
|
+
| 2 | ~11,500 messages/min | ~1h |
|
|
168
|
+
| 2, with `takeout` | ~115,000 messages/min | ~6m |
|
|
169
|
+
| Telegram Desktop export, for comparison | ~29,000 messages/min | ~25m (extrapolated) |
|
|
170
|
+
|
|
171
|
+
Telegram Desktop was measured on another group, text only, against tg-chat-dump with `takeout` on the same group: ~29,000 vs ~110,000 messages/min.
|
|
172
|
+
|
|
173
|
+
Telegram's per-account rate limit sets the ceiling: about 6,000 messages/min without takeout and about 60,000 with it. More than 3 workers per account does not help (3, 6 and 12 measured the same), and with many more Telegram adds flood waits. Turning on `takeout` gives the biggest jump; after that, more speed comes only from more accounts.
|
|
174
|
+
|
|
175
|
+
## Output
|
|
176
|
+
|
|
177
|
+
Each topic folder contains:
|
|
178
|
+
|
|
179
|
+
**`messages.txt`**: one line per message:
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
[2026-03-14 09:01:12] #1042 Alice: hi everyone [👍 3 · 120 views]
|
|
183
|
+
[2026-03-14 09:02:40] #1043 Bob (reply to #1042): <MessageMediaPhoto> look at this
|
|
184
|
+
[2026-03-14 09:05:00] #1044 Carol: <poll> Lunch? | Pizza (4) | Sushi (2) | 6 votes
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
**`messages.jsonl`**: one JSON object per message:
|
|
188
|
+
|
|
189
|
+
| Field | Meaning |
|
|
190
|
+
|---|---|
|
|
191
|
+
| `id` | message id |
|
|
192
|
+
| `date`, `edit_date` | ISO 8601, UTC |
|
|
193
|
+
| `topic_id` | forum topic (`1` = General, `null` in non-forum chats) |
|
|
194
|
+
| `sender_id`, `sender` | author id and display name |
|
|
195
|
+
| `text` | message text |
|
|
196
|
+
| `reply_to`, `reply_top` | replied-to message and thread root |
|
|
197
|
+
| `fwd_from` | original author of a forwarded message |
|
|
198
|
+
| `media` | media type (`MessageMediaPhoto`, `MessageMediaDocument`, ...) |
|
|
199
|
+
| `action` | service message type (`MessageActionTopicCreate`, ...) |
|
|
200
|
+
| `grouped_id` | album id |
|
|
201
|
+
| `text_md`, `views`, `forwards`, `replies`, `reactions`, `extra` | only with the matching [extras](#extras) |
|
|
202
|
+
|
|
203
|
+
The raw data is in `data/<chat id>.sqlite`, with tables `messages`, `topics` and `tasks` (download progress).
|
|
204
|
+
|
|
205
|
+
## Limitations
|
|
206
|
+
|
|
207
|
+
- Downloads text and metadata only. Media files are recorded by type and not downloaded.
|
|
208
|
+
- The account only sees what Telegram shows it: if the chat hides history from new members, older messages are not available.
|
|
209
|
+
- `--topics` cannot fetch the General topic on its own; the full-chat mode covers it.
|
|
210
|
+
|
|
211
|
+
## Responsible use
|
|
212
|
+
|
|
213
|
+
Using your own account through the API falls under Telegram's [API Terms of Service](https://core.telegram.org/api/terms). Dumps contain other people's messages, so keep them private and comply with your local data-protection laws. `data/*.session` files are full logins to your accounts: never share or commit them.
|
|
214
|
+
|
|
215
|
+
## Development
|
|
216
|
+
|
|
217
|
+
```sh
|
|
218
|
+
uv sync
|
|
219
|
+
uv run pytest
|
|
220
|
+
uv run ruff check . && uv run ruff format --check .
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
The code is in `tgdump/`: `fetch` downloads, `store` turns messages into database rows, `export` writes the folders, `cli` and `interactive` are the two front ends.
|
|
224
|
+
|
|
225
|
+
Issues and pull requests are welcome. Please run the tests and linters before opening a PR.
|
|
226
|
+
|
|
227
|
+
## License
|
|
228
|
+
|
|
229
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# tg-chat-dump
|
|
2
|
+
|
|
3
|
+
[](pyproject.toml)
|
|
4
|
+
[](https://github.com/LonamiWebs/Telethon)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
<p align="center"><img src="assets/speed.svg" width="100%" alt="Over 100,000 messages per minute: ~115,000 with tg-chat-dump, ~29,000 with Telegram Desktop's export, ~6,000 through the regular API"></p>
|
|
8
|
+
|
|
9
|
+
<p align="center"><img src="assets/demo.gif" width="100%" alt="Interactive mode: find a chat with live suggestions, then dump 420,912 messages in 3m 48s"><br><sub>A 420,912-message group dumped in 3m 48s with two accounts (the download part is sped up 12×; account names are blurred).</sub></p>
|
|
10
|
+
|
|
11
|
+
Dump an entire Telegram chat into SQLite and a folder per forum topic at **over 100,000 messages a minute**. It runs Telegram's own data-export mode with parallel workers on several accounts, so a chat of several hundred thousand messages is done in minutes: about 4× faster than Telegram Desktop's own export, and a whole forum in one pass instead of topic by topic.
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
out/1234567890_My_Chat/
|
|
15
|
+
├── 1_General/
|
|
16
|
+
│ ├── messages.jsonl
|
|
17
|
+
│ └── messages.txt
|
|
18
|
+
├── 12_Off-topic/
|
|
19
|
+
│ ├── messages.jsonl
|
|
20
|
+
│ └── messages.txt
|
|
21
|
+
└── 37_Announcements/
|
|
22
|
+
├── messages.jsonl
|
|
23
|
+
└── messages.txt
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Features
|
|
27
|
+
|
|
28
|
+
- Over 100,000 messages a minute with Telegram's export mode and two accounts: about 4× Telegram Desktop's export and 10× the regular API (see [Performance](#performance)).
|
|
29
|
+
- Run it without arguments for the interactive mode: add or remove accounts, find a chat as you type, watch a progress bar with ETA.
|
|
30
|
+
- A whole forum in one pass. Messages are sorted into topics as they arrive.
|
|
31
|
+
- The chat is split into message-id ranges that several workers download at once. Telegram limits each account separately, so every extra account adds about as much speed as the first.
|
|
32
|
+
- Progress is committed to SQLite every 100 messages. An interrupted run resumes where it stopped, and later runs fetch only new messages.
|
|
33
|
+
- Optional extras, each a toggle: reactions and views, poll results, formatting with hidden links, comments under channel posts, and Telegram's export mode (see [Extras](#extras)).
|
|
34
|
+
- Filters by date range, author or message type, such as links or documents.
|
|
35
|
+
- On `FLOOD_WAIT` the run sleeps and then continues on its own.
|
|
36
|
+
- SOCKS5 and HTTP proxies, for networks where Telegram is blocked.
|
|
37
|
+
- `messages.txt` for reading, `messages.jsonl` for scripts, and the SQLite database for queries.
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
1. Create an app at [my.telegram.org](https://my.telegram.org) → *API development tools* and note its `api_id` and `api_hash`.
|
|
42
|
+
2. Clone and install (with [uv](https://docs.astral.sh/uv/)):
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
git clone https://github.com/renkagod/tg-chat-dump.git
|
|
46
|
+
cd tg-chat-dump
|
|
47
|
+
uv sync
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Without uv: `pip install telethon python-dotenv "python-socks[asyncio]" prompt-toolkit`.
|
|
51
|
+
|
|
52
|
+
3. Run it:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
uv run dump.py
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The first start asks for `api_id` and `api_hash` and saves them to `.env`. After that it works like this:
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
Accounts:
|
|
62
|
+
1) Alice @alice
|
|
63
|
+
2) Bob
|
|
64
|
+
Output folder: D:\Telegram dumps
|
|
65
|
+
Extras: meta, polls
|
|
66
|
+
[a] add account [d N] remove [f] output folder [x] extras [Enter] continue >
|
|
67
|
+
Loading your chats… 312 found.
|
|
68
|
+
|
|
69
|
+
Search chat (name, @username or id; Tab completes, Enter lists all): forum
|
|
70
|
+
1) Forum Club @forumclub [forum] 2/2 accounts
|
|
71
|
+
2) Forum News @forumnews [channel] 1/2 accounts
|
|
72
|
+
Number, [p] search public chats, or Enter to search again > 1
|
|
73
|
+
|
|
74
|
+
Forum Club [forum] id -1001234567890
|
|
75
|
+
~152,310 messages
|
|
76
|
+
Dump it with 2 accounts, extras: meta, polls? [Y/n, f = filters]
|
|
77
|
+
[████████░░░░░░░░░░░░] 41% 62,104 saved +62,104 11,480/min ETA 7m 40s
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
- Matching chats pop up under the cursor as you type; Tab fills in the highlighted one and Enter picks it.
|
|
81
|
+
- If nothing in your chats matches the search, public groups and channels are searched too. Public chats can be dumped without joining.
|
|
82
|
+
- Every account that can see the chat is used.
|
|
83
|
+
- `x` toggles the extras, `f` sets the output folder. Both are saved. By default the output goes to `out/` next to `dump.py`.
|
|
84
|
+
- Answering `f` instead of `Y` asks for filters for this one dump.
|
|
85
|
+
- **Ctrl+C** stops the dump; the next run resumes it.
|
|
86
|
+
- If Telegram is blocked in your network, set `TG_PROXY` in `.env`, for example `TG_PROXY=socks5://127.0.0.1:1080`.
|
|
87
|
+
|
|
88
|
+
## Extras
|
|
89
|
+
|
|
90
|
+
All of them are off by default. Switch them on with `x` in the interactive mode, `--with` on the command line, or `TG_OPTIONS` in `.env`.
|
|
91
|
+
|
|
92
|
+
| Extra | Adds | Cost |
|
|
93
|
+
|---|---|---|
|
|
94
|
+
| `meta` | `reactions` (`{"👍": 12}`), `views`, `forwards`, `replies` | none, it is already in every message |
|
|
95
|
+
| `polls` | poll question, answers, votes; checklist items and which are done | none |
|
|
96
|
+
| `markdown` | `text_md`: the text as Markdown, with bold, links behind words, mentions | none |
|
|
97
|
+
| `comments` | for a channel: the comments under its posts, from the linked discussion group, in `comments/`, grouped by post | a second dump of the discussion group |
|
|
98
|
+
| `takeout` | runs the dump in Telegram's data-export mode, the one Telegram Desktop uses; it skips the usual rate limits, about 10× faster (see [Performance](#performance)) | the first time, Telegram asks you to allow the export from a phone logged in to that account, otherwise after 24 hours; until then the dump runs at normal speed |
|
|
99
|
+
|
|
100
|
+
Turning an extra on later does not touch messages that are already saved. A new run adds it to new messages only.
|
|
101
|
+
|
|
102
|
+
## Filters
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
uv run dump.py --chat @somegroup --since 2026-01-01 --until 2026-03-31 # a date range, inclusive
|
|
106
|
+
uv run dump.py --chat @somegroup --from @alice # one author
|
|
107
|
+
uv run dump.py --chat @somegroup --type links # links, docs, photos, videos, voice, pinned, ...
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A filtered dump gets its own database and folder, for example `out/1234567890_My_Chat_since-2026-01-01_links/`, so it never mixes with the full dump. Filters can be combined. They work with whole-chat dumps, not with `--topics`.
|
|
111
|
+
|
|
112
|
+
## Command line
|
|
113
|
+
|
|
114
|
+
With arguments it runs without questions, for scripts and cron. Settings come from `.env` (see `.env.example`):
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
uv run dump.py --chat @somegroup # whole chat (or only new messages on repeat runs)
|
|
118
|
+
uv run dump.py --chat @somegroup --topics 12,37 # only these forum topics
|
|
119
|
+
uv run dump.py --chat @somegroup --with meta,polls # with extras
|
|
120
|
+
uv run dump.py --chat @somegroup --export-only # rebuild the folders from the database, no network
|
|
121
|
+
uv run dump.py --login acc2 # add another account (see below)
|
|
122
|
+
uv run dump.py --chat @somegroup --workers 2 # workers per account, default 3
|
|
123
|
+
uv run dump.py --chat @somegroup --out "D:\Telegram dumps" # output folder, remembered in .env
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### More accounts, more speed
|
|
127
|
+
|
|
128
|
+
Every extra account that is a member of the chat adds throughput. Add one with `a` in the interactive mode, or:
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
uv run dump.py --login acc2 # log in once, saved to data/acc2.session
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Each `data/*.session` file is picked up automatically. Accounts that are not logged in or are not members of the chat are skipped with a warning.
|
|
135
|
+
|
|
136
|
+
### Performance
|
|
137
|
+
|
|
138
|
+
Measured on a large forum supergroup (several hundred thousand messages):
|
|
139
|
+
|
|
140
|
+
| Accounts | Speed | Full dump |
|
|
141
|
+
|---|---|---|
|
|
142
|
+
| 1 | ~6,000–7,000 messages/min | ~1h 45m |
|
|
143
|
+
| 2 | ~11,500 messages/min | ~1h |
|
|
144
|
+
| 2, with `takeout` | ~115,000 messages/min | ~6m |
|
|
145
|
+
| Telegram Desktop export, for comparison | ~29,000 messages/min | ~25m (extrapolated) |
|
|
146
|
+
|
|
147
|
+
Telegram Desktop was measured on another group, text only, against tg-chat-dump with `takeout` on the same group: ~29,000 vs ~110,000 messages/min.
|
|
148
|
+
|
|
149
|
+
Telegram's per-account rate limit sets the ceiling: about 6,000 messages/min without takeout and about 60,000 with it. More than 3 workers per account does not help (3, 6 and 12 measured the same), and with many more Telegram adds flood waits. Turning on `takeout` gives the biggest jump; after that, more speed comes only from more accounts.
|
|
150
|
+
|
|
151
|
+
## Output
|
|
152
|
+
|
|
153
|
+
Each topic folder contains:
|
|
154
|
+
|
|
155
|
+
**`messages.txt`**: one line per message:
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
[2026-03-14 09:01:12] #1042 Alice: hi everyone [👍 3 · 120 views]
|
|
159
|
+
[2026-03-14 09:02:40] #1043 Bob (reply to #1042): <MessageMediaPhoto> look at this
|
|
160
|
+
[2026-03-14 09:05:00] #1044 Carol: <poll> Lunch? | Pizza (4) | Sushi (2) | 6 votes
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
**`messages.jsonl`**: one JSON object per message:
|
|
164
|
+
|
|
165
|
+
| Field | Meaning |
|
|
166
|
+
|---|---|
|
|
167
|
+
| `id` | message id |
|
|
168
|
+
| `date`, `edit_date` | ISO 8601, UTC |
|
|
169
|
+
| `topic_id` | forum topic (`1` = General, `null` in non-forum chats) |
|
|
170
|
+
| `sender_id`, `sender` | author id and display name |
|
|
171
|
+
| `text` | message text |
|
|
172
|
+
| `reply_to`, `reply_top` | replied-to message and thread root |
|
|
173
|
+
| `fwd_from` | original author of a forwarded message |
|
|
174
|
+
| `media` | media type (`MessageMediaPhoto`, `MessageMediaDocument`, ...) |
|
|
175
|
+
| `action` | service message type (`MessageActionTopicCreate`, ...) |
|
|
176
|
+
| `grouped_id` | album id |
|
|
177
|
+
| `text_md`, `views`, `forwards`, `replies`, `reactions`, `extra` | only with the matching [extras](#extras) |
|
|
178
|
+
|
|
179
|
+
The raw data is in `data/<chat id>.sqlite`, with tables `messages`, `topics` and `tasks` (download progress).
|
|
180
|
+
|
|
181
|
+
## Limitations
|
|
182
|
+
|
|
183
|
+
- Downloads text and metadata only. Media files are recorded by type and not downloaded.
|
|
184
|
+
- The account only sees what Telegram shows it: if the chat hides history from new members, older messages are not available.
|
|
185
|
+
- `--topics` cannot fetch the General topic on its own; the full-chat mode covers it.
|
|
186
|
+
|
|
187
|
+
## Responsible use
|
|
188
|
+
|
|
189
|
+
Using your own account through the API falls under Telegram's [API Terms of Service](https://core.telegram.org/api/terms). Dumps contain other people's messages, so keep them private and comply with your local data-protection laws. `data/*.session` files are full logins to your accounts: never share or commit them.
|
|
190
|
+
|
|
191
|
+
## Development
|
|
192
|
+
|
|
193
|
+
```sh
|
|
194
|
+
uv sync
|
|
195
|
+
uv run pytest
|
|
196
|
+
uv run ruff check . && uv run ruff format --check .
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
The code is in `tgdump/`: `fetch` downloads, `store` turns messages into database rows, `export` writes the folders, `cli` and `interactive` are the two front ends.
|
|
200
|
+
|
|
201
|
+
Issues and pull requests are welcome. Please run the tests and linters before opening a PR.
|
|
202
|
+
|
|
203
|
+
## License
|
|
204
|
+
|
|
205
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "tg-chat-dump"
|
|
3
|
+
version = "1.1.0"
|
|
4
|
+
description = "Dump a whole Telegram chat or forum at 100k+ messages a minute into SQLite, JSONL and one folder per topic"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
authors = [{ name = "renkagod" }]
|
|
9
|
+
keywords = ["telegram", "export", "backup", "archive", "dump", "telethon", "forum", "takeout"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 5 - Production/Stable",
|
|
12
|
+
"Environment :: Console",
|
|
13
|
+
"Intended Audience :: End Users/Desktop",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Topic :: Communications :: Chat",
|
|
17
|
+
"Topic :: System :: Archiving :: Backup",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"telethon>=1.40",
|
|
21
|
+
"python-dotenv>=1.0",
|
|
22
|
+
"python-socks[asyncio]>=2.4",
|
|
23
|
+
"prompt-toolkit>=3.0.53",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
tg-chat-dump = "tgdump.cli:run"
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/renkagod/tg-chat-dump"
|
|
31
|
+
Issues = "https://github.com/renkagod/tg-chat-dump/issues"
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["hatchling"]
|
|
35
|
+
build-backend = "hatchling.build"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["tgdump"]
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.sdist]
|
|
41
|
+
include = ["tgdump", "tests", "dump.py", "README.md", "LICENSE"]
|
|
42
|
+
|
|
43
|
+
[dependency-groups]
|
|
44
|
+
dev = [
|
|
45
|
+
"pytest>=8",
|
|
46
|
+
"ruff>=0.6",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
pythonpath = ["."]
|
|
51
|
+
testpaths = ["tests"]
|
|
52
|
+
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
line-length = 120
|
|
55
|
+
target-version = "py311"
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "W", "I", "B", "UP"]
|
|
59
|
+
ignore = ["UP031"]
|