telegram-migration 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- telegram_migration/__init__.py +30 -0
- telegram_migration/__main__.py +3 -0
- telegram_migration/_version.py +1 -0
- telegram_migration/bigfiles.py +174 -0
- telegram_migration/cli.py +90 -0
- telegram_migration/config.py +296 -0
- telegram_migration/constants.py +25 -0
- telegram_migration/exportmedia.py +151 -0
- telegram_migration/formatfallback.py +59 -0
- telegram_migration/htmlexport.py +218 -0
- telegram_migration/htmlutil.py +36 -0
- telegram_migration/mediatools.py +219 -0
- telegram_migration/pipeline.py +358 -0
- telegram_migration/progress.py +195 -0
- telegram_migration/reaction_archive.py +277 -0
- telegram_migration/replylinks.py +82 -0
- telegram_migration/richtext.py +130 -0
- telegram_migration/runner.py +33 -0
- telegram_migration/sending.py +316 -0
- telegram_migration/textutil.py +168 -0
- telegram_migration/uploads.py +171 -0
- telegram_migration-0.1.0.dist-info/METADATA +230 -0
- telegram_migration-0.1.0.dist-info/RECORD +27 -0
- telegram_migration-0.1.0.dist-info/WHEEL +5 -0
- telegram_migration-0.1.0.dist-info/entry_points.txt +2 -0
- telegram_migration-0.1.0.dist-info/licenses/LICENSE +674 -0
- telegram_migration-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Rebuild a Telegram Desktop chat export in a channel, resumably, with a Telegram bot.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
from telegram_migration import Settings, run
|
|
6
|
+
|
|
7
|
+
settings = Settings.from_env() # reads TELEGRAM_* variables / .env / Colab Secrets
|
|
8
|
+
run(settings) # dry run unless settings.dry_run is False
|
|
9
|
+
|
|
10
|
+
or from a shell: ``python -m telegram_migration --help``.
|
|
11
|
+
|
|
12
|
+
Importing this package does not read environment variables or perform file I/O.
|
|
13
|
+
Configuration and filesystem access happen when you call :func:`run` / :func:`run_async`.
|
|
14
|
+
"""
|
|
15
|
+
from ._version import __version__
|
|
16
|
+
from .config import ConfigError, Settings
|
|
17
|
+
|
|
18
|
+
__all__ = ["ConfigError", "Settings", "__version__", "run", "run_async"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run(settings=None, **overrides):
|
|
22
|
+
"""Run (or resume) a migration. See :func:`telegram_migration.runner.run`."""
|
|
23
|
+
from .runner import run as _run
|
|
24
|
+
return _run(settings, **overrides)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def run_async(settings=None, **overrides):
|
|
28
|
+
"""Coroutine version of :func:`run`. See :func:`telegram_migration.pipeline.run_async`."""
|
|
29
|
+
from .pipeline import run_async as _run_async
|
|
30
|
+
return await _run_async(settings, **overrides)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Oversize-file handling: size check, manual-upload ledger, stop instructions."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
from . import config, progress
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def max_upload_bytes():
|
|
11
|
+
"""The size limit in bytes, or 0 when the check is disabled."""
|
|
12
|
+
return int(config.settings.max_upload_mb * 1024 * 1024) if config.settings.max_upload_mb else 0
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def format_mb(n_bytes):
|
|
16
|
+
return f"{n_bytes / (1024 * 1024):.1f} MB"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def find_oversize_media(media):
|
|
20
|
+
"""[(path, kind, size_bytes)] for every file bigger than the limit (strictly greater)."""
|
|
21
|
+
limit = max_upload_bytes()
|
|
22
|
+
if not limit:
|
|
23
|
+
return []
|
|
24
|
+
out = []
|
|
25
|
+
for path, kind in media:
|
|
26
|
+
try:
|
|
27
|
+
size = os.path.getsize(path)
|
|
28
|
+
except OSError:
|
|
29
|
+
continue # unreadable here: the upload itself will report it
|
|
30
|
+
if size > limit:
|
|
31
|
+
out.append((path, kind, size))
|
|
32
|
+
return out
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def load_manual_uploads():
|
|
36
|
+
"""Source message ids you handled by hand, from the manual-uploads file (settings.manual_uploads_file).
|
|
37
|
+
|
|
38
|
+
One id per line, optionally followed by the id of the message you posted in
|
|
39
|
+
the channel (so later replies can link to it):
|
|
40
|
+
1234
|
|
41
|
+
1234 5678 # the second number is the destination message id
|
|
42
|
+
Blank lines and anything after '#' are ignored.
|
|
43
|
+
Returns (set_of_source_ids, {source_id: destination_id}).
|
|
44
|
+
"""
|
|
45
|
+
done, mapped = set(), {}
|
|
46
|
+
if not os.path.exists(config.settings.manual_uploads_file):
|
|
47
|
+
return done, mapped
|
|
48
|
+
with open(config.settings.manual_uploads_file, encoding="utf-8") as f:
|
|
49
|
+
for raw in f:
|
|
50
|
+
line = raw.split("#", 1)[0].strip()
|
|
51
|
+
if not line:
|
|
52
|
+
continue
|
|
53
|
+
parts = line.replace(",", " ").split()
|
|
54
|
+
src = re.sub(r"^message", "", parts[0], flags=re.IGNORECASE)
|
|
55
|
+
if not src.isdigit():
|
|
56
|
+
print(f"Warning: ignoring unreadable line in {os.path.basename(config.settings.manual_uploads_file)}: {raw.strip()!r}")
|
|
57
|
+
continue
|
|
58
|
+
done.add(src)
|
|
59
|
+
if len(parts) > 1 and parts[1].isdigit():
|
|
60
|
+
mapped[src] = int(parts[1])
|
|
61
|
+
return done, mapped
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def mark_manual_upload(source_id, rel_files):
|
|
65
|
+
"""Append `source_id` to the manual-uploads file (once). True if it is recorded afterwards.
|
|
66
|
+
|
|
67
|
+
The line is "<id> # comment"; load_manual_uploads() ignores the comment, and you can
|
|
68
|
+
write the channel message id of your upload after the number to link replies to it.
|
|
69
|
+
"""
|
|
70
|
+
try:
|
|
71
|
+
done, _mapped = load_manual_uploads()
|
|
72
|
+
if str(source_id) in done:
|
|
73
|
+
return True
|
|
74
|
+
existing = ""
|
|
75
|
+
if os.path.exists(config.settings.manual_uploads_file):
|
|
76
|
+
with open(config.settings.manual_uploads_file, encoding="utf-8") as f:
|
|
77
|
+
existing = f.read()
|
|
78
|
+
lead = "" if (not existing or existing.endswith("\n")) else "\n"
|
|
79
|
+
header = "" if existing else (
|
|
80
|
+
"# Source message ids you uploaded by hand (the script skips them).\n"
|
|
81
|
+
"# Format: <id> or <id> <channel message id of your upload> # comment\n")
|
|
82
|
+
stamp = datetime.now().strftime("%Y-%m-%d %H:%M")
|
|
83
|
+
files = ", ".join(rel_files)
|
|
84
|
+
with open(config.settings.manual_uploads_file, "a", encoding="utf-8") as f:
|
|
85
|
+
f.write(f"{lead}{header}{source_id} # auto-added {stamp} after an oversize stop: upload {files} by hand\n")
|
|
86
|
+
f.flush()
|
|
87
|
+
return True
|
|
88
|
+
except Exception as exc:
|
|
89
|
+
print(f"Warning: could not write {config.settings.manual_uploads_file}: {type(exc).__name__}: {exc}")
|
|
90
|
+
return False
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def format_oversize_instructions(source_id, data, caption, oversize, last_dest_id, migrated, auto_marked=False):
|
|
94
|
+
big = {os.path.abspath(p) for p, _k, _s in oversize}
|
|
95
|
+
ts = data.get("timestamp")
|
|
96
|
+
when = ts.strftime("%Y-%m-%d %H:%M") if ts else "unknown time"
|
|
97
|
+
who = data.get("sender") or "unknown sender"
|
|
98
|
+
lines = [
|
|
99
|
+
"",
|
|
100
|
+
"═" * 72,
|
|
101
|
+
f"STOPPED: message id={source_id} has a file over the {format_mb(max_upload_bytes())} upload limit.",
|
|
102
|
+
"═" * 72,
|
|
103
|
+
"None of this message's files were uploaded by the script. It is logged in:",
|
|
104
|
+
f" {config.settings.failed_file}",
|
|
105
|
+
"",
|
|
106
|
+
f"Upload these file(s) to the channel by hand (original message: {when}, {who}):",
|
|
107
|
+
]
|
|
108
|
+
for path, kind in data["media"]:
|
|
109
|
+
ap = os.path.abspath(path)
|
|
110
|
+
try:
|
|
111
|
+
size = format_mb(os.path.getsize(path))
|
|
112
|
+
except OSError:
|
|
113
|
+
size = "size unknown"
|
|
114
|
+
lines.append(f" [{kind}] {ap} ({size}){' <-- TOO LARGE' if ap in big else ''}")
|
|
115
|
+
if caption and caption.strip():
|
|
116
|
+
shown = caption if len(caption) <= 1500 else caption[:1500] + "…"
|
|
117
|
+
lines += ["", "Caption to paste with it:"] + [" | " + ln for ln in shown.splitlines()]
|
|
118
|
+
where = (f"after the {migrated} message(s) already migrated"
|
|
119
|
+
+ (f" (the last one is message #{last_dest_id} in the channel)" if last_dest_id else ""))
|
|
120
|
+
if auto_marked:
|
|
121
|
+
lines += [
|
|
122
|
+
"",
|
|
123
|
+
"To resume:",
|
|
124
|
+
f" 1. Upload the file(s) now, BEFORE running the script again, so they land {where}",
|
|
125
|
+
" and ahead of the messages that follow.",
|
|
126
|
+
f" 2. Nothing to edit: message {source_id} is already recorded in",
|
|
127
|
+
f" {config.settings.manual_uploads_file}",
|
|
128
|
+
" so the next run skips it and continues with the following message.",
|
|
129
|
+
f" (Optional: write the channel message id of your upload after the number, e.g.",
|
|
130
|
+
f" \"{source_id} 5678\", so replies to this message link to it. To undo, delete the line.)",
|
|
131
|
+
" 3. Run the script again.",
|
|
132
|
+
"",
|
|
133
|
+
"WARNING: if you run the script again BEFORE uploading, this message will be skipped.",
|
|
134
|
+
]
|
|
135
|
+
else:
|
|
136
|
+
lines += [
|
|
137
|
+
"",
|
|
138
|
+
"To resume:",
|
|
139
|
+
f" 1. Upload the file(s) now, BEFORE running the script again, so they land {where}",
|
|
140
|
+
" and ahead of the messages that follow.",
|
|
141
|
+
f" 2. Add this line to {config.settings.manual_uploads_file}:",
|
|
142
|
+
f" {source_id}",
|
|
143
|
+
" (optionally add the channel message id of your upload after it, e.g.",
|
|
144
|
+
f" \"{source_id} 5678\", so replies to this message link to it).",
|
|
145
|
+
" 3. Run the script again. It continues with the next message.",
|
|
146
|
+
]
|
|
147
|
+
lines += [
|
|
148
|
+
"",
|
|
149
|
+
f"All progress is saved ({migrated} message(s) migrated so far).",
|
|
150
|
+
"═" * 72,
|
|
151
|
+
]
|
|
152
|
+
return "\n".join(lines)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def halt_for_oversize(source_id, data, caption, oversize, id_map, ctx):
|
|
156
|
+
"""Log the message id, print the manual-upload instructions and request a clean stop."""
|
|
157
|
+
reason = "FILE_TOO_LARGE: " + "; ".join(
|
|
158
|
+
f"{os.path.relpath(p, config.settings.base_dir)} ({format_mb(sz)} > {format_mb(max_upload_bytes())} limit)"
|
|
159
|
+
for p, _k, sz in oversize
|
|
160
|
+
)
|
|
161
|
+
progress.log_failure(source_id, reason)
|
|
162
|
+
last_dest = ctx.get("last_dest_id") or (max(id_map.values()) if id_map else None)
|
|
163
|
+
auto = False
|
|
164
|
+
if config.settings.auto_mark_manual_on_stop:
|
|
165
|
+
auto = mark_manual_upload(source_id, [os.path.relpath(p, config.settings.base_dir) for p, _k in data["media"]])
|
|
166
|
+
print(format_oversize_instructions(source_id, data, caption, oversize, last_dest, len(id_map), auto_marked=auto))
|
|
167
|
+
ctx["halt"].append(source_id)
|
|
168
|
+
ctx["stop"] = True
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def print_oversize_dry_run(source_id, data, oversize):
|
|
172
|
+
for path, kind, size in oversize:
|
|
173
|
+
print(f" [dry] ⚠ id={source_id:<6} OVERSIZE [{kind}] {os.path.abspath(path)} ({format_mb(size)})"
|
|
174
|
+
f" - a real run stops here")
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Command line: ``python -m telegram_migration`` or the ``telegram-migration`` command."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from ._version import __version__
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def confirm_target(channel, pending, n_files, settings):
|
|
9
|
+
print()
|
|
10
|
+
print("═" * 60)
|
|
11
|
+
print(f" Target: {getattr(channel, 'title', '?')!r}")
|
|
12
|
+
print(f" ID: {getattr(channel, 'id', '?')}")
|
|
13
|
+
uname = getattr(channel, "username", None)
|
|
14
|
+
if uname:
|
|
15
|
+
print(f" Handle: @{uname}")
|
|
16
|
+
print(f" Files: {n_files} export file(s)")
|
|
17
|
+
print(f" Pending: ~{pending} messages")
|
|
18
|
+
print("═" * 60)
|
|
19
|
+
if settings.dry_run:
|
|
20
|
+
print("DRY RUN — nothing will actually be sent.\n")
|
|
21
|
+
return True
|
|
22
|
+
if settings.skip_confirmation:
|
|
23
|
+
print("SKIP_CONFIRMATION is on — proceeding.\n")
|
|
24
|
+
return True
|
|
25
|
+
return input("Send to this channel? [y/N]: ").strip().lower() in ("y", "yes")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
29
|
+
parser = argparse.ArgumentParser(
|
|
30
|
+
prog="telegram-migration",
|
|
31
|
+
description="Rebuild a Telegram Desktop chat export in a channel with a Telegram bot. "
|
|
32
|
+
"Settings come from the TELEGRAM_* environment variables / .env file; "
|
|
33
|
+
"the options below override them. Dry run is ON unless you pass --send.",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
36
|
+
parser.add_argument("--env-file", metavar="PATH", help="path of a .env file to load (default: ./.env if present)")
|
|
37
|
+
parser.add_argument("--export-dir", metavar="PATH", help="folder with messages.html and the media folders")
|
|
38
|
+
parser.add_argument("--channel", metavar="PEER", help="destination: numeric peer id (-100...) or @username")
|
|
39
|
+
mode = parser.add_mutually_exclusive_group()
|
|
40
|
+
mode.add_argument("--dry-run", dest="dry_run", action="store_true", default=None,
|
|
41
|
+
help="preview only; nothing is sent")
|
|
42
|
+
mode.add_argument("--send", dest="dry_run", action="store_false",
|
|
43
|
+
help="really send the messages (turns the dry run off)")
|
|
44
|
+
parser.add_argument("--yes", dest="skip_confirmation", action="store_true", default=None,
|
|
45
|
+
help="do not ask for confirmation before sending")
|
|
46
|
+
parser.add_argument("--max-upload-mb", type=float, metavar="N",
|
|
47
|
+
help="stop at any message with a file larger than this (0 disables)")
|
|
48
|
+
parser.add_argument("--batch-limit", type=int, metavar="N", help="stop after this many messages")
|
|
49
|
+
parser.add_argument("--delay", type=float, dest="send_delay_seconds", metavar="SECONDS",
|
|
50
|
+
help="seconds to wait between messages")
|
|
51
|
+
return parser
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def main(argv=None) -> int:
|
|
55
|
+
args = build_parser().parse_args(argv)
|
|
56
|
+
|
|
57
|
+
overrides = {}
|
|
58
|
+
if args.export_dir is not None:
|
|
59
|
+
overrides["base_dir"] = args.export_dir
|
|
60
|
+
if args.channel is not None:
|
|
61
|
+
try:
|
|
62
|
+
overrides["channel_peer"] = int(args.channel)
|
|
63
|
+
except ValueError:
|
|
64
|
+
overrides["channel_peer"] = args.channel
|
|
65
|
+
for name in ("dry_run", "skip_confirmation", "batch_limit", "send_delay_seconds"):
|
|
66
|
+
if getattr(args, name) is not None:
|
|
67
|
+
overrides[name] = getattr(args, name)
|
|
68
|
+
if args.max_upload_mb is not None:
|
|
69
|
+
overrides["max_upload_mb"] = args.max_upload_mb or None
|
|
70
|
+
|
|
71
|
+
from .config import ConfigError, Settings
|
|
72
|
+
try:
|
|
73
|
+
settings = Settings.from_env(env_file=args.env_file, **overrides).validate()
|
|
74
|
+
except ConfigError as exc:
|
|
75
|
+
print(f"Configuration error: {exc}", file=sys.stderr)
|
|
76
|
+
return 2
|
|
77
|
+
|
|
78
|
+
from .runner import run
|
|
79
|
+
try:
|
|
80
|
+
result = run(settings)
|
|
81
|
+
except KeyboardInterrupt:
|
|
82
|
+
print("\nInterrupted. Progress made so far is saved; run again to resume.", file=sys.stderr)
|
|
83
|
+
return 130
|
|
84
|
+
# run() returns True on success, user abort, or a clean oversize stop, and False only
|
|
85
|
+
# when the migration could not start (no export, or the channel did not resolve).
|
|
86
|
+
return 0 if result is not False else 1
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Settings for a migration run.
|
|
2
|
+
|
|
3
|
+
Build a :class:`Settings` yourself, or call :meth:`Settings.from_env` to read the
|
|
4
|
+
``TELEGRAM_*`` environment variables (a local ``.env`` file and Google Colab Secrets are
|
|
5
|
+
supported). The package modules read the *active* settings through ``config.settings``
|
|
6
|
+
at call time; :func:`use_settings` scopes them to the current thread/async context.
|
|
7
|
+
|
|
8
|
+
Nothing is read from the environment at import time.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import dataclasses
|
|
13
|
+
import os
|
|
14
|
+
from contextvars import ContextVar
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Mapping, Optional, Tuple, Union
|
|
18
|
+
|
|
19
|
+
from .constants import BOT_CAPTION_LIMIT, MAX_CAPTION_LIMIT, MIN_TEXT_LIMIT, TEXT_LIMIT
|
|
20
|
+
|
|
21
|
+
PLACEHOLDER_API_ID = 12345678 # the value shipped in .env.example
|
|
22
|
+
_TRUE = {"1", "true", "yes", "y", "on"}
|
|
23
|
+
|
|
24
|
+
# Progress files kept inside the export folder: attribute name -> file name.
|
|
25
|
+
_DERIVED_FILES = {
|
|
26
|
+
"idmap_file": "migration_id_map.jsonl",
|
|
27
|
+
"failed_file": "failed_messages.txt",
|
|
28
|
+
"partial_file": "migration_partial_sends.jsonl",
|
|
29
|
+
"pending_replies_file": "migration_pending_replies.json",
|
|
30
|
+
"reaction_note_map_file": "migration_reaction_notes.jsonl",
|
|
31
|
+
"reaction_archive_file": "migration_reactions_archive.jsonl",
|
|
32
|
+
"reaction_source_file": "migration_reactions_source.jsonl",
|
|
33
|
+
"reply_audit_file": "migration_reply_audit.jsonl",
|
|
34
|
+
"format_debug_file": "migration_formatting_dropped.jsonl",
|
|
35
|
+
"manual_uploads_file": "migration_manual_uploads.txt",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ConfigError(ValueError):
|
|
40
|
+
"""The settings are incomplete or still contain example values."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _truthy(value: Optional[str], default: bool) -> bool:
|
|
44
|
+
if value is None or not str(value).strip():
|
|
45
|
+
return default
|
|
46
|
+
return str(value).strip().casefold() in _TRUE
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def load_secret(name: str, env: Optional[Mapping[str, str]] = None) -> str:
|
|
50
|
+
"""Read a value from the environment, then (in Google Colab) from Colab Secrets."""
|
|
51
|
+
source = os.environ if env is None else env
|
|
52
|
+
value = (source.get(name, "") or "").strip()
|
|
53
|
+
if value or env is not None:
|
|
54
|
+
return value
|
|
55
|
+
try:
|
|
56
|
+
from google.colab import userdata # type: ignore
|
|
57
|
+
return (userdata.get(name) or "").strip()
|
|
58
|
+
except Exception:
|
|
59
|
+
return ""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _load_dotenv(env_file: Optional[str]) -> None:
|
|
63
|
+
try:
|
|
64
|
+
from dotenv import load_dotenv
|
|
65
|
+
except ImportError:
|
|
66
|
+
print("Warning: python-dotenv is not installed; .env files will be ignored.")
|
|
67
|
+
return
|
|
68
|
+
load_dotenv(env_file) if env_file else load_dotenv()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class Settings:
|
|
73
|
+
"""Everything that can be configured. Defaults are the safe ones (dry run on)."""
|
|
74
|
+
|
|
75
|
+
# --- Telegram credentials -------------------------------------------------------------
|
|
76
|
+
api_id: Optional[int] = None
|
|
77
|
+
api_hash: str = field(default="", repr=False)
|
|
78
|
+
bot_token: str = field(default="", repr=False)
|
|
79
|
+
channel_peer: Union[int, str, None] = None # numeric peer id (-100...) or "@username"
|
|
80
|
+
session_name: str = "colab_bot_restorer_session"
|
|
81
|
+
|
|
82
|
+
# --- where things live ----------------------------------------------------------------
|
|
83
|
+
base_dir: str = "./telegram_export" # folder with messages.html, media folders, ...
|
|
84
|
+
# Progress files; when left as None they are placed inside base_dir.
|
|
85
|
+
idmap_file: Optional[str] = None
|
|
86
|
+
failed_file: Optional[str] = None
|
|
87
|
+
partial_file: Optional[str] = None
|
|
88
|
+
pending_replies_file: Optional[str] = None
|
|
89
|
+
reaction_note_map_file: Optional[str] = None
|
|
90
|
+
reaction_archive_file: Optional[str] = None
|
|
91
|
+
reaction_source_file: Optional[str] = None
|
|
92
|
+
reply_audit_file: Optional[str] = None
|
|
93
|
+
format_debug_file: Optional[str] = None
|
|
94
|
+
manual_uploads_file: Optional[str] = None
|
|
95
|
+
|
|
96
|
+
# --- behaviour --------------------------------------------------------------------------
|
|
97
|
+
dry_run: bool = True # preview only; nothing is sent
|
|
98
|
+
skip_confirmation: bool = False # do not ask before sending
|
|
99
|
+
# Largest file the run will try to upload; a message with a bigger file stops the run
|
|
100
|
+
# (nothing from it is sent) so it can be uploaded by hand. None or 0 disables the check.
|
|
101
|
+
# 2000 MiB is believed to be Telegram's MTProto ceiling; verify it with your bot.
|
|
102
|
+
max_upload_mb: Optional[float] = 2000
|
|
103
|
+
# On an oversize stop, record that message in manual_uploads_file so the next run skips it.
|
|
104
|
+
# UPLOAD THE FILE(S) BEFORE RERUNNING, or that message is left out of the channel.
|
|
105
|
+
auto_mark_manual_on_stop: bool = True
|
|
106
|
+
batch_limit: int = 1000 # stop after this many messages (sent + failed)
|
|
107
|
+
max_retries: int = 5
|
|
108
|
+
max_flood_waits: int = 10
|
|
109
|
+
send_delay_seconds: float = 2.5
|
|
110
|
+
preserve_sender_names: bool = True
|
|
111
|
+
preserve_reactions_as_annotations: bool = True
|
|
112
|
+
json_reaction_file_candidates: Tuple[str, ...] = ("result.json", "messages.json")
|
|
113
|
+
max_json_fallback_mb: int = 512
|
|
114
|
+
verify_replies: bool = True
|
|
115
|
+
# Telegram's message limit is 4096 for every account; lower it only to split long text into
|
|
116
|
+
# shorter parts. Settings.validate() rejects values above Telegram's maximum and below
|
|
117
|
+
# MIN_TEXT_LIMIT, where the splitter's surrogate-pair handling could produce a chunk
|
|
118
|
+
# slightly larger than the requested limit.
|
|
119
|
+
text_limit: int = TEXT_LIMIT
|
|
120
|
+
# Caption limit for media. A bot account is not Premium, so this is BOT_CAPTION_LIMIT
|
|
121
|
+
# (1024) rather than MAX_CAPTION_LIMIT (2048, Premium user accounts only).
|
|
122
|
+
caption_limit: int = BOT_CAPTION_LIMIT
|
|
123
|
+
# Try a native forward first when the export links the original message; a native
|
|
124
|
+
# forward cannot carry our timestamp/sender line, so forward_annotate sends it as a reply.
|
|
125
|
+
native_forward: bool = True
|
|
126
|
+
forward_annotate: bool = True
|
|
127
|
+
|
|
128
|
+
def __post_init__(self) -> None:
|
|
129
|
+
self.base_dir = os.path.abspath(os.path.expanduser(str(self.base_dir)))
|
|
130
|
+
for attr, filename in _DERIVED_FILES.items():
|
|
131
|
+
if getattr(self, attr) is None:
|
|
132
|
+
setattr(self, attr, os.path.join(self.base_dir, filename))
|
|
133
|
+
|
|
134
|
+
# ------------------------------------------------------------------------------------
|
|
135
|
+
def replace(self, **changes) -> "Settings":
|
|
136
|
+
"""Copy with some fields changed. Changing base_dir moves the progress files too."""
|
|
137
|
+
if "base_dir" in changes:
|
|
138
|
+
for attr in _DERIVED_FILES:
|
|
139
|
+
changes.setdefault(attr, None)
|
|
140
|
+
return dataclasses.replace(self, **changes)
|
|
141
|
+
|
|
142
|
+
def problems(self) -> list:
|
|
143
|
+
"""Human-readable list of what is missing or still an example value."""
|
|
144
|
+
found = []
|
|
145
|
+
if self.api_id is None:
|
|
146
|
+
found.append("TELEGRAM_API_ID is not set")
|
|
147
|
+
elif self.api_id == PLACEHOLDER_API_ID:
|
|
148
|
+
found.append("TELEGRAM_API_ID is still the example value; use your own from my.telegram.org")
|
|
149
|
+
for env_name, value in (("TELEGRAM_API_HASH", self.api_hash), ("TELEGRAM_BOT_TOKEN", self.bot_token)):
|
|
150
|
+
if not value:
|
|
151
|
+
found.append(f"{env_name} is not set")
|
|
152
|
+
elif value.startswith("replace_with"):
|
|
153
|
+
found.append(f"{env_name} is still the example value")
|
|
154
|
+
if self.channel_peer in (None, ""):
|
|
155
|
+
found.append("TELEGRAM_CHANNEL_PEER is not set (numeric peer id or @username)")
|
|
156
|
+
if not MIN_TEXT_LIMIT <= self.text_limit <= TEXT_LIMIT:
|
|
157
|
+
found.append(
|
|
158
|
+
f"text_limit must be between {MIN_TEXT_LIMIT} and {TEXT_LIMIT} "
|
|
159
|
+
f"(Telegram's maximum; the lower bound avoids a splitter edge case)"
|
|
160
|
+
)
|
|
161
|
+
if not MIN_TEXT_LIMIT <= self.caption_limit <= BOT_CAPTION_LIMIT:
|
|
162
|
+
found.append(
|
|
163
|
+
f"caption_limit must be between {MIN_TEXT_LIMIT} and {BOT_CAPTION_LIMIT} "
|
|
164
|
+
f"(the limit for bot accounts; Premium user accounts can reach {MAX_CAPTION_LIMIT}; "
|
|
165
|
+
f"the lower bound avoids a splitter edge case)"
|
|
166
|
+
)
|
|
167
|
+
return found
|
|
168
|
+
|
|
169
|
+
def validate(self) -> "Settings":
|
|
170
|
+
"""Raise ConfigError if credentials or the destination are missing."""
|
|
171
|
+
found = self.problems()
|
|
172
|
+
if found:
|
|
173
|
+
raise ConfigError(
|
|
174
|
+
"; ".join(found) + ". Set them in .env/the environment or as Colab Secrets. "
|
|
175
|
+
"Never commit credentials to Git."
|
|
176
|
+
)
|
|
177
|
+
return self
|
|
178
|
+
|
|
179
|
+
@classmethod
|
|
180
|
+
def from_env(cls, env: Optional[Mapping[str, str]] = None, *, env_file: Optional[str] = None,
|
|
181
|
+
use_dotenv: bool = True, **overrides) -> "Settings":
|
|
182
|
+
"""Read the TELEGRAM_* variables. Explicit keyword arguments win over the environment.
|
|
183
|
+
|
|
184
|
+
With env=None the process environment is used, a local .env file is loaded first
|
|
185
|
+
(it never overrides variables that are already set) and Colab Secrets are consulted.
|
|
186
|
+
Pass a mapping as ``env`` to read from it only (useful in tests).
|
|
187
|
+
"""
|
|
188
|
+
if env is None and use_dotenv:
|
|
189
|
+
_load_dotenv(env_file)
|
|
190
|
+
get = lambda name: load_secret(name, env) # noqa: E731
|
|
191
|
+
|
|
192
|
+
values: dict = {}
|
|
193
|
+
raw_id = get("TELEGRAM_API_ID")
|
|
194
|
+
if raw_id:
|
|
195
|
+
try:
|
|
196
|
+
values["api_id"] = int(raw_id)
|
|
197
|
+
except ValueError:
|
|
198
|
+
raise ConfigError(f"TELEGRAM_API_ID must be a number, got {raw_id!r}") from None
|
|
199
|
+
values["api_hash"] = get("TELEGRAM_API_HASH")
|
|
200
|
+
values["bot_token"] = get("TELEGRAM_BOT_TOKEN")
|
|
201
|
+
raw_peer = get("TELEGRAM_CHANNEL_PEER")
|
|
202
|
+
if raw_peer:
|
|
203
|
+
try:
|
|
204
|
+
values["channel_peer"] = int(raw_peer)
|
|
205
|
+
except ValueError:
|
|
206
|
+
values["channel_peer"] = raw_peer
|
|
207
|
+
source = os.environ if env is None else env
|
|
208
|
+
if source.get("TELEGRAM_EXPORT_DIR"):
|
|
209
|
+
values["base_dir"] = source["TELEGRAM_EXPORT_DIR"]
|
|
210
|
+
if source.get("TELEGRAM_SESSION_NAME"):
|
|
211
|
+
values["session_name"] = source["TELEGRAM_SESSION_NAME"]
|
|
212
|
+
values["dry_run"] = _truthy(source.get("TELEGRAM_DRY_RUN"), True)
|
|
213
|
+
values["skip_confirmation"] = _truthy(source.get("TELEGRAM_SKIP_CONFIRMATION"), False)
|
|
214
|
+
|
|
215
|
+
raw = (source.get("TELEGRAM_MAX_UPLOAD_MB") or "").strip().casefold()
|
|
216
|
+
if raw:
|
|
217
|
+
values["max_upload_mb"] = None if raw in {"0", "none", "off", "false"} else _number(raw, "TELEGRAM_MAX_UPLOAD_MB", float)
|
|
218
|
+
raw = (source.get("TELEGRAM_BATCH_LIMIT") or "").strip()
|
|
219
|
+
if raw:
|
|
220
|
+
values["batch_limit"] = _number(raw, "TELEGRAM_BATCH_LIMIT", int)
|
|
221
|
+
raw = (source.get("TELEGRAM_SEND_DELAY") or "").strip()
|
|
222
|
+
if raw:
|
|
223
|
+
values["send_delay_seconds"] = _number(raw, "TELEGRAM_SEND_DELAY", float)
|
|
224
|
+
|
|
225
|
+
values.update(overrides)
|
|
226
|
+
return cls(**values)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _number(raw: str, name: str, kind):
|
|
230
|
+
try:
|
|
231
|
+
return kind(raw)
|
|
232
|
+
except ValueError:
|
|
233
|
+
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
# Defaults only: nothing is read from the environment until Settings.from_env() (or run()).
|
|
237
|
+
_default_settings = Settings()
|
|
238
|
+
_active_settings = ContextVar("telegram_migration_settings", default=None)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class _SettingsProxy:
|
|
242
|
+
"""Resolve settings in the current async context, falling back to defaults."""
|
|
243
|
+
|
|
244
|
+
def __getattr__(self, name):
|
|
245
|
+
return getattr(get_settings(), name)
|
|
246
|
+
|
|
247
|
+
# Read-only on purpose. Writing through the proxy would change whichever Settings happens to be
|
|
248
|
+
# active (and unittest.mock.patch.object, which restores a value by *deleting* it, would then
|
|
249
|
+
# leave a field reset to its class default instead of its original value).
|
|
250
|
+
def __setattr__(self, name, value):
|
|
251
|
+
raise AttributeError(
|
|
252
|
+
"config.settings is read-only; build a Settings and activate it with "
|
|
253
|
+
"config.use_settings(...) or config.set_settings(...)"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def __delattr__(self, name):
|
|
257
|
+
raise AttributeError(
|
|
258
|
+
"config.settings is read-only; build a Settings and activate it with "
|
|
259
|
+
"config.use_settings(...) or config.set_settings(...)"
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
def __repr__(self):
|
|
263
|
+
return repr(get_settings())
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
settings = _SettingsProxy()
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def get_settings() -> Settings:
|
|
270
|
+
"""Return the settings active in the current thread/async context."""
|
|
271
|
+
active = _active_settings.get()
|
|
272
|
+
return active if active is not None else _default_settings
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def set_settings(new: Settings) -> Settings:
|
|
276
|
+
"""Set fallback settings for callers overriding the default outside `use_settings`."""
|
|
277
|
+
global _default_settings
|
|
278
|
+
_default_settings = new
|
|
279
|
+
return new
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@contextmanager
|
|
283
|
+
def use_settings(new: Settings):
|
|
284
|
+
"""Temporarily activate `new` in this thread/async context only."""
|
|
285
|
+
token = _active_settings.set(new)
|
|
286
|
+
try:
|
|
287
|
+
yield new
|
|
288
|
+
finally:
|
|
289
|
+
_active_settings.reset(token)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def resolve_settings(settings_arg: Optional[Settings] = None, **overrides) -> Settings:
|
|
293
|
+
"""`settings_arg` (or Settings.from_env()) with keyword overrides applied."""
|
|
294
|
+
if settings_arg is None:
|
|
295
|
+
return Settings.from_env(**overrides)
|
|
296
|
+
return settings_arg.replace(**overrides) if overrides else settings_arg
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Fixed values that are part of how Telegram and the export work (not user settings)."""
|
|
2
|
+
|
|
3
|
+
# Telegram's hard limit for the text of one message (every account, Premium included).
|
|
4
|
+
TEXT_LIMIT = 4096
|
|
5
|
+
|
|
6
|
+
# The largest caption any account can send (Premium user accounts); bots are limited to 1024.
|
|
7
|
+
MAX_CAPTION_LIMIT = 2048
|
|
8
|
+
|
|
9
|
+
# The caption limit this tool can actually use: the script authenticates with a bot token,
|
|
10
|
+
# and bot accounts cannot be Premium, so 1024 is the effective ceiling.
|
|
11
|
+
BOT_CAPTION_LIMIT = 1024
|
|
12
|
+
|
|
13
|
+
# The smallest text_limit or caption_limit that makes sense. Below 4, the splitter's
|
|
14
|
+
# surrogate-pair avoidance can produce a chunk that exceeds the requested limit by one
|
|
15
|
+
# UTF-16 unit; Settings.validate() rejects such values so the splitter never faces that
|
|
16
|
+
# choice through the public API.
|
|
17
|
+
MIN_TEXT_LIMIT = 4
|
|
18
|
+
|
|
19
|
+
# messages.html, messages2.html, ... in a Telegram Desktop HTML export.
|
|
20
|
+
HTML_FILE_PATTERN = r"^messages(\d*)\.html$"
|
|
21
|
+
|
|
22
|
+
# Sticker files need sticker-specific upload attributes and cannot be captioned like
|
|
23
|
+
# ordinary photos/videos, so their text is sent as a separate message first.
|
|
24
|
+
STICKER_KINDS = ("sticker_static", "sticker_animated", "sticker_video", "sticker_unknown")
|
|
25
|
+
NO_CAPTION_KINDS = ("voice", "round_video", *STICKER_KINDS)
|