tlgr-cli 2.0.1__py3-none-any.whl

tlgr/core/telethon_compat.py

@@ -0,0 +1,295 @@
+"""Every private Telethon API tlgr touches, behind a feature probe.
+
+tlgr needs four things Telethon 1.44 does not expose publicly: persisting
+update state without disconnecting, learning that a reconnect happened,
+learning that the server said "your gap is too long", and reading a request's
+flood-wait memory. Reaching into `client._save_states_and_entities` from five
+call sites would mean five tracebacks the day Telethon renames it.
+
+The rule here: **probe once, warn once, degrade**. Each adapter checks for the
+attribute it needs, logs a single warning naming the Telethon version if it is
+gone, and returns a value that lets the caller carry on. Losing the periodic
+state save costs at most one `catch_up()`; losing the `*TooLong` hook costs a
+resync the daemon would have done anyway on its 15-minute backstop. Neither is
+worth a crash, and both are worth a log line that names the cause.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+from collections.abc import Callable
+from typing import Any
+
+log = logging.getLogger("tlgr.compat")
+
+__all__ = [
+    "TOO_LONG_CHANNEL",
+    "TOO_LONG_GLOBAL",
+    "entity_count",
+    "install_reconnect_hook",
+    "install_too_long_hook",
+    "probe",
+    "save_state",
+    "session_state",
+    "set_session_state",
+    "telethon_version",
+]
+
+TOO_LONG_GLOBAL = "global"
+TOO_LONG_CHANNEL = "channel"
+
+_warned: set[str] = set()
+
+
+def telethon_version() -> str:
+    try:
+        import telethon
+
+        return str(telethon.__version__)
+    except Exception:  # pragma: no cover - telethon is a hard dependency
+        return "unknown"
+
+
+def _warn_once(feature: str, detail: str = "") -> None:
+    if feature in _warned:
+        return
+    _warned.add(feature)
+    log.warning(
+        "telethon %s does not expose %s%s; tlgr degrades that behaviour",
+        telethon_version(),
+        feature,
+        f" ({detail})" if detail else "",
+    )
+
+
+def probe(client: Any, attribute: str) -> bool:
+    """Is *attribute* present on this client? Warns once when it is not."""
+    if hasattr(client, attribute):
+        return True
+    _warn_once(attribute)
+    return False
+
+
+async def save_state(client: Any) -> bool:
+    """Persist update state and the entity cache without disconnecting.
+
+    Telethon writes `pts`/`qts`/`date` only on `disconnect()`. A daemon that is
+    SIGKILLed, or a laptop that loses power, therefore replays from whatever
+    the session file last held — in v1, from the last clean shutdown, which
+    could be days. Called every `[daemon] state_save_interval` seconds and on
+    every shutdown path.
+    """
+    ok = True
+    saver = getattr(client, "_save_states_and_entities", None)
+    if callable(saver):
+        try:
+            result = saver()
+            if hasattr(result, "__await__"):
+                await result
+        except Exception as exc:  # pragma: no cover - depends on session backend
+            log.debug("state save failed: %s", exc)
+            ok = False
+    else:
+        _warn_once("_save_states_and_entities")
+        ok = False
+
+    session = getattr(client, "session", None)
+    save = getattr(session, "save", None)
+    if callable(save):
+        try:
+            save()
+        except Exception as exc:  # pragma: no cover
+            log.debug("session save failed: %s", exc)
+            ok = False
+    return ok
+
+
+def install_reconnect_hook(client: Any, callback: Callable[[], Any]) -> bool:
+    """Call *callback* after Telethon finishes an automatic reconnect.
+
+    Telethon's `_handle_auto_reconnect` re-runs `get_me()` and nothing else, so
+    an account that dropped for ten minutes comes back with a stale `pts` and
+    silently misses everything that happened (checklist 1/2). Wrapping it is
+    the only hook there is; when it is gone the supervisor's own reconnect path
+    still calls `catch_up()`, so this is an optimisation, not a requirement.
+    """
+    original = getattr(client, "_handle_auto_reconnect", None)
+    if not callable(original):
+        _warn_once("_handle_auto_reconnect")
+        return False
+
+    async def wrapper() -> Any:
+        result = original()
+        if hasattr(result, "__await__"):
+            result = await result
+        try:
+            outcome = callback()
+            if hasattr(outcome, "__await__"):
+                await outcome
+        except Exception as exc:  # pragma: no cover - the callback logs its own
+            log.debug("reconnect hook failed: %s", exc)
+        return result
+
+    client._handle_auto_reconnect = wrapper
+    return True
+
+
+def install_too_long_hook(client: Any, callback: Callable[[str, int | None], Any]) -> bool:
+    """Report `differenceTooLong` / `channelDifferenceTooLong` to *callback*.
+
+    Telethon consumes both inside `MessageBox` and delivers nothing to
+    handlers, so a client that was offline long enough to blow the server's
+    difference window silently skips history (checklist 9). The hook wraps
+    `MessageBox.apply_difference` / `apply_channel_difference` **on this
+    client's own box instance**, never on the class, so one account's hook
+    cannot fire for another's.
+
+    `callback(scope, channel_id)` is called with `TOO_LONG_GLOBAL`/`None` or
+    `TOO_LONG_CHANNEL`/`<id>`.
+    """
+    box = getattr(client, "_message_box", None)
+    if box is None:
+        _warn_once("_message_box")
+        return False
+
+    installed = False
+
+    original_global = getattr(box, "apply_difference", None)
+    if callable(original_global):
+
+        def apply_difference(diff: Any, chat_hashes: Any, _orig: Any = original_global) -> Any:
+            if type(diff).__name__ == "DifferenceTooLong":
+                _safe(callback, TOO_LONG_GLOBAL, None)
+            return _orig(diff, chat_hashes)
+
+        box.apply_difference = apply_difference
+        installed = True
+    else:  # pragma: no cover - present in 1.44
+        _warn_once("MessageBox.apply_difference")
+
+    original_channel = getattr(box, "apply_channel_difference", None)
+    if callable(original_channel):
+
+        def apply_channel_difference(
+            request: Any, diff: Any, chat_hashes: Any, _orig: Any = original_channel
+        ) -> Any:
+            if type(diff).__name__ == "ChannelDifferenceTooLong":
+                channel = getattr(request, "channel", None)
+                _safe(callback, TOO_LONG_CHANNEL, getattr(channel, "channel_id", None))
+            return _orig(request, diff, chat_hashes)
+
+        box.apply_channel_difference = apply_channel_difference
+        installed = True
+    else:  # pragma: no cover - present in 1.44
+        _warn_once("MessageBox.apply_channel_difference")
+
+    return installed
+
+
+def _safe(callback: Callable[..., Any], *args: Any) -> None:
+    try:
+        callback(*args)
+    except Exception as exc:  # pragma: no cover
+        log.debug("too-long hook failed: %s", exc)
+
+
+def clear_config_cache(client: Any) -> bool:
+    """Drop Telethon's cached `help.getConfig` so the next call refetches.
+
+    `UpdateConfig`/`UpdateDcOptions` mean the DC list or the limits changed
+    (checklist 12). Telethon caches the config for an hour and does not listen
+    for the update, so a client that saw a DC migration keeps using the old
+    address list until the cache ages out.
+    """
+    if hasattr(client, "_config"):
+        client._config = None
+        return True
+    _warn_once("_config")
+    return False
+
+
+def session_state(client: Any) -> tuple[dict[str, Any], dict[int, int]]:
+    """`({pts, qts, seq, date}, {channel_id: pts})` from the session.
+
+    Telethon 1.44 has no public accessor for its update state: the common box
+    lives in the session's `update_state` table under entity id 0 and the
+    per-channel boxes under their channel ids. Reading it here — once, behind
+    a name — is what lets `sync status` answer "how far behind is this
+    account" without every caller reaching into a private table.
+    """
+    common: dict[str, Any] = {}
+    channels: dict[int, int] = {}
+    session = getattr(client, "session", None)
+    getter = getattr(session, "get_update_states", None)
+    if not callable(getter):
+        _warn_once("session.get_update_states")
+        return common, channels
+    try:
+        rows = list(getter())
+    except Exception as exc:  # pragma: no cover - depends on session backend
+        log.debug("update state read failed: %s", exc)
+        return common, channels
+    for entity_id, state in rows:
+        if int(entity_id) == 0:
+            date = getattr(state, "date", None)
+            common = {
+                "pts": getattr(state, "pts", None),
+                "qts": getattr(state, "qts", None),
+                "seq": getattr(state, "seq", None),
+                "date": date.strftime("%Y-%m-%dT%H:%M:%SZ") if date is not None else None,
+                "date_unix": int(date.timestamp()) if date is not None else None,
+                "unread_count": getattr(state, "unread_count", None),
+            }
+        else:
+            channels[int(entity_id)] = int(getattr(state, "pts", 0) or 0)
+    return common, channels
+
+
+def set_session_state(client: Any, state: Any, entity_id: int = 0) -> bool:
+    """Write one update-state row. The `sync reset` half of the pair above."""
+    session = getattr(client, "session", None)
+    setter = getattr(session, "set_update_state", None)
+    if not callable(setter):
+        _warn_once("session.set_update_state")
+        return False
+    try:
+        setter(entity_id, state)
+    except Exception as exc:  # pragma: no cover - depends on session backend
+        log.debug("update state write failed: %s", exc)
+        return False
+    return True
+
+
+def entity_count(client: Any) -> int:
+    """How many peers the session has an access hash for.
+
+    Not a statistic: an entity missing from here is a channel `catch_up()`
+    will silently skip, because `getChannelDifference` needs the access hash
+    and Telethon will not ask for one it does not have.
+    """
+    session = getattr(client, "session", None)
+    cursor = getattr(session, "_cursor", None)
+    if callable(cursor):
+        try:
+            row = cursor().execute("select count(*) from entities").fetchone()
+            return int(row[0]) if row else 0
+        except Exception as exc:  # pragma: no cover - depends on session backend
+            log.debug("entity count failed: %s", exc)
+    cache = getattr(client, "_entity_cache", None)
+    with contextlib.suppress(TypeError):
+        return len(cache) if cache is not None else 0
+    return 0
+
+
+def flood_waited_requests(client: Any) -> dict[int, float]:
+    """Telethon's in-process flood memory, `{constructor_id: until_unix}`.
+
+    Read-only, and lost on restart — which is why tlgr keeps its own persisted
+    copy (§6.4). Exposed here so the daemon can report both in `/v1/status`.
+    """
+    waited = getattr(client, "_flood_waited_requests", None)
+    if isinstance(waited, dict):
+        return dict(waited)
+    _warn_once("_flood_waited_requests")
+    return {}

tlgr/core/text.py

@@ -0,0 +1,211 @@
+"""Message text: parse modes, spoilers, and entity JSON.
+
+Two rules shape this module.
+
+*Parsing is input-only.* Telethon's markdown/HTML parsers are used to turn
+what a user typed into `(text, entities)`, and never to turn a received
+message back into markup: `unparse` is lossy — it cannot express overlapping
+runs, custom emoji or a URL containing the delimiter it would have to escape —
+so tlgr sends the raw `text` plus `entities` and lets the consumer decide.
+
+*Spoilers are ours.* Telethon 1.44 drops `||x||` and `<tg-spoiler>` silently:
+the markers survive as literal text in markdown and vanish without an entity
+in HTML. Either way the user's intent is lost with no error, so the markers
+are lifted out here, before and after the parser runs.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from tlgr.core.errors import UsageError
+from tlgr.models.message import MessageEntity
+
+__all__ = [
+    "PARSE_MODES",
+    "default_parse_mode",
+    "entities_from_json",
+    "entities_to_json",
+    "parse_text",
+    "utf16_len",
+]
+
+PARSE_MODES = ("md", "html", "none")
+
+#: Private-use sentinels: the parsers pass them through untouched, so a
+#: spoiler's boundaries survive markdown/HTML parsing and can be converted to
+#: real offsets afterwards. They are stripped from user input first.
+_SPOILER_OPEN = ""
+_SPOILER_CLOSE = ""
+
+
+def default_parse_mode() -> str:
+    """`[defaults] parse_mode`, itself defaulting to `none` (COR-21)."""
+    try:
+        from tlgr.core.config import load_app_config
+
+        mode = str(getattr(load_app_config().defaults, "parse_mode", "none") or "none")
+    except Exception:
+        mode = "none"
+    return mode if mode in PARSE_MODES else "none"
+
+
+def utf16_len(text: str) -> int:
+    """Length in UTF-16 code units — the unit Telegram measures offsets in.
+
+    An emoji is one Python character and two UTF-16 units, which is why
+    counting characters puts every entity after the first emoji in the wrong
+    place.
+    """
+    return len(text.encode("utf-16-le")) // 2
+
+
+def _strip_sentinels(text: str) -> str:
+    return text.replace(_SPOILER_OPEN, "").replace(_SPOILER_CLOSE, "")
+
+
+def _mark_spoilers(text: str, mode: str) -> str:
+    """Replace the mode's spoiler markup with sentinels, before parsing."""
+    if mode == "md":
+        parts = text.split("||")
+        if len(parts) < 3:
+            return text
+        out: list[str] = [parts[0]]
+        for index, part in enumerate(parts[1:], start=1):
+            # Odd boundaries open, even boundaries close; a trailing unmatched
+            # `||` is left as typed rather than guessed at.
+            out.append(_SPOILER_OPEN if index % 2 else _SPOILER_CLOSE)
+            out.append(part)
+        if len(parts) % 2 == 0:
+            out[-2] = "||"
+        return "".join(out)
+    if mode == "html":
+        for tag in ("tg-spoiler", "spoiler"):
+            text = text.replace(f"<{tag}>", _SPOILER_OPEN).replace(f"</{tag}>", _SPOILER_CLOSE)
+        return text
+    return text
+
+
+def _extract_spoilers(text: str, entities: list[MessageEntity]) -> tuple[str, list[MessageEntity]]:
+    """Remove sentinels from parsed text, shifting offsets, emitting spoilers."""
+    if _SPOILER_OPEN not in text and _SPOILER_CLOSE not in text:
+        return text, entities
+
+    out: list[str] = []
+    removed = 0  # UTF-16 units removed so far
+    shifts: list[tuple[int, int]] = []  # (original utf-16 offset, units removed before it)
+    open_at: int | None = None
+    spoilers: list[tuple[int, int]] = []  # (start, length) in *final* offsets
+    position = 0
+
+    for char in text:
+        if char in (_SPOILER_OPEN, _SPOILER_CLOSE):
+            removed += 1
+            shifts.append((position, removed))
+            if char == _SPOILER_OPEN:
+                open_at = position - removed + 1
+            elif open_at is not None:
+                spoilers.append((open_at, (position - removed + 1) - open_at))
+                open_at = None
+        else:
+            out.append(char)
+        position += utf16_len(char)
+
+    def shift(offset: int) -> int:
+        total = 0
+        for at, cumulative in shifts:
+            if at < offset:
+                total = cumulative
+        return offset - total
+
+    moved = [
+        MessageEntity(
+            type=entity.type,
+            offset=shift(entity.offset),
+            length=shift(entity.offset + entity.length) - shift(entity.offset),
+            url=entity.url,
+            user_id=entity.user_id,
+            language=entity.language,
+            document_id=entity.document_id,
+            collapsed=entity.collapsed,
+        )
+        for entity in entities
+    ]
+    moved.extend(
+        MessageEntity(type="spoiler", offset=start, length=length)
+        for start, length in spoilers
+        if length > 0
+    )
+    moved.sort(key=lambda e: (e.offset, e.length))
+    return "".join(out), moved
+
+
+def _tl_entity_type(obj: Any) -> str:
+    """`MessageEntityTextUrl` → `text_url`."""
+    name = type(obj).__name__.removeprefix("MessageEntity")
+    result: list[str] = []
+    for index, char in enumerate(name):
+        if char.isupper() and index:
+            result.append("_")
+        result.append(char.lower())
+    return "".join(result)
+
+
+def _from_tl(obj: Any) -> MessageEntity:
+    return MessageEntity(
+        type=_tl_entity_type(obj),
+        offset=int(getattr(obj, "offset", 0)),
+        length=int(getattr(obj, "length", 0)),
+        url=getattr(obj, "url", None),
+        user_id=getattr(getattr(obj, "user_id", None), "user_id", getattr(obj, "user_id", None)),
+        language=getattr(obj, "language", None),
+        document_id=getattr(obj, "document_id", None),
+        collapsed=getattr(obj, "collapsed", None),
+    )
+
+
+def parse_text(text: str, mode: str = "none") -> tuple[str, list[MessageEntity]]:
+    """Turn user input into `(plain text, entities)`.
+
+    Telethon is imported lazily so that `import tlgr.core.text` — which the
+    CLI does to validate `--parse` — stays free of it (§2.2).
+    """
+    if mode not in PARSE_MODES:
+        raise UsageError(f"unknown parse mode {mode!r}: expected md, html or none", field="parse")
+
+    clean = _strip_sentinels(text)
+    if mode == "none":
+        return clean, []
+
+    marked = _mark_spoilers(clean, mode)
+    from telethon.extensions import html as tl_html
+    from telethon.extensions import markdown as tl_markdown
+
+    parser = tl_markdown if mode == "md" else tl_html
+    parsed, raw_entities = parser.parse(marked)
+    entities = [_from_tl(entity) for entity in raw_entities or []]
+    return _extract_spoilers(parsed, entities)
+
+
+def entities_to_json(entities: list[MessageEntity]) -> str:
+    """Entities as the JSON `--entities` accepts back."""
+    import msgspec
+
+    return msgspec.json.encode(entities).decode()
+
+
+def entities_from_json(raw: str) -> list[MessageEntity]:
+    """Decode `--entities JSON`, rejecting anything malformed as USAGE."""
+    import msgspec
+
+    try:
+        loaded = json.loads(raw)
+    except ValueError as exc:
+        raise UsageError(f"--entities is not valid JSON: {exc}", field="entities") from exc
+    if not isinstance(loaded, list):
+        raise UsageError("--entities must be a JSON array of entity objects", field="entities")
+    try:
+        return msgspec.convert(loaded, type=list[MessageEntity])
+    except msgspec.ValidationError as exc:
+        raise UsageError(f"--entities: {exc}", field="entities") from exc