telefi 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- telefi/__init__.py +9 -0
- telefi/__main__.py +127 -0
- telefi/adapters.py +95 -0
- telefi/commands.py +63 -0
- telefi/compat.py +35 -0
- telefi/core.py +291 -0
- telefi/inbound.py +240 -0
- telefi/logs.py +69 -0
- telefi/render.py +80 -0
- telefi/samples.py +67 -0
- telefi/schema.py +61 -0
- telefi/schemas/commands.v1.json +72 -0
- telefi/schemas/envelope.v1.json +178 -0
- telefi/schemas/hub_config.v1.json +193 -0
- telefi/schemas/inbound.v1.json +80 -0
- telefi-0.3.0.dist-info/METADATA +199 -0
- telefi-0.3.0.dist-info/RECORD +21 -0
- telefi-0.3.0.dist-info/WHEEL +5 -0
- telefi-0.3.0.dist-info/entry_points.txt +2 -0
- telefi-0.3.0.dist-info/licenses/LICENSE +21 -0
- telefi-0.3.0.dist-info/top_level.txt +1 -0
telefi/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""TeleFi: a Telegram adapter layer for two-way messaging; channels and reply sinks are adapters.
|
|
2
|
+
|
|
3
|
+
Producers: `from telefi import envelope, publish` (Python) or write an envelope.v1 JSON file
|
|
4
|
+
into the spool's out/ directory (any language). Design: docs/HUB_DESIGN.md. Format: render.py.
|
|
5
|
+
"""
|
|
6
|
+
from telefi.core import drain, envelope, publish, spool_root
|
|
7
|
+
from telefi.render import render_telegram
|
|
8
|
+
|
|
9
|
+
__all__ = ["drain", "envelope", "publish", "render_telegram", "spool_root"]
|
telefi/__main__.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""TeleFi CLI.
|
|
2
|
+
|
|
3
|
+
python -m telefi drain deliver everything in the spool once (the scheduled task)
|
|
4
|
+
python -m telefi serve [--interval 30] drain forever; also long-polls Telegram if inbound.enabled
|
|
5
|
+
python -m telefi preview [type ...] print each sample exactly as the owner would see it (no send)
|
|
6
|
+
python -m telefi samples [--send] route the samples; mock unless --send (then really delivers)
|
|
7
|
+
python -m telefi validate <file.json> check an envelope or inbound event against its schema
|
|
8
|
+
python -m telefi schema <name> print a published schema (envelope.v1 | inbound.v1 | hub_config.v1)
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
import traceback
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from telefi import core, inbound, schema
|
|
21
|
+
from telefi.render import render_plain
|
|
22
|
+
from telefi.samples import samples
|
|
23
|
+
|
|
24
|
+
SERVE_RETRY_S = 5
|
|
25
|
+
SERVE_RETRY_MAX_S = 300
|
|
26
|
+
PRUNE_EVERY_S = 3600
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _preview(names: list[str], buttons: bool) -> int:
|
|
30
|
+
for name, env in samples().items():
|
|
31
|
+
if names and name not in names:
|
|
32
|
+
continue
|
|
33
|
+
print(f"===== {name} (routes to: {', '.join(core.route(env, core.DEFAULT_CONFIG)) or 'nothing'})")
|
|
34
|
+
print(render_plain(env, buttons=buttons))
|
|
35
|
+
print()
|
|
36
|
+
return 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _samples(send: bool) -> int:
|
|
40
|
+
cfg = core.load_config()
|
|
41
|
+
if not send:
|
|
42
|
+
cfg = {**cfg, "mode": "mock"}
|
|
43
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
44
|
+
spool = Path(tmp) / "spool"
|
|
45
|
+
for env in samples().values():
|
|
46
|
+
core.publish(env, spool=spool)
|
|
47
|
+
for rec in core.drain(cfg, spool=spool):
|
|
48
|
+
env = rec["envelope"]
|
|
49
|
+
res = ", ".join(f"{d['channel']}={'ok' if d['ok'] else 'FAILED ' + str(d['error'])}" for d in rec["deliveries"])
|
|
50
|
+
print(f"{env['source']}.{env['kind']} [{env['severity']}] -> {res or 'no route'}")
|
|
51
|
+
print("mode:", "LIVE (really sent)" if send else "mock (nothing sent)")
|
|
52
|
+
return 0
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _serve(interval: float) -> int:
|
|
56
|
+
"""Drain + long-poll forever. A failing round is logged to ~/.telefi/hub.log and retried; it
|
|
57
|
+
never ends the process (the scheduled task would only restart it 5 minutes later)."""
|
|
58
|
+
cfg = core.load_config()
|
|
59
|
+
log = core.HOME / "hub.log"
|
|
60
|
+
core.append_jsonl(log, {"at": core.now_iso(), "event": "serve started", "inbound": cfg.get("inbound", {}).get("enabled")})
|
|
61
|
+
if cfg.get("inbound", {}).get("enabled"):
|
|
62
|
+
menu = inbound.register_menu(cfg)
|
|
63
|
+
core.append_jsonl(log, {"at": core.now_iso(), "event": "command menu", "ok": menu["ok"], "error": menu.get("error")})
|
|
64
|
+
failures, pruned_at = 0, 0.0
|
|
65
|
+
while True:
|
|
66
|
+
try:
|
|
67
|
+
core.drain(cfg)
|
|
68
|
+
if time.monotonic() - pruned_at > PRUNE_EVERY_S:
|
|
69
|
+
core.prune_done(core.spool_root(cfg))
|
|
70
|
+
pruned_at = time.monotonic()
|
|
71
|
+
if cfg.get("inbound", {}).get("enabled"):
|
|
72
|
+
inbound.poll_once(cfg, timeout_s=int(interval))
|
|
73
|
+
else:
|
|
74
|
+
time.sleep(interval)
|
|
75
|
+
failures = 0
|
|
76
|
+
except Exception:
|
|
77
|
+
failures += 1
|
|
78
|
+
wait = min(SERVE_RETRY_S * 2 ** (failures - 1), SERVE_RETRY_MAX_S) # an outage never spins
|
|
79
|
+
core.append_jsonl(log, {"at": core.now_iso(), "event": "round failed", "retry_in_s": wait,
|
|
80
|
+
"error": traceback.format_exc()[-2000:]})
|
|
81
|
+
time.sleep(wait)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def main(argv=None) -> int:
|
|
85
|
+
if sys.stdout is not None: # None under pythonw.exe (no console)
|
|
86
|
+
sys.stdout.reconfigure(encoding="utf-8") # Windows consoles default to cp1252
|
|
87
|
+
ap = argparse.ArgumentParser(prog="python -m telefi")
|
|
88
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
89
|
+
sub.add_parser("drain")
|
|
90
|
+
s = sub.add_parser("serve")
|
|
91
|
+
s.add_argument("--interval", type=float, default=30.0)
|
|
92
|
+
p = sub.add_parser("preview")
|
|
93
|
+
p.add_argument("types", nargs="*")
|
|
94
|
+
p.add_argument("--no-buttons", action="store_true", help="hide action buttons (as sent while inbound is disabled)")
|
|
95
|
+
sm = sub.add_parser("samples")
|
|
96
|
+
sm.add_argument("--send", action="store_true")
|
|
97
|
+
v = sub.add_parser("validate")
|
|
98
|
+
v.add_argument("file")
|
|
99
|
+
sc = sub.add_parser("schema")
|
|
100
|
+
sc.add_argument("name", choices=schema.SCHEMAS)
|
|
101
|
+
a = ap.parse_args(argv)
|
|
102
|
+
|
|
103
|
+
if a.cmd == "drain":
|
|
104
|
+
recs = core.drain(core.load_config())
|
|
105
|
+
failed = [r for r in recs if r.get("rejected") or any(not d["ok"] for d in r["deliveries"])]
|
|
106
|
+
print(f"[hub] delivered {len(recs) - len(failed)}/{len(recs)}", flush=True)
|
|
107
|
+
return 1 if failed else 0
|
|
108
|
+
if a.cmd == "serve":
|
|
109
|
+
return _serve(a.interval)
|
|
110
|
+
if a.cmd == "preview":
|
|
111
|
+
return _preview(a.types, buttons=not a.no_buttons)
|
|
112
|
+
if a.cmd == "samples":
|
|
113
|
+
return _samples(a.send)
|
|
114
|
+
if a.cmd == "validate":
|
|
115
|
+
obj = json.loads(Path(a.file).read_text(encoding="utf-8"))
|
|
116
|
+
name = "inbound.v1" if str(obj.get("schema", "")).endswith(".inbound.v1") else "envelope.v1"
|
|
117
|
+
bad = schema.validate(obj, name)
|
|
118
|
+
print("\n".join(bad) or f"valid {name}")
|
|
119
|
+
return 1 if bad else 0
|
|
120
|
+
if a.cmd == "schema":
|
|
121
|
+
print(json.dumps(schema.load(a.name), indent=2))
|
|
122
|
+
return 0
|
|
123
|
+
return 2
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
sys.exit(main())
|
telefi/adapters.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Outbound channel adapters: `send(envelope, *, buttons) -> {"ok", "error", "message_id"?}`.
|
|
2
|
+
|
|
3
|
+
A new channel (Slack, SMS, email) is one class here plus one `type` in hub_config.v1.json.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import urllib.request
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Callable
|
|
13
|
+
|
|
14
|
+
from telefi import logs
|
|
15
|
+
from telefi.render import render_telegram
|
|
16
|
+
|
|
17
|
+
HTTP_TIMEOUT_S = 10
|
|
18
|
+
TELEGRAM_API = "https://api.telegram.org"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def telegram_call(token: str, method: str, body: dict, *, opener: Callable | None = None,
|
|
22
|
+
timeout: float = HTTP_TIMEOUT_S) -> dict:
|
|
23
|
+
"""One Bot API call. Returns {"ok", "result"?, "error"?}; never raises, never echoes the URL
|
|
24
|
+
(it carries the token)."""
|
|
25
|
+
req = urllib.request.Request(f"{TELEGRAM_API}/bot{token}/{method}", data=json.dumps(body).encode("utf-8"),
|
|
26
|
+
headers={"Content-Type": "application/json"}, method="POST")
|
|
27
|
+
try:
|
|
28
|
+
with (opener or urllib.request.urlopen)(req, timeout=timeout) as resp:
|
|
29
|
+
raw = resp.read() if hasattr(resp, "read") else b"{}"
|
|
30
|
+
data = json.loads(raw or b"{}")
|
|
31
|
+
if resp.status != 200 or not data.get("ok", True):
|
|
32
|
+
return {"ok": False, "error": f"HTTP {resp.status}: {str(data.get('description', ''))[:120]}"}
|
|
33
|
+
return {"ok": True, "result": data.get("result")}
|
|
34
|
+
except Exception as ex:
|
|
35
|
+
return {"ok": False, "error": f"{type(ex).__name__}: {getattr(ex, 'code', '')}".strip()}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class MockAdapter:
|
|
40
|
+
"""Records what would be sent; never touches the network. The default in tests and mode=mock."""
|
|
41
|
+
name: str
|
|
42
|
+
sent: list = field(default_factory=list)
|
|
43
|
+
fail: bool = False
|
|
44
|
+
|
|
45
|
+
def send(self, env: dict, *, buttons: bool) -> dict:
|
|
46
|
+
if self.fail:
|
|
47
|
+
return {"ok": False, "error": "mock failure"}
|
|
48
|
+
self.sent.append({"envelope_id": env["id"], "rendered": render_telegram(env, buttons=buttons)})
|
|
49
|
+
return {"ok": True, "error": None, "message_id": len(self.sent)}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class TelegramAdapter:
|
|
54
|
+
name: str
|
|
55
|
+
token: str | None
|
|
56
|
+
chat_id: str | None
|
|
57
|
+
opener: Callable | None = None
|
|
58
|
+
|
|
59
|
+
def send(self, env: dict, *, buttons: bool) -> dict:
|
|
60
|
+
if not (self.token and self.chat_id):
|
|
61
|
+
return {"ok": False, "error": "telegram not configured (token/chat env missing)"}
|
|
62
|
+
r = render_telegram(env, buttons=buttons)
|
|
63
|
+
body = {"chat_id": self.chat_id, "text": r["text"], "parse_mode": "HTML", "disable_web_page_preview": True}
|
|
64
|
+
if r["reply_markup"]:
|
|
65
|
+
body["reply_markup"] = r["reply_markup"]
|
|
66
|
+
res = telegram_call(self.token, "sendMessage", body, opener=self.opener)
|
|
67
|
+
return {"ok": res["ok"], "error": res.get("error"),
|
|
68
|
+
"message_id": (res.get("result") or {}).get("message_id")}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class FileAdapter:
|
|
73
|
+
name: str
|
|
74
|
+
path: Path
|
|
75
|
+
|
|
76
|
+
def send(self, env: dict, *, buttons: bool) -> dict:
|
|
77
|
+
logs.append(self.path, {"at": logs.now_iso(), "envelope": env}) # rotates like every record
|
|
78
|
+
return {"ok": True, "error": None}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def telegram_creds(cfg: dict, name: str = "telegram") -> tuple[str | None, str | None]:
|
|
82
|
+
c = cfg["channels"].get(name, {})
|
|
83
|
+
return os.environ.get(c.get("token_env", "TELEGRAM_BOT_TOKEN")), os.environ.get(c.get("chat_env", "TELEGRAM_CHAT_ID"))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def build_adapters(cfg: dict, *, opener: Callable | None = None, resolve: Callable = lambda raw: Path(os.path.expanduser(raw))) -> dict:
|
|
87
|
+
out = {}
|
|
88
|
+
for name, c in cfg["channels"].items():
|
|
89
|
+
if cfg["mode"] == "mock" or c["type"] == "mock":
|
|
90
|
+
out[name] = MockAdapter(name)
|
|
91
|
+
elif c["type"] == "telegram":
|
|
92
|
+
out[name] = TelegramAdapter(name, *telegram_creds(cfg, name), opener)
|
|
93
|
+
elif c["type"] == "file":
|
|
94
|
+
out[name] = FileAdapter(name, resolve(c["path"]))
|
|
95
|
+
return out
|
telefi/commands.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""The command registry: one commands.v1 manifest per app -> /help, /list, the '/' menu, routing.
|
|
2
|
+
|
|
3
|
+
/help every app, every command, usage + one line
|
|
4
|
+
/help <app> one app
|
|
5
|
+
/list one line per app: its verbs
|
|
6
|
+
/<app> <verb> x routed to the app's sink (rewritten to legacy_text for an app's old bare form)
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from telefi import schema
|
|
15
|
+
|
|
16
|
+
HUB_MANIFEST: dict = {
|
|
17
|
+
"schema": "telefi.commands.v1", "app": "hub", "summary": "the messaging hub itself", "sink": "hub",
|
|
18
|
+
"commands": [
|
|
19
|
+
{"verb": "help", "usage": "/help [app]", "summary": "every command, or one app's"},
|
|
20
|
+
{"verb": "list", "usage": "/list", "summary": "apps and their commands, one line each"},
|
|
21
|
+
{"verb": "ack", "usage": "/hub ack", "summary": "stop re-alerts for the latest failure"},
|
|
22
|
+
],
|
|
23
|
+
}
|
|
24
|
+
TOP_LEVEL = ("help", "list") # hub verbs that also work as bare commands
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load_manifests(paths: list[str]) -> dict[str, dict]:
|
|
28
|
+
"""app -> manifest, the hub's own first. An invalid manifest raises: a typo must not silently
|
|
29
|
+
drop an app's commands from /help."""
|
|
30
|
+
out = {"hub": HUB_MANIFEST}
|
|
31
|
+
for raw in paths:
|
|
32
|
+
p = Path(os.path.expanduser(raw))
|
|
33
|
+
m = json.loads(p.read_text(encoding="utf-8"))
|
|
34
|
+
bad = schema.validate(m, "commands.v1")
|
|
35
|
+
if bad:
|
|
36
|
+
raise ValueError(f"commands manifest {p}: " + "; ".join(bad))
|
|
37
|
+
out[m["app"]] = m
|
|
38
|
+
return out
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def verb(manifests: dict, app: str, name: str) -> dict | None:
|
|
42
|
+
return next((c for c in manifests.get(app, {}).get("commands", []) if c["verb"] == name), None)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def help_text(manifests: dict, app: str | None = None) -> str:
|
|
46
|
+
apps = [manifests[app]] if app in manifests else list(manifests.values())
|
|
47
|
+
blocks = []
|
|
48
|
+
for m in apps:
|
|
49
|
+
lines = [f"{m['app']}: {m['summary']}"] + [f" {c['usage']} - {c['summary']}" for c in m["commands"]]
|
|
50
|
+
blocks.append("\n".join(lines))
|
|
51
|
+
head = "" if app in manifests else ("Unknown app: " + app + "\n\n" if app else "")
|
|
52
|
+
return head + "\n\n".join(blocks)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def list_text(manifests: dict) -> str:
|
|
56
|
+
return "\n".join(f"/{m['app']} " + " · ".join(c["verb"] for c in m["commands"]) for m in manifests.values())
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def bot_commands(manifests: dict) -> list[dict]:
|
|
60
|
+
"""Telegram's '/' menu (setMyCommands): help, list, then one entry per app."""
|
|
61
|
+
menu = [{"command": "help", "description": "every command, or /help <app>"},
|
|
62
|
+
{"command": "list", "description": "apps and their commands"}]
|
|
63
|
+
return menu + [{"command": m["app"], "description": m["summary"][:60]} for m in manifests.values() if m["app"] != "hub"]
|
telefi/compat.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""A minimal direct-send API for simple scripts.
|
|
2
|
+
|
|
3
|
+
`TelegramChannel(token, chat).send(title, body, level, reply_markup=...)` keeps working, but sends
|
|
4
|
+
through TeleFi's one Bot API client (`adapters.telegram_call`). Prefer publishing envelopes
|
|
5
|
+
(`telefi.envelope` + `telefi.publish`): they get routing, dedup, records and replies.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import html
|
|
10
|
+
from typing import Callable
|
|
11
|
+
|
|
12
|
+
from telefi.adapters import telegram_call
|
|
13
|
+
|
|
14
|
+
TELEGRAM_MAX_BODY = 3800
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TelegramChannel:
|
|
18
|
+
name = "telegram"
|
|
19
|
+
external = True
|
|
20
|
+
|
|
21
|
+
def __init__(self, bot_token: str | None, chat_id: str | None, *, opener: Callable | None = None) -> None:
|
|
22
|
+
self.bot_token, self.chat_id, self._opener = bot_token, chat_id, opener
|
|
23
|
+
|
|
24
|
+
def is_configured(self) -> bool:
|
|
25
|
+
return bool(self.bot_token and self.chat_id)
|
|
26
|
+
|
|
27
|
+
def send(self, title: str, body: str, level: str = "info", *, reply_markup: dict | None = None, **meta) -> bool:
|
|
28
|
+
"""Same message shape as before: bold title, preformatted body, optional inline keyboard."""
|
|
29
|
+
if not self.is_configured():
|
|
30
|
+
return False
|
|
31
|
+
text = f"<b>{html.escape(title)}</b>\n\n<pre>{html.escape(body[:TELEGRAM_MAX_BODY])}</pre>"
|
|
32
|
+
payload = {"chat_id": self.chat_id, "text": text, "parse_mode": "HTML"}
|
|
33
|
+
if reply_markup is not None:
|
|
34
|
+
payload["reply_markup"] = reply_markup
|
|
35
|
+
return telegram_call(self.bot_token, "sendMessage", payload, opener=self._opener)["ok"]
|
telefi/core.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""TeleFi core: config, publish, route, drain, the local record.
|
|
2
|
+
|
|
3
|
+
spool/out/<id>.json published, not yet delivered
|
|
4
|
+
spool/done/<id>.json delivered (or deduplicated / acknowledged); pruned after 7 days
|
|
5
|
+
spool/failed/<id>.json rejected (schema) or undeliverable after retries, with <id>.error.txt
|
|
6
|
+
spool/in/<namespace>/ inbound events for a namespace's queue sink
|
|
7
|
+
spool/state/ acks.json, attempts.json, pending_prompts.json, telegram.json
|
|
8
|
+
outbox.jsonl one line per envelope: the envelope + every channel's delivery result
|
|
9
|
+
inbox.jsonl one line per inbound event and where it went
|
|
10
|
+
|
|
11
|
+
Design: docs/DESIGN.md. Contracts: telefi/schemas/*.json. Format: render.py. Rotation: logs.py.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import uuid
|
|
18
|
+
from datetime import datetime, timedelta, timezone
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from telefi import logs, schema
|
|
22
|
+
from telefi.adapters import build_adapters
|
|
23
|
+
|
|
24
|
+
# TELEFI_* is current; the legacy NOTIFLY_* names are still read for installs that predate the rename.
|
|
25
|
+
HOME = Path(os.environ.get("TELEFI_HOME") or os.environ.get("NOTIFLY_HOME") or Path.home() / ".telefi")
|
|
26
|
+
CONFIG_PATH = HOME / "hub.json"
|
|
27
|
+
OUTBOX_SCAN_LINES = 5000
|
|
28
|
+
DEFAULT_ACK_DAYS = 7
|
|
29
|
+
DONE_RETENTION_DAYS = 7
|
|
30
|
+
MAX_DELIVERY_ATTEMPTS = 10 # with backoff 30 s .. 1 h, about 6 hours of retrying
|
|
31
|
+
RETRY_BASE_S = 30
|
|
32
|
+
RETRY_MAX_S = 3600
|
|
33
|
+
|
|
34
|
+
DEFAULT_CONFIG: dict = {
|
|
35
|
+
"schema": "telefi.hub_config.v1",
|
|
36
|
+
"mode": "live",
|
|
37
|
+
"dedup_window_min": 360,
|
|
38
|
+
"channels": {
|
|
39
|
+
"telegram": {"type": "telegram", "token_env": "TELEGRAM_BOT_TOKEN", "chat_env": "TELEGRAM_CHAT_ID"},
|
|
40
|
+
"file": {"type": "file", "path": "bell.jsonl"},
|
|
41
|
+
},
|
|
42
|
+
"routes": [
|
|
43
|
+
{"match": {"severity": ["error", "critical"]}, "channels": ["telegram", "file"]},
|
|
44
|
+
{"match": {"kind": ["blocker", "review", "result"]}, "channels": ["telegram", "file"]},
|
|
45
|
+
{"match": {}, "channels": ["file"]},
|
|
46
|
+
],
|
|
47
|
+
"inbound": {"enabled": False, "channel": "telegram", "ack_days": DEFAULT_ACK_DAYS,
|
|
48
|
+
"commands": {},
|
|
49
|
+
"sinks": {"*": {"type": "queue"}}},
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
now_iso = logs.now_iso
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def load_config(path: Path = CONFIG_PATH) -> dict:
|
|
57
|
+
cfg = json.loads(path.read_text(encoding="utf-8")) if path.exists() else DEFAULT_CONFIG
|
|
58
|
+
bad = schema.validate(cfg, "hub_config.v1")
|
|
59
|
+
if bad:
|
|
60
|
+
raise ValueError(f"hub config {path}: " + "; ".join(bad))
|
|
61
|
+
logs.configure(cfg)
|
|
62
|
+
return cfg
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def resolve(raw: str) -> Path:
|
|
66
|
+
"""A config path: `~` expands; a relative path is relative to TELEFI_HOME (so a config copied
|
|
67
|
+
into a throwaway home never writes to the real one)."""
|
|
68
|
+
p = Path(os.path.expanduser(raw))
|
|
69
|
+
return p if p.is_absolute() else HOME / p
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def spool_root(cfg: dict) -> Path:
|
|
73
|
+
env = os.environ.get("TELEFI_SPOOL") or os.environ.get("NOTIFLY_HUB_SPOOL")
|
|
74
|
+
return Path(os.path.expanduser(env)) if env else resolve(cfg.get("spool", "spool"))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def append_jsonl(path: Path, rec: dict) -> None:
|
|
78
|
+
logs.append(path, rec)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def read_state(root: Path, name: str) -> dict:
|
|
82
|
+
p = root / "state" / name
|
|
83
|
+
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else {}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def write_state(root: Path, name: str, data: dict) -> None:
|
|
87
|
+
p = root / "state" / name
|
|
88
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
+
tmp = p.with_suffix(".tmp")
|
|
90
|
+
tmp.write_text(json.dumps(data), encoding="utf-8")
|
|
91
|
+
os.replace(tmp, p)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ---------------------------------------------------------------- publish (producer side)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def envelope(*, source: str, kind: str, severity: str, title: str, insight: str = "", body: str = "",
|
|
98
|
+
evidence: dict | None = None, actions: list | None = None, dedup_key: str | None = None,
|
|
99
|
+
not_before: str | None = None, dedup_window_min: float | None = None) -> dict:
|
|
100
|
+
env = {"schema": "telefi.envelope.v1", "id": f"{source}-{uuid.uuid4().hex[:12]}", "source": source,
|
|
101
|
+
"kind": kind, "severity": severity, "title": title[:200], "body": body[:3500], "created_at": now_iso()}
|
|
102
|
+
if insight:
|
|
103
|
+
env["insight"] = insight[:300]
|
|
104
|
+
if evidence:
|
|
105
|
+
env["evidence"] = evidence
|
|
106
|
+
if actions:
|
|
107
|
+
env["actions"] = actions
|
|
108
|
+
if dedup_key:
|
|
109
|
+
env["dedup_key"] = dedup_key[:200]
|
|
110
|
+
if not_before:
|
|
111
|
+
env["not_before"] = not_before
|
|
112
|
+
if dedup_window_min is not None:
|
|
113
|
+
env["dedup_window_min"] = dedup_window_min
|
|
114
|
+
return env
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def publish(env: dict, *, spool: Path | None = None) -> Path:
|
|
118
|
+
"""Validate, then write atomically into <spool>/out/. Raises ValueError on a schema violation."""
|
|
119
|
+
bad = schema.validate(env, "envelope.v1")
|
|
120
|
+
if bad:
|
|
121
|
+
raise ValueError("; ".join(bad))
|
|
122
|
+
out = (spool or spool_root(DEFAULT_CONFIG)) / "out"
|
|
123
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
tmp = out / f".{env['id']}.tmp"
|
|
125
|
+
tmp.write_text(json.dumps(env, ensure_ascii=False), encoding="utf-8")
|
|
126
|
+
dest = out / f"{env['id']}.json"
|
|
127
|
+
os.replace(tmp, dest)
|
|
128
|
+
return dest
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# ---------------------------------------------------------------- route + drain
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def route(env: dict, cfg: dict) -> list[str]:
|
|
135
|
+
for r in cfg["routes"]:
|
|
136
|
+
if all(env.get(k) in v for k, v in r["match"].items()):
|
|
137
|
+
return r["channels"]
|
|
138
|
+
return []
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def outbox_path(root: Path) -> Path:
|
|
142
|
+
return root.parent / "outbox.jsonl"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def outbox_records(root: Path) -> list[dict]:
|
|
146
|
+
return logs.read(outbox_path(root), last=OUTBOX_SCAN_LINES)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def find_envelope(root: Path, envelope_id: str) -> dict | None:
|
|
150
|
+
for rec in reversed(outbox_records(root)):
|
|
151
|
+
if rec.get("envelope", {}).get("id") == envelope_id:
|
|
152
|
+
return rec["envelope"]
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def sent_messages(root: Path) -> dict[str, str]:
|
|
157
|
+
"""Telegram message id -> envelope id, for every hub message that reached Telegram."""
|
|
158
|
+
out = {}
|
|
159
|
+
for rec in outbox_records(root):
|
|
160
|
+
for d in rec.get("deliveries", []):
|
|
161
|
+
if d.get("channel") == "telegram" and d.get("message_id") is not None:
|
|
162
|
+
out[str(d["message_id"])] = rec["envelope"]["id"]
|
|
163
|
+
return out
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def conversation_path(root: Path) -> Path:
|
|
167
|
+
return root.parent / "conversation.jsonl"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def ack(root: Path, dedup_key: str) -> None:
|
|
171
|
+
acks = read_state(root, "acks.json")
|
|
172
|
+
acks[dedup_key] = now_iso()
|
|
173
|
+
write_state(root, "acks.json", acks)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _acked(root: Path, days: float) -> set[str]:
|
|
177
|
+
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
|
178
|
+
return {k for k, at in read_state(root, "acks.json").items() if at >= cutoff}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _last_delivered(root: Path) -> dict[str, str]:
|
|
182
|
+
"""dedup_key -> when it was last delivered to at least one channel."""
|
|
183
|
+
last: dict[str, str] = {}
|
|
184
|
+
for r in outbox_records(root):
|
|
185
|
+
key = r.get("envelope", {}).get("dedup_key")
|
|
186
|
+
if key and any(d.get("ok") for d in r.get("deliveries", [])):
|
|
187
|
+
last[key] = max(last.get(key, ""), r.get("at", ""))
|
|
188
|
+
return last
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _ago(minutes: float) -> str:
|
|
192
|
+
return (datetime.now(timezone.utc) - timedelta(minutes=minutes)).isoformat(timespec="seconds")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _when(raw: str) -> datetime | None:
|
|
196
|
+
"""An ISO time in any offset -> aware UTC datetime (a naive one is read as UTC); None if invalid."""
|
|
197
|
+
try:
|
|
198
|
+
t = datetime.fromisoformat(raw)
|
|
199
|
+
except (TypeError, ValueError):
|
|
200
|
+
return None
|
|
201
|
+
return (t if t.tzinfo else t.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _move(p: Path, folder: Path) -> None:
|
|
205
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
206
|
+
os.replace(p, folder / p.name)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _quarantine(p: Path, root: Path, reasons: list[str]) -> None:
|
|
210
|
+
_move(p, root / "failed")
|
|
211
|
+
(root / "failed" / f"{p.stem}.error.txt").write_text("\n".join(reasons), encoding="utf-8")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def prune_done(root: Path, *, days: float = DONE_RETENTION_DAYS) -> int:
|
|
215
|
+
"""Delete delivered envelopes older than `days` (their record stays in outbox.jsonl)."""
|
|
216
|
+
cutoff = datetime.now(timezone.utc).timestamp() - days * 86400
|
|
217
|
+
n = 0
|
|
218
|
+
for p in (root / "done").glob("*.json"):
|
|
219
|
+
if p.stat().st_mtime < cutoff:
|
|
220
|
+
p.unlink(missing_ok=True)
|
|
221
|
+
n += 1
|
|
222
|
+
return n
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def drain(cfg: dict, *, adapters: dict | None = None, spool: Path | None = None) -> list[dict]:
|
|
226
|
+
"""Deliver every due envelope in <spool>/out. Returns the outbox records written.
|
|
227
|
+
|
|
228
|
+
A delivery that reaches no channel is retried with exponential backoff (state/attempts.json)
|
|
229
|
+
and quarantined in failed/ after MAX_DELIVERY_ATTEMPTS, so a Telegram outage never loses a
|
|
230
|
+
message. Every attempt is recorded in outbox.jsonl."""
|
|
231
|
+
root = spool or spool_root(cfg)
|
|
232
|
+
adapters = adapters if adapters is not None else build_adapters(cfg, resolve=resolve)
|
|
233
|
+
inbound = cfg.get("inbound", {})
|
|
234
|
+
buttons = bool(inbound.get("enabled"))
|
|
235
|
+
last = _last_delivered(root)
|
|
236
|
+
acked = _acked(root, inbound.get("ack_days", DEFAULT_ACK_DAYS))
|
|
237
|
+
attempts = read_state(root, "attempts.json")
|
|
238
|
+
now = datetime.now(timezone.utc)
|
|
239
|
+
records = []
|
|
240
|
+
for p in sorted((root / "out").glob("*.json")):
|
|
241
|
+
try:
|
|
242
|
+
env = json.loads(p.read_text(encoding="utf-8"))
|
|
243
|
+
bad = schema.validate(env, "envelope.v1")
|
|
244
|
+
except FileNotFoundError: # another drainer took it
|
|
245
|
+
continue
|
|
246
|
+
except ValueError as ex:
|
|
247
|
+
env, bad = {"id": p.stem}, [f"unreadable JSON: {ex}"]
|
|
248
|
+
if not bad and env.get("not_before") and _when(env["not_before"]) is None:
|
|
249
|
+
bad = [f"$.not_before: {env['not_before']!r} is not an ISO time"]
|
|
250
|
+
if not bad and env.get("not_before") and _when(env["not_before"]) > now:
|
|
251
|
+
continue # held: a grace period is still running
|
|
252
|
+
retry = attempts.get(p.stem)
|
|
253
|
+
if retry and (_when(retry["next"]) or now) > now:
|
|
254
|
+
continue # backing off after a failed delivery
|
|
255
|
+
key = env.get("dedup_key")
|
|
256
|
+
window = env.get("dedup_window_min", cfg.get("dedup_window_min", 360))
|
|
257
|
+
if bad:
|
|
258
|
+
_quarantine(p, root, bad)
|
|
259
|
+
rec = {"at": now_iso(), "envelope": env, "rejected": bad, "deliveries": []}
|
|
260
|
+
elif key in acked:
|
|
261
|
+
_move(p, root / "done")
|
|
262
|
+
rec = {"at": now_iso(), "envelope": env, "acknowledged": True, "deliveries": []}
|
|
263
|
+
elif key and last.get(key, "") >= _ago(window):
|
|
264
|
+
_move(p, root / "done")
|
|
265
|
+
rec = {"at": now_iso(), "envelope": env, "deduplicated": True, "deliveries": []}
|
|
266
|
+
else:
|
|
267
|
+
deliveries = []
|
|
268
|
+
for name in route(env, cfg):
|
|
269
|
+
a = adapters.get(name)
|
|
270
|
+
res = a.send(env, buttons=buttons) if a else {"ok": False, "error": f"no channel named {name}"}
|
|
271
|
+
deliveries.append({"channel": name, **res})
|
|
272
|
+
rec = {"at": now_iso(), "envelope": env, "deliveries": deliveries}
|
|
273
|
+
if not deliveries or any(d["ok"] for d in deliveries):
|
|
274
|
+
_move(p, root / "done")
|
|
275
|
+
attempts.pop(p.stem, None)
|
|
276
|
+
if key and deliveries:
|
|
277
|
+
last[key] = rec["at"]
|
|
278
|
+
else:
|
|
279
|
+
n = (retry or {}).get("n", 0) + 1
|
|
280
|
+
if n >= MAX_DELIVERY_ATTEMPTS:
|
|
281
|
+
_quarantine(p, root, [f"{n} delivery attempts failed: {d['channel']}: {d['error']}" for d in deliveries])
|
|
282
|
+
attempts.pop(p.stem, None)
|
|
283
|
+
rec["gave_up"] = n
|
|
284
|
+
else:
|
|
285
|
+
wait = min(RETRY_BASE_S * 2 ** (n - 1), RETRY_MAX_S)
|
|
286
|
+
attempts[p.stem] = {"n": n, "next": (now + timedelta(seconds=wait)).isoformat(timespec="seconds")}
|
|
287
|
+
rec["retry_in_s"] = wait
|
|
288
|
+
append_jsonl(outbox_path(root), rec)
|
|
289
|
+
records.append(rec)
|
|
290
|
+
write_state(root, "attempts.json", attempts)
|
|
291
|
+
return records
|