android-watcher 1.0.0__py3-none-any.whl

android_watcher/notify/slack.py

@@ -0,0 +1,124 @@
+"""Slack notifier: bot token (chat.postMessage + threaded file upload)."""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from android_watcher.config import Config
+from android_watcher.models import Digest, DigestGroup
+from android_watcher.notify.base import NOTIFIERS, NotifyError
+from android_watcher.notify.html import render_html
+from android_watcher.notify.render import render_slack
+
+logger = logging.getLogger(__name__)
+
+
+def _member_ids(groups: list[DigestGroup]) -> set[int]:
+	return {m.id for g in groups for m in g.members if m.id is not None}
+
+
+def _targets(raw: str) -> list[str]:
+	return [t.strip() for t in raw.split(",") if t.strip()]
+
+
+@NOTIFIERS.register("slack")
+class SlackNotifier:
+	name = "slack"
+
+	def send(self, digest: Digest, config: Config) -> set[int]:
+		sl = config.slack
+		if not (sl.bot_token and sl.channel):
+			raise NotifyError("slack enabled but bot_token + channel not configured")
+		payload = render_slack(digest, thread_page=True)
+		delivered: set[int] = set()
+		for target in _targets(sl.channel):
+			logger.info("slack: posting message to %s then uploading digest page", target)
+			channel_id, ts = self._send_bot(sl.bot_token, target, payload)
+			logger.info(
+				"slack: message posted (channel_id=%s ts=%s); uploading HTML page",
+				channel_id,
+				ts,
+			)
+			uploaded = self._deliver_page(sl.bot_token, channel_id, ts, digest)
+			if uploaded:
+				# Full HTML reached the channel: all groups (incl. carried) are visible.
+				delivered |= _member_ids(digest.groups)
+			else:
+				# Upload failed entirely; only the capped message reached Slack.
+				delivered |= _member_ids(digest.message_groups())
+		return delivered
+
+	def _send_bot(self, bot_token: str, channel: str, payload: dict) -> tuple[str, str]:
+		try:
+			resp = httpx.post(
+				"https://slack.com/api/chat.postMessage",
+				headers={"Authorization": f"Bearer {bot_token}"},
+				json={"channel": channel, **payload},
+				timeout=30.0,
+			)
+			resp.raise_for_status()
+		except (httpx.HTTPStatusError, httpx.RequestError) as exc:
+			raise NotifyError(f"slack send failed: {type(exc).__name__}") from exc
+		body = resp.json()
+		if not body.get("ok"):
+			raise NotifyError(f"slack chat.postMessage failed: {body.get('error')}")
+		return body["channel"], body["ts"]
+
+	def _deliver_page(self, bot_token: str, channel_id: str, ts: str, digest: Digest) -> bool:
+		"""Deliver the full-digest HTML page. Try a threaded reply first; if that
+		fails, fall back to a standalone message in the channel so the page still
+		lands. Non-fatal: the main message is already delivered, so a total failure
+		logs and returns False (carried changes then retry next run)."""
+		data = render_html(digest).encode("utf-8")
+		if self._upload_page(bot_token, channel_id, data, thread_ts=ts):
+			return True
+		logger.warning(
+			"slack: threaded page upload failed; retrying as a standalone channel message"
+		)
+		if self._upload_page(bot_token, channel_id, data, thread_ts=None):
+			return True
+		logger.error("slack: digest page upload failed (thread and standalone); page not delivered")
+		return False
+
+	def _upload_page(
+		self, bot_token: str, channel_id: str, data: bytes, *, thread_ts: str | None
+	) -> bool:
+		"""One external-upload sequence (getUploadURLExternal -> PUT -> complete).
+		Posts into the thread when thread_ts is set, else as a new channel message."""
+		where = f"thread {thread_ts}" if thread_ts else "channel"
+		try:
+			headers = {"Authorization": f"Bearer {bot_token}"}
+			r1 = httpx.get(
+				"https://slack.com/api/files.getUploadURLExternal",
+				headers=headers,
+				params={"filename": "digest.html", "length": len(data)},
+				timeout=30.0,
+			)
+			r1.raise_for_status()
+			b1 = r1.json()
+			if not b1.get("ok"):
+				raise NotifyError(f"getUploadURLExternal: {b1.get('error')}")
+			httpx.post(b1["upload_url"], content=data, timeout=30.0).raise_for_status()
+			body: dict = {
+				"files": [{"id": b1["file_id"], "title": "Android Watcher Digest"}],
+				"channel_id": channel_id,
+			}
+			if thread_ts:
+				body["thread_ts"] = thread_ts
+			r2 = httpx.post(
+				"https://slack.com/api/files.completeUploadExternal",
+				headers=headers,
+				json=body,
+				timeout=30.0,
+			)
+			r2.raise_for_status()
+			b2 = r2.json()
+			if not b2.get("ok"):
+				raise NotifyError(f"completeUploadExternal: {b2.get('error')}")
+			logger.info("slack: digest page uploaded to %s", where)
+			return True
+		except (httpx.HTTPStatusError, httpx.RequestError, NotifyError, KeyError) as exc:
+			logger.warning("slack: digest page upload to %s failed: %s", where, exc)
+			return False

android_watcher/notify/telegram.py

@@ -0,0 +1,46 @@
+"""Telegram notifier — delivers digests via the Telegram Bot API."""
+
+from __future__ import annotations
+
+import httpx
+
+from android_watcher.config import Config
+from android_watcher.models import Digest, NotifyError
+from android_watcher.notify.base import NOTIFIERS
+from android_watcher.notify.render import render_telegram
+
+
+def _chat_ids(raw: str) -> list[str]:
+	"""Split a comma-separated list of chat ids into individual targets."""
+	return [c.strip() for c in raw.split(",") if c.strip()]
+
+
+@NOTIFIERS.register("telegram")
+class TelegramNotifier:
+	name = "telegram"
+
+	def send(self, digest: Digest, config: Config) -> set[int]:
+		token = config.telegram.bot_token
+		text = render_telegram(digest)
+		url = f"https://api.telegram.org/bot{token}/sendMessage"
+		for chat_id in _chat_ids(config.telegram.chat_id):
+			try:
+				resp = httpx.post(
+					url,
+					json={
+						"chat_id": chat_id,
+						"text": text,
+						"parse_mode": "HTML",
+						"disable_web_page_preview": True,
+					},
+					timeout=30.0,
+				)
+				resp.raise_for_status()
+			except httpx.HTTPStatusError as exc:
+				raise NotifyError(
+					f"telegram send failed: {exc.response.status_code} {exc.response.text[:200]}"
+				) from exc
+			except httpx.RequestError as exc:
+				detail = exc.args[0] if exc.args else "request error"
+				raise NotifyError(f"telegram send failed: {type(exc).__name__}: {detail}") from exc
+		return {m.id for g in digest.message_groups() for m in g.members if m.id is not None}

android_watcher/rank.py

@@ -0,0 +1,84 @@
+"""Rank a list of Changes into a Digest.
+
+Scoring:
+  base = source.default_weight if nonzero, else CATEGORY_WEIGHTS[source.category]
+         (falls back to DEFAULT_CATEGORY_WEIGHT for unknown categories or missing sources)
+  score = base + config.sort override (source_id key takes precedence over category key)
+
+Tie-break: detected_at DESC.
+
+Groups are sorted globally by (score, members[0].detected_at) DESC; max_items caps
+how many appear in the on-channel message vs. carried over.
+"""
+
+from __future__ import annotations
+
+from .config import Config
+from .group import group_changes
+from .models import Change, Digest, DigestGroup, Source
+
+CATEGORY_WEIGHTS: dict[str, int] = {
+	"platform-release": 100,
+	"api-reference": 80,
+	"tooling": 70,
+	"guides": 50,
+	"dev-blog": 40,
+	"design": 30,
+	"news": 20,
+}
+DEFAULT_CATEGORY_WEIGHT = 10
+
+# Display order for category subheadings. Anything not listed falls to "Other".
+CATEGORY_ORDER: list[tuple[str, str]] = [
+	("platform-release", "Platform & Releases"),
+	("api-reference", "API Reference"),
+	("tooling", "Developer Tooling"),
+	("guides", "Guides & Blog"),
+	("dev-blog", "Guides & Blog"),
+	("design", "Design"),
+	("news", "News"),
+]
+
+
+def _score(change: Change, source: Source | None, config: Config) -> int:
+	# Unknown source_id (not in sources map) => DEFAULT_CATEGORY_WEIGHT, no override.
+	if source is None:
+		return DEFAULT_CATEGORY_WEIGHT
+
+	if source.default_weight:
+		base = source.default_weight
+	else:
+		base = CATEGORY_WEIGHTS.get(source.category, DEFAULT_CATEGORY_WEIGHT)
+
+	# source_id override takes precedence over category override
+	override = config.sort.get(change.source_id)
+	if override is None:
+		override = config.sort.get(source.category)
+	return base + (override or 0)
+
+
+def rank(changes: list[Change], sources: dict[str, Source], config: Config) -> Digest:
+	substantive = [c for c in changes if c.verdict == "substantive"]
+	groups = group_changes(substantive, sources, config)
+	groups.sort(key=lambda g: (g.score, g.members[0].detected_at), reverse=True)
+	return Digest(groups=groups, max_items=config.digest.max_items)
+
+
+def by_category(groups: list[DigestGroup]) -> list[tuple[str, str, list[DigestGroup]]]:
+	"""Bucket groups under category labels in CATEGORY_ORDER; preserve rank order."""
+	label_for = dict(CATEGORY_ORDER)
+	order = [cid for cid, _ in CATEGORY_ORDER]
+	buckets: dict[str, list[DigestGroup]] = {}
+	for g in groups:
+		buckets.setdefault(g.category, []).append(g)
+	out: list[tuple[str, str, list[DigestGroup]]] = []
+	seen: set[str] = set()
+	for cid in order:
+		if cid in buckets and cid not in seen:
+			out.append((cid, label_for[cid], buckets[cid]))
+			seen.add(cid)
+	# Unknown categories last, under "Other".
+	other = [g for cid, gs in buckets.items() if cid not in label_for for g in gs]
+	if other:
+		out.append(("other", "Other", other))
+	return out

android_watcher/registry.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class Registry(Generic[T]):
+	"""A name->implementation registry shared by detectors, triagers, notifiers.
+
+	Registries store CLASSES, not instances. ``@reg.register("feed")`` decorates
+	a class; ``reg.get("feed")`` returns the class; the caller instantiates with
+	no args (``DETECTORS.get("feed")()``).
+	"""
+
+	def __init__(self, kind: str) -> None:
+		self.kind = kind
+		self._items: dict[str, type[T]] = {}
+
+	def register(self, name: str) -> Callable[[type[T]], type[T]]:
+		def decorator(impl: type[T]) -> type[T]:
+			if name in self._items:
+				raise ValueError(f"{self.kind} {name!r} is already registered")
+			self._items[name] = impl
+			return impl
+
+		return decorator
+
+	def get(self, name: str) -> type[T]:
+		try:
+			return self._items[name]
+		except KeyError:
+			avail = ", ".join(self.available()) or "(none registered)"
+			raise KeyError(f"{self.kind} {name!r} not found; available: {avail}") from None
+
+	def available(self) -> list[str]:
+		return sorted(self._items)

android_watcher/run.py

@@ -0,0 +1,283 @@
+"""The run_once pipeline: source resolution, detection, and orchestration.
+
+The first section covers source selection and the isolated async detector
+driver; the rest is the full ``run_once`` orchestration.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import logging.handlers
+import time
+from datetime import datetime
+from pathlib import Path
+
+from android_watcher import __version__
+from android_watcher.catalog import load_catalog
+from android_watcher.config import Config, data_path, db_path, log_path
+from android_watcher.detect.base import DETECTORS
+from android_watcher.fetch import USER_AGENT, Fetcher
+from android_watcher.lock import run_lock
+from android_watcher.models import INTERVAL_DELTA, UTC, Change, Digest, Source
+from android_watcher.notify.base import NOTIFIERS, NotifyError
+from android_watcher.rank import rank
+from android_watcher.seed import apply_seed_if_empty
+from android_watcher.store import Store
+from android_watcher.triage.base import TRIAGERS, TriageResult
+from android_watcher.triage.claude_cli import MAX_TRIAGE_BATCH
+
+log = logging.getLogger("android_watcher.run")
+
+
+def configure_file_logging() -> str:
+	"""Attach a rotating file handler to the package logger; return the log path.
+
+	Idempotent: repeated calls do not stack handlers. Called by the CLI so both
+	manual and scheduled runs append to the same log; not called from tests,
+	which exercise run_once() directly and must not write to the user log dir.
+	"""
+	path = log_path()
+	Path(path).parent.mkdir(parents=True, exist_ok=True)
+	pkg = logging.getLogger("android_watcher")
+	pkg.setLevel(logging.INFO)
+	if not any(isinstance(h, logging.handlers.RotatingFileHandler) for h in pkg.handlers):
+		handler = logging.handlers.RotatingFileHandler(
+			path, maxBytes=1_000_000, backupCount=3, encoding="utf-8"
+		)
+		handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
+		pkg.addHandler(handler)
+	return path
+
+
+def resolve_sources(config: Config) -> list[Source]:
+	"""Resolve the watched sources from catalog + config.
+
+	Start from the catalog entries that are enabled by their own flag. If
+	``enabled_source_ids`` is non-empty, keep only those ids (an override). An
+	empty/absent selection means "use the catalog enabled flags", never "watch
+	nothing". Custom sources are always watched and override a catalog source on
+	id collision.
+	"""
+	watched = [s for s in load_catalog() if s.enabled]
+	if config.enabled_source_ids:
+		watched = [s for s in watched if s.id in config.enabled_source_ids]
+	by_id: dict[str, Source] = {s.id: s for s in watched}
+	for s in config.custom_sources:  # custom always included, overrides catalog
+		by_id[s.id] = s
+	return list(by_id.values())
+
+
+async def _detect_all(sources: list[Source], store: Store, fetcher: Fetcher) -> list[Change]:
+	"""Run each source's detector, isolating per-source failures.
+
+	One source raising never aborts the run: it is logged and skipped. Returns
+	the flattened list of changes across all sources that succeeded.
+	"""
+	changes: list[Change] = []
+	total = len(sources)
+	for i, source in enumerate(sources, 1):
+		log.info("detecting [%d/%d] %s (%s)", i, total, source.id, source.detector)
+		t0 = time.monotonic()
+		try:
+			detector = DETECTORS.get(source.detector)()
+			found = await detector.detect(source, store, fetcher)
+			changes.extend(found)
+			log.info("  %s: %d change(s) in %.1fs", source.id, len(found), time.monotonic() - t0)
+		except Exception:  # isolation: one source must not abort the run
+			log.exception("source %s failed after %.1fs", source.id, time.monotonic() - t0)
+	return changes
+
+
+def _enabled_channels(config: Config) -> set[str]:
+	channels: set[str] = set()
+	if config.email.enabled:
+		channels.add("email")
+	if config.slack.enabled:
+		channels.add("slack")
+	if config.telegram.enabled:
+		channels.add("telegram")
+	return channels
+
+
+def _source_index(config: Config) -> dict[str, Source]:
+	sources = {s.id: s for s in load_catalog()}
+	sources.update({s.id: s for s in config.custom_sources})  # custom wins on collision
+	return sources
+
+
+def _catch_up_due(store: Store, config: Config, force: bool) -> bool:
+	"""Whether this run should cover the current cycle.
+
+	Due when forced, when nothing has ever run, or when the last successful run
+	is at least one schedule interval in the past. cron has no fixed delta, so
+	it is always due; the native scheduler enforces cron timing.
+	"""
+	last = store.last_successful_run()
+	if force or last is None:
+		return True
+	delta = INTERVAL_DELTA.get(config.schedule.interval)
+	if delta is None:  # cron => always due
+		return True
+	return datetime.now(UTC) - last >= delta
+
+
+async def _run_async(sources: list[Source], store: Store, fetcher: Fetcher) -> list[Change]:
+	try:
+		return await _detect_all(sources, store, fetcher)
+	finally:
+		await fetcher.close()
+
+
+def _triage_batched(triager, changes, ai_config, batch_size=MAX_TRIAGE_BATCH):
+	all_changes: list = []
+	tldr: str | None = None
+	unavailable: str | None = None
+	for i in range(0, len(changes), batch_size):
+		batch = changes[i : i + batch_size]
+		res = triager.triage(batch, ai_config)
+		all_changes.extend(res.changes)
+		if tldr is None:
+			tldr = res.tldr
+		if unavailable is None:
+			unavailable = res.unavailable
+	return TriageResult(changes=all_changes, tldr=tldr, unavailable=unavailable)
+
+
+def _build_ledger_digest(
+	store: Store,
+	config: Config,
+	channels: set[str],
+	tldr: str | None,
+	unavailable: str | None,
+) -> Digest:
+	digest = rank(store.changes_for_digest(channels), _source_index(config), config)
+	digest.tldr = tldr  # intentionally unrendered — no TL;DR preamble in any channel output
+	digest.ai_unavailable = unavailable
+	# Scan-scope footer: how many sources are watched and how many pages are under
+	# baseline. Shown in every delivered digest so the reader sees the coverage.
+	digest.sources_scanned = len(resolve_sources(config))
+	digest.pages_watched = store.snapshot_count()
+	return digest
+
+
+def _deliver_into(store: Store, digest: Digest, config: Config, channels: set[str]) -> None:
+	"""Open a digest, deliver per channel, record exactly the change ids each
+	channel conveyed, then commit. A channel that fails is left for next run."""
+	digest_id = store.open_digest()
+	for channel in channels:
+		try:
+			delivered = NOTIFIERS.get(channel)().send(digest, config)
+		except NotifyError:
+			log.exception("channel %s delivery failed; leaving for next run", channel)
+			continue
+		for change_id in delivered:
+			store.record_delivery(change_id, channel)
+	# Supersede older undelivered rows for every delivered (source_id, url).
+	for g in digest.groups:
+		for m in g.members:
+			if m.id is not None:
+				store.supersede_older(m.source_id, m.url, m.id)
+	store.commit_digest(digest_id)
+
+
+def run_once(config: Config, *, force: bool = False, dry_run: bool = False) -> Digest:
+	"""Run the full detection-to-delivery pipeline once.
+
+	The digest is built from the ledger (substantive changes not yet delivered
+	to every enabled channel), never from this-run detections, so a prior run's
+	undelivered backlog is always retried. ``dry_run`` previews that standing
+	backlog without detecting, persisting, superseding, sending, or marking the
+	run successful.
+	"""
+	store = Store(db_path())
+	store.migrate()
+	# Fresh DB: import the shipped baseline seed so the first run diffs against it
+	# instead of crawling every page. No-op once any snapshot exists, or if no
+	# seed is bundled (the detectors then baseline fetch-free on first sight).
+	seeded = apply_seed_if_empty(store)
+	if seeded:
+		log.info("imported baseline seed dated %s (%d snapshots)", seeded, store.snapshot_count())
+	channels = _enabled_channels(config)
+	with run_lock(data_path()):
+		# Zero channels: nothing can be delivered. Do no reconcile/detect/triage/
+		# send, but still advance the catch-up window on the live path so
+		# re-enabling a channel later is not treated as "missed every cycle".
+		if not channels:
+			if not dry_run:
+				store.mark_successful_run(datetime.now(UTC))
+			return Digest(groups=[])
+
+		# dry_run: render from the existing ledger only; mutate nothing.
+		if dry_run:
+			return _build_ledger_digest(store, config, channels, None, None)
+
+		# Reconcile a crashed run: re-deliver the still-owed changes (per-channel
+		# idempotent, so no resend), then commit the stale inflight digest.
+		# Mirror the live-path empty-digest gate: only call _deliver_into when
+		# there is something to send (or config.digest.empty == "send"), but
+		# ALWAYS commit the inflight row so it does not recur on the next run.
+		inflight = store.inflight_digest()
+		if inflight is not None:
+			recon = _build_ledger_digest(store, config, channels, None, None)
+			recon_send_empty = recon.is_empty and config.digest.empty == "send"
+			if not recon.is_empty or recon_send_empty:
+				_deliver_into(store, recon, config, channels)
+			store.commit_digest(inflight)
+
+		# Catch-up gate: skip when the last successful run already covers this
+		# cycle and we are not forced.
+		if not _catch_up_due(store, config, force):
+			return Digest(groups=[])
+
+		sources = resolve_sources(config)
+		log.info(
+			"run starting: %d source(s), channels=%s, force=%s",
+			len(sources),
+			",".join(sorted(channels)) or "none",
+			force,
+		)
+		fetcher = Fetcher(store, user_agent=USER_AGENT.format(version=__version__))
+		t_detect = time.monotonic()
+		changes = asyncio.run(_run_async(sources, store, fetcher))
+		log.info("detection phase: %.1fs (%d changes)", time.monotonic() - t_detect, len(changes))
+		for change in changes:
+			change.id = store.record_change(change)  # idempotent on (source,url,hash)
+
+		# Triage is WRITE-ONCE: only rows whose verdict is still NULL. Re-detected
+		# rows already carry a final verdict and must not be re-triaged.
+		untriaged = [c for c in changes if c.verdict is None]
+		mode = config.ai.mode if config.ai.mode != "off" else "noop"
+		t_triage = time.monotonic()
+		result = _triage_batched(TRIAGERS.get(mode)(), untriaged, config.ai)
+		log.info("triage phase: %.1fs (%d triaged)", time.monotonic() - t_triage, len(untriaged))
+		for change in result.changes:
+			if change.id is not None and change.verdict is not None:
+				store.set_verdict(
+					change.id,
+					change.verdict,
+					change.description,
+					change.group_key,
+					change.group_summary,
+					change.group_title,
+				)
+
+		# Digest comes from the ledger (undelivered backlog), not this-run changes.
+		digest = _build_ledger_digest(store, config, channels, result.tldr, result.unavailable)
+
+		# Empty "nothing notable" digests go out at most once per catch-up window:
+		# only here, where the gate was due and mark_successful_run (below) will
+		# advance the window so the next empty run is not due. Non-empty digests
+		# are idempotent via the delivery ledger regardless.
+		send_empty = digest.is_empty and config.digest.empty == "send"
+		if not digest.is_empty or send_empty:
+			_deliver_into(store, digest, config, channels)
+
+		store.mark_successful_run(datetime.now(UTC))
+		log.info(
+			"run finished: %d detected, %d in digest, delivered to %s",
+			len(changes),
+			digest.change_count(),
+			",".join(sorted(channels)) or "none",
+		)
+		return digest