android-watcher 1.0.0__py3-none-any.whl

android_watcher/cli.py

@@ -0,0 +1,161 @@
+"""android-watcher CLI: argparse router."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .catalog import load_catalog
+from .config import load_config
+from .doctor import run_doctor
+from .models import AlreadyRunning, ConfigError
+from .notify.render import render_email
+from .run import configure_file_logging, run_once
+from .schedule import install_schedule, remove_schedule, schedule_status
+from .tui.app import AndroidWatcher
+from .tui.configio import load_or_default
+
+
+def _cmd_catalog(args: argparse.Namespace) -> int:
+	for source in load_catalog():
+		state = "on " if source.enabled else "off"
+		print(f"[{state}] {source.id:<28} {source.detector:<16} {source.category}")
+	return 0
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+	try:
+		config = load_config(args.config)
+	except ConfigError as exc:
+		print(f"android-watcher: configuration error: {exc}", file=sys.stderr)
+		return 1
+	log_file = configure_file_logging()
+	print(f"android-watcher: running… (progress logged to {log_file})", file=sys.stderr)
+	try:
+		digest = run_once(config, force=args.force)
+	except AlreadyRunning:
+		print("android-watcher: another run is in progress; exiting.")
+		return 0
+	print(f"android-watcher: {digest.change_count()} change(s) delivered.")
+	return 0
+
+
+def _cmd_test(args: argparse.Namespace) -> int:
+	try:
+		config = load_config(args.config)
+	except ConfigError as exc:
+		print(f"android-watcher: configuration error: {exc}", file=sys.stderr)
+		return 1
+	configure_file_logging()
+	checks = run_doctor(config)
+	failed = [
+		c for c in checks if not c.ok and (c.name == "ai-backend" or c.name.startswith("channel"))
+	]
+	digest = run_once(config, dry_run=True)
+	_, plaintext = render_email(digest)
+	print(plaintext)
+	if failed:
+		for c in failed:
+			print(f"FAIL {c.name}: {c.detail}")
+		return 1
+	return 0
+
+
+def _cmd_doctor(args: argparse.Namespace) -> int:
+	try:
+		config = load_config(args.config)
+	except ConfigError as exc:
+		print(f"android-watcher: configuration error: {exc}", file=sys.stderr)
+		return 1
+	print("Running checks (verifying the sitemap may take up to ~30s)…", file=sys.stderr)
+	checks = run_doctor(config)
+	any_failed = False
+	for c in checks:
+		status = "OK  " if c.ok else "FAIL"
+		if not c.ok:
+			any_failed = True
+		print(f"{status} {c.name}: {c.detail}")
+	return 1 if any_failed else 0
+
+
+def _cmd_schedule(args: argparse.Namespace) -> int:
+	if args.action == "install":
+		config = load_config(args.config) if args.config else load_or_default()[0]
+		install_schedule(config)
+		return 0
+	if args.action == "remove":
+		remove_schedule()
+		return 0
+	# status
+	check = schedule_status()
+	status = "OK  " if check.ok else "FAIL"
+	print(f"{status} {check.name}: {check.detail}")
+	return 0 if check.ok else 1
+
+
+def _cmd_tui(args: argparse.Namespace) -> int:
+	config, existed = load_or_default()
+	# Configuration is incomplete until a delivery channel is set, so run the
+	# wizard from the start in that case too — not only when the file is absent.
+	# (Short-circuits on a fresh install before touching the config fields.)
+	first_run = (not existed) or not (
+		config.email.enabled or config.slack.enabled or config.telegram.enabled
+	)
+	result = AndroidWatcher(config=config, first_run=first_run).run()
+	if isinstance(result, str):
+		print(result)
+	return 0
+
+
+def _build_parser() -> argparse.ArgumentParser:
+	parser = argparse.ArgumentParser(
+		prog="android-watcher",
+		description="Watch official Google Android sites and deliver a ranked digest.",
+	)
+	parser.add_argument("--version", action="version", version=f"android-watcher {__version__}")
+	parser.add_argument(
+		"--config",
+		default=None,
+		metavar="PATH",
+		help="Path to config file (default: platform config dir).",
+	)
+	parser.set_defaults(func=_cmd_tui)
+
+	sub = parser.add_subparsers(dest="command")
+
+	run_p = sub.add_parser("run", help="Run one detection-triage-notify cycle.")
+	run_p.add_argument(
+		"--force",
+		action="store_true",
+		help="Run even if the last cycle is not yet due.",
+	)
+
+	sub.add_parser("test", help="Dry-run and render the current digest to stdout.")
+	sub.add_parser("catalog", help="List configured sources from the catalog.")
+	sub.add_parser("doctor", help="Run health checks.")
+
+	schedule_p = sub.add_parser("schedule", help="Manage the native scheduled job.")
+	schedule_p.add_argument(
+		"action",
+		choices=["install", "remove", "status"],
+		help="Schedule action to perform.",
+	)
+
+	# Wire up handlers after all subparsers are registered.
+	for name, func in [
+		("run", _cmd_run),
+		("test", _cmd_test),
+		("catalog", _cmd_catalog),
+		("doctor", _cmd_doctor),
+		("schedule", _cmd_schedule),
+	]:
+		sub.choices[name].set_defaults(func=func)
+
+	return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+	parser = _build_parser()
+	args = parser.parse_args(argv)
+	return args.func(args)

android_watcher/config.py

@@ -0,0 +1,262 @@
+"""Config dataclasses, path helpers, TOML loader, and env interpolation."""
+
+from __future__ import annotations
+
+import os
+import re
+import tomllib
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+import platformdirs
+
+from .models import ConfigError, Source  # ConfigError defined once in models.py
+
+__all__ = [
+	"AIConfig",
+	"Config",
+	"ConfigError",
+	"DigestConfig",
+	"EmailChannel",
+	"ScheduleConfig",
+	"SlackChannel",
+	"TelegramChannel",
+	"config_path",
+	"data_path",
+	"db_path",
+	"load_config",
+	"log_path",
+]
+
+APP_NAME = "android-watcher"
+_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+_VALID_INTERVALS = {"hourly", "daily", "weekly", "cron"}
+_VALID_AI_MODES = {"claude_cli", "off"}
+_VALID_EMPTY = {"send", "skip"}
+
+
+@dataclass
+class ScheduleConfig:
+	interval: Literal["hourly", "daily", "weekly", "cron"] = "daily"
+	at: str = "09:00"  # one or more HH:MM, comma-separated
+	days: str = "mon"  # weekly only: comma-separated weekday abbrevs (mon..sun)
+	cron: str = ""
+
+
+@dataclass
+class AIConfig:
+	mode: Literal["claude_cli", "off"] = "claude_cli"
+	model: str = "claude-sonnet-4-6"
+
+
+@dataclass
+class DigestConfig:
+	max_items: int = 10
+	empty: Literal["send", "skip"] = "send"
+
+
+@dataclass
+class EmailChannel:
+	enabled: bool = False
+	smtp_host: str = ""
+	smtp_port: int = 465
+	username: str = ""
+	password: str = ""
+	sender: str = ""  # maps to TOML key "from"
+	recipient: str = ""  # maps to TOML key "to"
+
+
+@dataclass
+class SlackChannel:
+	enabled: bool = False
+	bot_token: str = ""  # secret; supports ${ENV_VAR}
+	channel: str = ""
+
+
+@dataclass
+class TelegramChannel:
+	enabled: bool = False
+	bot_token: str = ""  # secret; supports ${ENV_VAR}
+	chat_id: str = ""
+
+
+@dataclass
+class Config:
+	schedule: ScheduleConfig
+	ai: AIConfig
+	digest: DigestConfig
+	sort: dict[str, int]
+	email: EmailChannel
+	slack: SlackChannel
+	telegram: TelegramChannel
+	custom_sources: list[Source]
+	enabled_source_ids: set[str] = field(default_factory=set)
+
+
+def config_path() -> str:
+	return os.path.join(platformdirs.user_config_dir(APP_NAME), "config.toml")
+
+
+def data_path() -> str:
+	return platformdirs.user_data_dir(APP_NAME)
+
+
+def db_path() -> str:
+	return os.path.join(data_path(), "state.db")
+
+
+def log_path() -> str:
+	"""Path to the run log. ~/Library/Logs/android-watcher.log on macOS."""
+	import sys  # noqa: PLC0415 - localized; keeps module import graph lean
+
+	if sys.platform == "darwin":
+		return os.path.join(os.path.expanduser("~/Library/Logs"), "android-watcher.log")
+	return os.path.join(platformdirs.user_log_dir(APP_NAME), "android-watcher.log")
+
+
+def _interpolate(value: str, *, expand: bool) -> str:
+	"""Resolve ${ENV} references in a secret-bearing field.
+
+	When expand=False, leave the literal untouched and never raise (the TUI
+	editor re-saves config without baking secrets into the file).
+	"""
+	if not expand:
+		return value
+
+	def repl(match: re.Match[str]) -> str:
+		name = match.group(1)
+		if name not in os.environ:
+			raise ConfigError(f"config references undefined env var ${{{name}}}")
+		return os.environ[name]
+
+	return _ENV_RE.sub(repl, value)
+
+
+def load_config(path: str | None = None, *, expand: bool = True) -> Config:
+	"""Load config from *path* (defaults to the platform config path).
+
+	Missing file returns all defaults. Raises ConfigError on invalid TOML or
+	contradictory schedule, or when slack is enabled without bot_token+channel.
+	Interpolation of ${ENV_VAR} applies only to secret-bearing fields (email
+	password, slack bot_token, telegram bot_token).
+	"""
+	target = path or config_path()
+	try:
+		with open(target, "rb") as fh:
+			raw = tomllib.load(fh)
+	except FileNotFoundError:
+		raw = {}
+	except tomllib.TOMLDecodeError as exc:
+		raise ConfigError(f"invalid TOML in {target}: {exc}") from exc
+
+	# Interpolation is scoped to secret-bearing fields only (see _load_email /
+	# _load_slack / _load_telegram). Every other string, including URLs, is
+	# passed through verbatim so a literal "${" is never an error.
+	schedule = _load_schedule(raw.get("schedule", {}))
+	ai = _load_ai(raw.get("ai", {}))
+	digest = _load_digest(raw.get("digest", {}))
+	sort = _load_sort(raw.get("sort", {}))
+	channels = raw.get("channels", {})
+	email = _load_email(channels.get("email", {}), expand=expand)
+	slack = _load_slack(channels.get("slack", {}), expand=expand)
+	telegram = _load_telegram(channels.get("telegram", {}), expand=expand)
+	custom_sources = [_load_source(e) for e in raw.get("custom_source", [])]
+	enabled = set(raw.get("enabled_sources", []))
+
+	if slack.enabled and not (slack.bot_token and slack.channel):
+		raise ConfigError("slack channel enabled but bot_token and channel are required")
+
+	return Config(
+		schedule=schedule,
+		ai=ai,
+		digest=digest,
+		sort=sort,
+		email=email,
+		slack=slack,
+		telegram=telegram,
+		custom_sources=custom_sources,
+		enabled_source_ids=enabled,
+	)
+
+
+def _load_schedule(d: dict[str, Any]) -> ScheduleConfig:
+	interval = d.get("interval", "daily")
+	if interval not in _VALID_INTERVALS:
+		raise ConfigError(
+			f"schedule.interval must be one of {sorted(_VALID_INTERVALS)}, got {interval!r}"
+		)
+	cron = d.get("cron", "")
+	if interval == "cron" and not cron:
+		raise ConfigError("schedule.interval = 'cron' requires a non-empty schedule.cron")
+	if interval != "cron" and cron:
+		raise ConfigError(
+			f"schedule.cron is set but interval is {interval!r}; "
+			"set interval = 'cron' or clear cron"
+		)
+	return ScheduleConfig(
+		interval=interval, at=d.get("at", "09:00"), days=d.get("days", "mon"), cron=cron
+	)
+
+
+def _load_ai(d: dict[str, Any]) -> AIConfig:
+	mode = d.get("mode", "claude_cli")
+	if mode not in _VALID_AI_MODES:
+		raise ConfigError(f"ai.mode must be one of {sorted(_VALID_AI_MODES)}, got {mode!r}")
+	return AIConfig(mode=mode, model=d.get("model", "claude-sonnet-4-6"))
+
+
+def _load_digest(d: dict[str, Any]) -> DigestConfig:
+	empty = d.get("empty", "send")
+	if empty not in _VALID_EMPTY:
+		raise ConfigError(f"digest.empty must be one of {sorted(_VALID_EMPTY)}, got {empty!r}")
+	return DigestConfig(
+		max_items=int(d.get("max_items", 10)),
+		empty=empty,
+	)
+
+
+def _load_sort(d: dict[str, Any]) -> dict[str, int]:
+	return {str(k): int(v) for k, v in d.items()}
+
+
+def _load_email(d: dict[str, Any], *, expand: bool) -> EmailChannel:
+	return EmailChannel(
+		enabled=bool(d.get("enabled", False)),
+		smtp_host=d.get("smtp_host", ""),
+		smtp_port=int(d.get("smtp_port", 465)),
+		username=d.get("username", ""),
+		password=_interpolate(d.get("password", ""), expand=expand),  # secret
+		sender=d.get("from", ""),
+		recipient=d.get("to", ""),
+	)
+
+
+def _load_slack(d: dict[str, Any], *, expand: bool) -> SlackChannel:
+	return SlackChannel(
+		enabled=bool(d.get("enabled", False)),
+		bot_token=_interpolate(d.get("bot_token", ""), expand=expand),  # secret
+		channel=d.get("channel", ""),
+	)
+
+
+def _load_telegram(d: dict[str, Any], *, expand: bool) -> TelegramChannel:
+	return TelegramChannel(
+		enabled=bool(d.get("enabled", False)),
+		bot_token=_interpolate(d.get("bot_token", ""), expand=expand),  # secret
+		chat_id=d.get("chat_id", ""),
+	)
+
+
+def _load_source(e: dict[str, Any]) -> Source:
+	return Source(
+		id=e["id"],
+		name=e["name"],
+		category=e["category"],
+		detector=e["detector"],
+		url=e["url"],
+		enabled=e.get("enabled", True),
+		path_prefix=e.get("path_prefix", ""),
+		feed_url=e.get("feed_url", ""),
+		content_selector=e.get("content_selector", ""),
+		default_weight=e.get("default_weight", 0),
+	)

android_watcher/detect/__init__.py

@@ -0,0 +1 @@
+from . import android_sitemap, content, feed, sitemap  # noqa: F401

android_watcher/detect/_normalize.py

@@ -0,0 +1,192 @@
+"""Content normalisation helpers shared across detectors.
+
+Uses only stdlib (``html.parser``) — no third-party HTML library required.
+Later detectors (sitemap content-confirm, etc.) import from here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from html.parser import HTMLParser
+
+# A page whose normalised main-text falls below this character count is treated
+# as a client-side render shell.  The content detector refuses to baseline it
+# and emits no Change; ``doctor`` surfaces the condition separately.
+EMPTY_RENDER_THRESHOLD: int = 50
+
+# Tags whose text content we discard entirely.
+_SKIP_TAGS: frozenset[str] = frozenset({"script", "style", "noscript", "template"})
+
+
+class _TextExtractor(HTMLParser):
+	"""Walk HTML and collect visible text, discarding script/style."""
+
+	def __init__(self) -> None:
+		super().__init__(convert_charrefs=True)
+		self._skip_depth: int = 0
+		self._parts: list[str] = []
+
+	def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+		if tag in _SKIP_TAGS:
+			self._skip_depth += 1
+
+	def handle_endtag(self, tag: str) -> None:
+		if tag in _SKIP_TAGS and self._skip_depth > 0:
+			self._skip_depth -= 1
+
+	def handle_data(self, data: str) -> None:
+		if self._skip_depth == 0:
+			stripped = data.strip()
+			if stripped:
+				self._parts.append(stripped)
+
+	def text(self) -> str:
+		return " ".join(self._parts)
+
+
+class _SelectorExtractor(HTMLParser):
+	"""Extract the inner HTML of the first element matching a simple selector.
+
+	Supported selector forms (the subset needed by this project):
+	- ``#id``   — matches the first element with that id attribute.
+	- ``tag``   — matches the first element with that tag name.
+	- ``""``    — no selector; returns the full HTML unchanged (caller should
+	              pass the raw HTML back to ``_TextExtractor``).
+	"""
+
+	def __init__(self, selector: str) -> None:
+		super().__init__(convert_charrefs=False)
+		self._selector = selector.strip()
+		self._match_id: str | None = None
+		self._match_tag: str | None = None
+
+		if self._selector.startswith("#"):
+			self._match_id = self._selector[1:]
+		elif self._selector:
+			self._match_tag = self._selector
+
+		self._depth: int = 0  # nesting depth inside the matched element
+		self._capturing: bool = False
+		self._found: bool = False
+		self._raw_parts: list[str] = []
+
+	def _matches(self, tag: str, attrs: list[tuple[str, str | None]]) -> bool:
+		if self._match_id is not None:
+			attr_dict = dict(attrs)
+			return attr_dict.get("id") == self._match_id
+		if self._match_tag is not None:
+			return tag == self._match_tag
+		return False
+
+	def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+		if self._found:
+			return
+		if self._capturing:
+			self._depth += 1
+			# Re-serialise the opening tag so inner elements are kept.
+			attr_str = "".join(f' {k}="{v}"' if v is not None else f" {k}" for k, v in attrs)
+			self._raw_parts.append(f"<{tag}{attr_str}>")
+			return
+		if self._matches(tag, attrs):
+			self._capturing = True
+			self._depth = 1
+
+	def handle_endtag(self, tag: str) -> None:
+		if not self._capturing or self._found:
+			return
+		self._depth -= 1
+		if self._depth == 0:
+			self._capturing = False
+			self._found = True
+		else:
+			self._raw_parts.append(f"</{tag}>")
+
+	def handle_data(self, data: str) -> None:
+		if self._capturing and not self._found:
+			self._raw_parts.append(data)
+
+	def handle_entityref(self, name: str) -> None:  # type: ignore[override]
+		if self._capturing and not self._found:
+			self._raw_parts.append(f"&{name};")
+
+	def handle_charref(self, name: str) -> None:  # type: ignore[override]
+		if self._capturing and not self._found:
+			self._raw_parts.append(f"&#{name};")
+
+	def fragment(self) -> str | None:
+		"""Return the captured inner HTML, or None if no match was found."""
+		return "".join(self._raw_parts) if self._found else None
+
+
+def extract_main(html: str, selector: str = "") -> str:
+	"""Return the HTML fragment addressed by *selector*, or *html* if no match.
+
+	Selector forms: ``#id``, ``tagname``, or ``""`` (whole document).
+	Falls back to the full *html* when the selector matches nothing so the
+	caller always gets something to normalise.
+	"""
+	if not selector:
+		return html
+	extractor = _SelectorExtractor(selector)
+	extractor.feed(html)
+	return extractor.fragment() or html
+
+
+def normalize_text(html_fragment: str) -> str:
+	"""Strip all markup and attributes; collapse whitespace to single spaces.
+
+	Two fragments that differ only in CSS classes, random data attributes, or
+	extra whitespace will produce the same normalised string and therefore the
+	same :func:`content_hash`.
+	"""
+	parser = _TextExtractor()
+	parser.feed(html_fragment)
+	text = parser.text()
+	# Collapse any remaining internal whitespace runs.
+	return re.sub(r"\s+", " ", text).strip()
+
+
+def content_hash(text: str) -> str:
+	"""SHA-256 of the normalised text (hex string)."""
+	return hashlib.sha256(text.encode()).hexdigest()
+
+
+class _TitleExtractor(HTMLParser):
+	"""Capture the text inside the first <title> element."""
+
+	def __init__(self) -> None:
+		super().__init__(convert_charrefs=True)
+		self._in = False
+		self._done = False
+		self._parts: list[str] = []
+
+	def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+		if tag == "title" and not self._done:
+			self._in = True
+
+	def handle_endtag(self, tag: str) -> None:
+		if tag == "title" and self._in:
+			self._in = False
+			self._done = True
+
+	def handle_data(self, data: str) -> None:
+		if self._in:
+			self._parts.append(data)
+
+	def title(self) -> str:
+		return re.sub(r"\s+", " ", "".join(self._parts)).strip()
+
+
+def extract_title(html: str) -> str:
+	"""The page's <title> text (whitespace-collapsed), or "" if absent/unparsable.
+
+	The stdlib HTML parser raises on some binary payloads; callers treat a
+	failure as "no title", so this never propagates an exception.
+	"""
+	parser = _TitleExtractor()
+	try:
+		parser.feed(html)
+	except Exception:  # noqa: BLE001 - binary/garbage HTML => no title
+		return ""
+	return parser.title()