android-watcher 1.0.0__py3-none-any.whl

android_watcher/detect/feed.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import hashlib
+from urllib.parse import urlsplit, urlunsplit
+
+from defusedxml import ElementTree as ET
+
+from ..models import Change, Source
+from .base import DETECTORS
+
+_ATOM = "{http://www.w3.org/2005/Atom}"
+
+
+def _normalize_link(link: str) -> str:
+	parts = urlsplit(link.strip())
+	# Strip query and fragment; keep scheme/host/path; drop trailing slash on path.
+	path = parts.path.rstrip("/") or "/"
+	return urlunsplit((parts.scheme, parts.netloc, path, "", ""))
+
+
+def _hash(title: str, summary: str) -> str:
+	h = hashlib.sha256()
+	h.update(title.strip().encode())
+	h.update(b"\x00")
+	h.update(summary.strip().encode())
+	return h.hexdigest()
+
+
+def _text(el: ET.Element | None) -> str:
+	return (el.text or "").strip() if el is not None else ""
+
+
+def _parse_items(xml: str) -> list[dict]:
+	root = ET.fromstring(xml)
+	items: list[dict] = []
+
+	# Atom
+	for entry in root.findall(f"{_ATOM}entry"):
+		link_el = entry.find(f"{_ATOM}link")
+		link = (link_el.get("href", "") if link_el is not None else "").strip()
+		id_raw = _text(entry.find(f"{_ATOM}id"))
+		title = _text(entry.find(f"{_ATOM}title"))
+		summary_el = entry.find(f"{_ATOM}summary")
+		if summary_el is None:
+			summary_el = entry.find(f"{_ATOM}content")
+		summary = _text(summary_el)
+		# Atom <id> is always treated as a permalink identity (opaque IRI).
+		items.append(
+			{
+				"id_raw": id_raw,
+				"id_is_permalink": bool(id_raw),
+				"link": link,
+				"title": title,
+				"summary": summary,
+			}
+		)
+
+	# RSS (root tag is <rss> or <channel> is a child)
+	channel = root.find("channel")
+	if channel is None and root.tag == "channel":
+		channel = root
+	if channel is not None:
+		for item in channel.findall("item"):
+			guid_el = item.find("guid")
+			guid_raw = _text(guid_el)
+			is_permalink = (
+				guid_el is not None
+				and guid_el.get("isPermaLink", "true").lower() != "false"
+				and bool(guid_raw)
+			)
+			link = _text(item.find("link"))
+			title = _text(item.find("title"))
+			summary = _text(item.find("description"))
+			items.append(
+				{
+					"id_raw": guid_raw,
+					"id_is_permalink": is_permalink,
+					"link": link,
+					"title": title,
+					"summary": summary,
+				}
+			)
+
+	return items
+
+
+def _identity(item: dict) -> str:
+	# Prefer a permalink id/guid; use it VERBATIM (an Atom <id> is an opaque IRI,
+	# often a tag: URI that must not be URL-normalized). Only the link-URL
+	# fallback is normalized. Never trust a non-permalink raw guid alone
+	# (Medium/Blogger reuse them).
+	if item["id_is_permalink"] and item["id_raw"]:
+		return item["id_raw"]
+	return _normalize_link(item["link"])
+
+
+@DETECTORS.register("feed")
+class FeedDetector:
+	async def detect(self, source: Source, store, fetcher) -> list[Change]:
+		url = source.feed_url or source.url
+		res = await fetcher.fetch(url)
+		if res.not_modified or not res.text:
+			return []
+		changes: list[Change] = []
+		for item in _parse_items(res.text):
+			identity = _identity(item)
+			if not identity:
+				continue
+			content_hash = _hash(item["title"], item["summary"])
+			prior = store.seen_feed_item(source.id, identity)
+			if prior is None:
+				changes.append(
+					Change(
+						source_id=source.id,
+						url=item["link"] or identity,
+						change_kind="new",
+						title=item["title"],
+						raw_diff=f"{item['title']}\n\n{item['summary']}".strip()[:500],
+						fetched_hash=content_hash,
+					)
+				)
+				store.upsert_seen_feed_item(source.id, identity, content_hash)
+			elif prior != content_hash:
+				changes.append(
+					Change(
+						source_id=source.id,
+						url=item["link"] or identity,
+						change_kind="updated",
+						title=item["title"],
+						raw_diff=f"{item['title']}\n\n{item['summary']}".strip()[:500],
+						fetched_hash=content_hash,
+					)
+				)
+				store.upsert_seen_feed_item(source.id, identity, content_hash)
+		return changes

android_watcher/detect/sitemap.py

@@ -0,0 +1,203 @@
+"""Generic sitemap detector: candidate-then-confirm.
+
+A <lastmod> bump in the sitemap is a *candidate* — it is never recorded as a
+Change on its own.  The page is fetched and its normalized content is hashed;
+only a real content-hash move produces a Change.
+
+If the confirm fetch returns 304 (ETag/If-Modified-Since unchanged), the
+sitemap lastmod is bumped in the snapshot (so the candidate quiesces next run)
+but no Change is emitted.
+"""
+
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlsplit
+
+import defusedxml.ElementTree as ET
+
+from ..models import Change, Source
+from ._normalize import (
+	EMPTY_RENDER_THRESHOLD,
+	content_hash,
+	extract_main,
+	extract_title,
+	normalize_text,
+)
+from .base import DETECTORS
+
+logger = logging.getLogger(__name__)
+
+_SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
+
+
+def _parse_urlset(xml_text: str) -> list[tuple[str, str]]:
+	"""Return [(loc, lastmod), ...] from a sitemap <urlset> XML string.
+
+	Missing or empty <lastmod> values are returned as empty strings.
+	"""
+	root = ET.fromstring(xml_text)
+	entries: list[tuple[str, str]] = []
+	for url_el in root.findall(f"{{{_SITEMAP_NS}}}url"):
+		loc_el = url_el.find(f"{{{_SITEMAP_NS}}}loc")
+		lastmod_el = url_el.find(f"{{{_SITEMAP_NS}}}lastmod")
+		if loc_el is None or not (loc_el.text or "").strip():
+			continue
+		loc = loc_el.text.strip()
+		lastmod = (lastmod_el.text or "").strip() if lastmod_el is not None else ""
+		entries.append((loc, lastmod))
+	return entries
+
+
+def _matches_prefix(loc: str, prefix: str) -> bool:
+	if not prefix:
+		return True
+	return urlsplit(loc).path.startswith(prefix)
+
+
+async def confirm_candidate(
+	source: Source,
+	store: object,
+	fetcher: object,
+	loc: str,
+	lastmod: str,
+	*,
+	emit_new: bool = False,
+) -> Change | None:
+	"""Per-URL confirm that never raises. Any failure — robots-blocked
+	(``Disallowed``), a binary masquerading as HTML (the stdlib HTML parser
+	throws on it), a transport error — is logged and skipped, so one bad URL
+	cannot abort a source's whole detection run."""
+	try:
+		return await _confirm_candidate(source, store, fetcher, loc, lastmod, emit_new=emit_new)
+	except Exception as exc:  # noqa: BLE001 - per-URL isolation is the point
+		logger.warning("confirm_candidate skipped %r: %s", loc, exc)
+		return None
+
+
+async def _confirm_candidate(
+	source: Source,
+	store: object,
+	fetcher: object,
+	loc: str,
+	lastmod: str,
+	*,
+	emit_new: bool = False,
+) -> Change | None:
+	"""Fetch *loc*, hash normalized content, return a Change only on a real move.
+
+	Contract (pinned):
+	- First content capture (never seen, or fetch-free baseline with an empty
+	  content_hash): baseline silently, return None — UNLESS ``emit_new`` is set
+	  (a genuinely new URL discovered after a baseline exists), in which case the
+	  captured content is returned as Change(change_kind="new").
+	- Identical re-confirm (hash unchanged): return None.
+	- Content-hash move: return Change(change_kind="updated").
+	- 304 from server: persist new lastmod (quiesce), return None.
+	"""
+	res = await fetcher.fetch(loc, conditional=True)  # type: ignore[union-attr]
+	snap = store.get_snapshot(source.id, loc)  # type: ignore[union-attr]
+
+	if res.not_modified:
+		# Server confirms content is unchanged despite a lastmod advance.
+		# Persist the new lastmod so this candidate won't re-fire next run.
+		if snap is not None:
+			store.upsert_snapshot(  # type: ignore[union-attr]
+				source.id,
+				loc,
+				signal_type="sitemap",
+				content_hash=snap.content_hash,
+				lastmod=lastmod,
+				excerpt=snap.excerpt,
+			)
+		return None
+
+	text = normalize_text(extract_main(res.text, source.content_selector))
+
+	if len(text) < EMPTY_RENDER_THRESHOLD:
+		logger.warning(
+			"sitemap detector: page at %r returned a JS-shell (text length %d < %d) — "
+			"skipping baseline/change; doctor will surface this",
+			loc,
+			len(text),
+			EMPTY_RENDER_THRESHOLD,
+		)
+		return None
+
+	new_hash = content_hash(text)
+	# The page's own <title> names the change; the source name is only a fallback
+	# (so a digest never reads "Android Open Source Project" for every page).
+	title = extract_title(res.text) or source.name
+
+	# First content capture is silent: either the URL was never seen, or it was
+	# baselined fetch-free on a prior run (empty content_hash) and this is its
+	# first real fetch. A Change requires a genuine prior content hash that
+	# moved — a lastmod bump alone never counts.
+	first_capture = snap is None or not snap.content_hash
+	store.upsert_snapshot(  # type: ignore[union-attr]
+		source.id,
+		loc,
+		signal_type="sitemap",
+		content_hash=new_hash,
+		lastmod=lastmod,
+		excerpt=text[:500],
+	)
+
+	if first_capture:
+		if emit_new:
+			# Genuinely new URL after baseline: report its first capture as "new".
+			return Change(
+				source_id=source.id,
+				url=loc,
+				change_kind="new",
+				title=title,
+				raw_diff=text[:500],
+				fetched_hash=new_hash,
+			)
+		return None  # baseline silently
+
+	if snap.content_hash == new_hash:
+		return None  # content re-confirmed identical
+
+	return Change(
+		source_id=source.id,
+		url=loc,
+		change_kind="updated",
+		title=title,
+		raw_diff=text[:500],
+		fetched_hash=new_hash,
+	)
+
+
+@DETECTORS.register("sitemap")
+class SitemapDetector:
+	async def detect(self, source: Source, store: object, fetcher: object) -> list[Change]:
+		res = await fetcher.fetch(source.url, conditional=True)  # type: ignore[union-attr]
+		if res.not_modified or not res.text:
+			return []
+
+		entries = _parse_urlset(res.text)
+		matched = [
+			(loc, lastmod) for loc, lastmod in entries if _matches_prefix(loc, source.path_prefix)
+		]
+
+		if source.path_prefix and not matched:
+			logger.warning(
+				"sitemap detector: no URLs matched path_prefix %r for source %r — "
+				"check the prefix or the sitemap URL",
+				source.path_prefix,
+				source.id,
+			)
+			return []
+
+		changes: list[Change] = []
+		for loc, lastmod in matched:
+			snap = store.get_snapshot(source.id, loc)  # type: ignore[union-attr]
+			# Skip if lastmod is present and unchanged — not even a candidate
+			if snap is not None and lastmod and snap.lastmod == lastmod:
+				continue
+			change = await confirm_candidate(source, store, fetcher, loc, lastmod)
+			if change is not None:
+				changes.append(change)
+
+		return changes

android_watcher/doctor.py

@@ -0,0 +1,125 @@
+"""Doctor health checks.
+
+``run_doctor(config)`` returns a list of ``Check`` objects covering:
+- sitemap path-prefix resolution for each android_sitemap source
+- AI backend availability
+- schedule status (soft dependency on the schedule module, imported lazily)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+
+from android_watcher import __version__
+from android_watcher.config import Config, db_path
+from android_watcher.detect.android_sitemap import (
+	INDEX_URL,
+	_index_url_for,
+	load_sitemap,
+	prefix_count,
+)
+from android_watcher.fetch import USER_AGENT, Fetcher
+from android_watcher.models import Check
+from android_watcher.run import resolve_sources
+from android_watcher.store import Store
+
+
+def _check_ai(config: Config) -> Check:
+	if config.ai.mode == "off":
+		return Check("ai-backend", True, "AI disabled")
+	path = shutil.which("claude")
+	if path:
+		return Check("ai-backend", True, f"claude found at {path}")
+	return Check("ai-backend", False, "claude not found on PATH")
+
+
+def _check_seed() -> Check:
+	"""Report the imported baseline seed date and snapshot count, if any."""
+	store = Store(db_path())
+	store.migrate()
+	try:
+		count = store.snapshot_count()
+		date = store.seed_date()
+	finally:
+		store.close()
+	if count == 0:
+		return Check("seed", True, "no baseline yet; first run will establish one")
+	if date:
+		return Check("seed", True, f"baseline seeded {date} ({count} snapshots)")
+	return Check("seed", True, f"baseline established ({count} snapshots)")
+
+
+def _check_schedule() -> Check:
+	try:
+		from android_watcher.schedule import schedule_status  # noqa: PLC0415
+	except ImportError:
+		return Check("schedule", False, "schedule module unavailable")
+	return schedule_status()
+
+
+def _load_sitemap_entries(index_url: str = INDEX_URL) -> list[tuple[str, str]]:
+	"""Fetch one host's sitemap once and return the flat entry list.
+
+	Extracted so tests can patch this function directly instead of having to
+	wire up a real Store + Fetcher + asyncio event loop.
+	"""
+
+	async def _load() -> list[tuple[str, str]]:
+		store = Store(db_path())
+		store.migrate()
+		fetcher = Fetcher(store, user_agent=USER_AGENT.format(version=__version__))
+		try:
+			# Time-box it: a sitemap can be large (~300 MB uncached), so doctor
+			# reports a slow/unavailable sitemap rather than appearing to hang.
+			return await asyncio.wait_for(load_sitemap(fetcher, index_url), timeout=30)
+		finally:
+			await fetcher.close()
+
+	return asyncio.run(_load())
+
+
+def _check_prefixes(config: Config) -> list[Check]:
+	targets = [s for s in resolve_sources(config) if s.detector == "android_sitemap"]
+	if not targets:
+		return []
+
+	by_host: dict[str, list] = {}
+	for s in targets:
+		by_host.setdefault(_index_url_for(s), []).append(s)
+
+	checks: list[Check] = []
+	for index_url, srcs in by_host.items():
+		host = index_url.split("/sitemap.xml")[0]
+		try:
+			entries = _load_sitemap_entries(index_url)
+		except TimeoutError:
+			checks.append(Check(f"sitemap:{host}", True, "fetch slow; run once to cache"))
+			continue
+		except Exception as exc:  # noqa: BLE001 - any fetch/parse failure is a soft check
+			checks.append(Check(f"sitemap:{host}", False, f"unavailable; run once first ({exc})"))
+			continue
+		if not entries:
+			checks.append(Check(f"sitemap:{host}", True, "cached (304); not re-verified"))
+			continue
+		for s in srcs:
+			if not s.path_prefix:
+				checks.append(Check(f"prefix:{s.id}", True, f"watches host ({len(entries)} URLs)"))
+				continue
+			count = prefix_count(entries, s.path_prefix)
+			if count == 0:
+				checks.append(
+					Check(f"prefix:{s.path_prefix}", False, "stale prefix: 0 sitemap URLs match")
+				)
+			else:
+				checks.append(Check(f"prefix:{s.path_prefix}", True, f"resolves ({count} URLs)"))
+	return checks
+
+
+def run_doctor(config: Config) -> list[Check]:
+	checks: list[Check] = []
+	checks.extend(_check_prefixes(config))
+	checks.append(_check_seed())
+	checks.append(_check_ai(config))
+	checks.append(_check_schedule())
+	return checks

android_watcher/fetch.py

@@ -0,0 +1,162 @@
+"""Async HTTP fetch layer for android-watcher.
+
+Provides ``Fetcher``, a concurrency-limited async client that:
+- Sets a descriptive User-Agent.
+- Honors robots.txt per host (urllib.robotparser).
+- Applies a per-host crawl delay.
+- Retries with exponential backoff on 5xx / transport errors.
+- Supports conditional GET via Store.http_cache_get/put.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import urllib.robotparser
+from urllib.parse import urlsplit
+
+import httpx
+
+from .models import Disallowed, FetchResult
+from .store import Store
+
+log = logging.getLogger("android_watcher.fetch")
+
+USER_AGENT = "android-watcher/{version}"
+
+MAX_RETRIES = 4
+BACKOFF_BASE = 0.5
+# Per-request httpx timeouts (connect/read/write/pool). read is the gap between
+# bytes, not total, so a large steady download is fine; a stalled one trips it.
+_TIMEOUT = httpx.Timeout(connect=10.0, read=30.0, write=30.0, pool=10.0)
+# Hard ceiling on a single fetch including retries + backoff, so one bad URL can
+# never hang a run indefinitely.
+FETCH_DEADLINE = 120.0
+
+
+class Fetcher:
+	def __init__(
+		self,
+		store: Store,
+		*,
+		user_agent: str,
+		concurrency: int = 4,
+		crawl_delay: float = 0.5,
+	):
+		self._store = store
+		self._user_agent = user_agent
+		self._crawl_delay = crawl_delay
+		self._sem = asyncio.Semaphore(concurrency)
+		self._client = httpx.AsyncClient(
+			headers={"User-Agent": user_agent},
+			follow_redirects=True,
+			timeout=_TIMEOUT,
+		)
+		self._robots: dict[str, urllib.robotparser.RobotFileParser | None] = {}
+		self._last_fetch: dict[str, float] = {}
+
+	async def fetch(self, url: str, *, conditional: bool = False) -> FetchResult:
+		if not await self._robots_ok(url):
+			raise Disallowed(url)
+
+		headers: dict[str, str] = {}
+		if conditional:
+			etag, last_modified = self._store.http_cache_get(url)
+			if etag:
+				headers["If-None-Match"] = etag
+			if last_modified:
+				headers["If-Modified-Since"] = last_modified
+
+		async with self._sem:
+			await self._respect_crawl_delay(url)
+			log.info("downloading %s", url)
+			resp = await asyncio.wait_for(
+				self._get_with_backoff(url, headers), timeout=FETCH_DEADLINE
+			)
+
+		if resp.status_code == 304:
+			return FetchResult(url=url, status=304, text="", not_modified=True)
+
+		etag = resp.headers.get("ETag", "")
+		last_modified = resp.headers.get("Last-Modified", "")
+		# Only persist validators when at least one is non-empty; never clobber
+		# an existing cache entry with ("","") on a validator-less 200.
+		if conditional and (etag or last_modified):
+			self._store.http_cache_put(url, etag, last_modified)
+
+		return FetchResult(
+			url=url,
+			status=resp.status_code,
+			text=resp.text,
+			etag=etag,
+			last_modified=last_modified,
+		)
+
+	async def close(self) -> None:
+		await self._client.aclose()
+
+	async def _get_with_backoff(self, url: str, headers: dict[str, str]) -> httpx.Response:
+		# Merge User-Agent into per-request headers so it is sent even when
+		# tests inject a bare _client that has no default headers set.
+		request_headers = {"User-Agent": self._user_agent, **headers}
+		last_exc: Exception | None = None
+		for attempt in range(MAX_RETRIES):
+			try:
+				resp = await self._client.get(url, headers=request_headers)
+			except (httpx.TransportError, httpx.TimeoutException) as exc:
+				last_exc = exc
+				resp = None
+			if resp is not None and resp.status_code < 500:
+				return resp
+			if attempt == MAX_RETRIES - 1:
+				if resp is not None:
+					return resp
+				raise last_exc  # type: ignore[misc]
+			await asyncio.sleep(BACKOFF_BASE * (2**attempt))
+		raise last_exc  # unreachable
+
+	async def _respect_crawl_delay(self, url: str) -> None:
+		host = _host_root(url)
+		delay = self._crawl_delay_for(url)
+		loop = asyncio.get_event_loop()
+		now = loop.time()
+		last = self._last_fetch.get(host)
+		if last is not None:
+			wait = delay - (now - last)
+			if wait > 0:
+				await asyncio.sleep(wait)
+		self._last_fetch[host] = loop.time()
+
+	def _crawl_delay_for(self, url: str) -> float:
+		rp = self._robots.get(_host_root(url))
+		if rp is not None:
+			cd = rp.crawl_delay(self._user_agent)
+			if cd is not None:
+				return float(cd)
+		return self._crawl_delay
+
+	async def _robots_ok(self, url: str) -> bool:
+		host = _host_root(url)
+		if host not in self._robots:
+			rp: urllib.robotparser.RobotFileParser | None = urllib.robotparser.RobotFileParser()
+			try:
+				# Fetch via the timed httpx client. urllib's RobotFileParser.read()
+				# uses urlopen with NO timeout and can hang a run forever if a host
+				# stalls; the shared client carries a 30s timeout instead.
+				resp = await self._client.get(f"{host}/robots.txt")
+				if resp.status_code >= 400:
+					rp = None  # treat missing/forbidden robots as "allow"
+				else:
+					rp.parse(resp.text.splitlines())
+			except (httpx.HTTPError, httpx.InvalidURL):
+				rp = None  # robots unavailable => allow
+			self._robots[host] = rp
+		rp = self._robots[host]
+		if rp is None:
+			return True
+		return rp.can_fetch(self._user_agent, url)
+
+
+def _host_root(url: str) -> str:
+	parts = urlsplit(url)
+	return f"{parts.scheme}://{parts.netloc}"

android_watcher/group.py

@@ -0,0 +1,79 @@
+"""Group ledger changes into DigestGroups: model group_key first, heuristic fallback."""
+
+from __future__ import annotations
+
+import re
+
+from .config import Config
+from .models import Change, DigestGroup, Source
+
+_WORD = re.compile(r"[a-z0-9]+")
+_DIGITS = re.compile(r"\d")
+
+
+def _title_case(text: str) -> str:
+	"""Capitalize the first letter of each word, preserving existing capitals so
+	acronyms survive (GKI stays GKI, OS stays OS, not Gki/Os)."""
+	return " ".join(w[:1].upper() + w[1:] if w else w for w in text.split(" "))
+
+
+def heuristic_prefix(title: str) -> str:
+	"""A coarse grouping key for changes without a model group_key.
+
+	Lowercase the title, keep leading words up to the first numeric run (so
+	'Android 13 release builds' and 'Android 14 release builds' collide), capped
+	at four words so unrelated titles do not over-merge.
+	"""
+	words = _WORD.findall(title.lower())
+	kept: list[str] = []
+	for w in words:
+		if _DIGITS.search(w):
+			break
+		kept.append(w)
+		if len(kept) >= 4:
+			break
+	return " ".join(kept) if kept else title.strip().lower()
+
+
+def group_changes(
+	changes: list[Change], sources: dict[str, Source], config: Config
+) -> list[DigestGroup]:
+	# Lazy import avoids a rank<->group module cycle: rank.py will import
+	# group_changes at the top level (Task 4), so a top-level import of _score
+	# here would create rank -> group -> rank with _score not yet defined.
+	from .rank import _score
+
+	buckets: dict[str, list[Change]] = {}
+	for c in changes:
+		sub = c.group_key or heuristic_prefix(c.title)
+		key = f"{c.source_id}::{sub}"
+		buckets.setdefault(key, []).append(c)
+
+	groups: list[DigestGroup] = []
+	for key, members in buckets.items():
+		members.sort(key=lambda c: (c.detected_at, c.id or 0), reverse=True)
+		source = sources.get(members[0].source_id)
+		# Heading: prefer the model's group headline; else the representative page
+		# title. Summary: prefer the model's group summary; else the representative
+		# change's own one-line description (so every group shows a sentence).
+		summary = next((m.group_summary for m in members if m.group_summary), None) or next(
+			(m.description for m in members if m.description), None
+		)
+		raw_title = next((m.group_title for m in members if m.group_title), None) or next(
+			(m.title for m in members if m.title), None
+		)
+		title = _title_case(raw_title) if raw_title else members[0].url
+		score = max(_score(m, source, config) for m in members)
+		groups.append(
+			DigestGroup(
+				key=key,
+				title=title,
+				summary=summary,
+				category=source.category if source else "",
+				source_id=members[0].source_id,
+				change_kind=members[0].change_kind,
+				members=members,
+				score=score,
+			)
+		)
+	return groups