agentgraph-connector-rss 0.5.0__py3-none-any.whl

agentgraph_connector_rss/__init__.py

@@ -0,0 +1,807 @@
+"""RSS and Atom connector."""
+
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false
+# pyright: reportUnknownArgumentType=false
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from email.utils import parsedate_to_datetime
+from importlib import import_module
+from typing import Any, cast
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import httpx
+
+from agentgraph.connectors.base import (
+    BaseConnector,
+    ConnectorAccount,
+    ConnectorCommandEffects,
+    EdgeRecord,
+    EntityBatch,
+    EntityRecord,
+    FetchPolicy,
+    PersonRecord,
+    ResourceType,
+    SourceReference,
+)
+from agentgraph.core.context import get_backend
+from agentgraph_connector_rss.auth import (
+    add_feed_urls,
+    import_opml_feeds,
+    load_rss_settings,
+    preview_feed,
+    remove_feed_urls,
+    resolve_feed_sources,
+    run_rss_flow,
+    verify_rss_auth,
+)
+
+_STALE_AFTER = 30 * 60
+_FEED_TIMEOUT = httpx.Timeout(10.0, connect=5.0)
+_MAX_FEED_BYTES = 5_000_000
+_MAX_OBSERVATION_ENTRIES_PER_FEED = 8
+_MAX_OBSERVATION_PATTERNS_PER_FEED = 5
+_TRACKING_QUERY_KEYS = {
+    "fbclid",
+    "gclid",
+    "dclid",
+    "msclkid",
+    "mc_cid",
+    "mc_eid",
+    "_hsenc",
+    "_hsmi",
+}
+logger = logging.getLogger(__name__)
+
+
+class RssConnector(BaseConnector):
+    source = "rss"
+    fetch_policy = FetchPolicy(stale_after_seconds=_STALE_AFTER)
+    poll_interval: timedelta | None = timedelta(minutes=15)  # type: ignore[assignment]
+    url_patterns = []
+    auth_label = "rss"
+    auth_description = "RSS/Atom feeds: feed URLs are fetched directly and entries are indexed as Document entities."
+    onboard_prompt = "Set up RSS feeds?"
+    appears_in_auth_status = False
+
+    def __init__(self) -> None:
+        self._observation_patterns: list[str] | None = None
+
+    @classmethod
+    def run_auth_flow(
+        cls,
+        account_id: str | None = None,
+        add: bool = False,
+        args: list[str] | None = None,
+    ) -> None:
+        if args:
+            raise ValueError(f"RSS setup does not accept auth options: {' '.join(args)}")
+        run_rss_flow(account_id=account_id, add=add)
+
+    @classmethod
+    def get_authenticated_user(cls) -> str | None:
+        try:
+            settings = load_rss_settings()
+        except RuntimeError:
+            return None
+        return "RSS" if settings.feed_urls else None
+
+    @classmethod
+    def list_accounts(cls) -> list[ConnectorAccount]:
+        return []
+
+    @classmethod
+    async def verify_auth(cls, account_id: str | None = None) -> tuple[str, str | None]:
+        return await verify_rss_auth(account_id)
+
+    @classmethod
+    def run_cli_command(cls, args: list[str]) -> dict[str, Any]:
+        if not args:
+            raise ValueError(_rss_usage())
+        command, *rest = args
+        if command == "add":
+            sources = _parse_feed_url_args(rest, command="add")
+            feed_urls = resolve_feed_sources(sources)
+            config = add_feed_urls(feed_urls, validate=False)
+            return {
+                "status": "ok",
+                "source": cls.source,
+                "feed_urls": config.feed_urls,
+                "added": feed_urls,
+            }
+        if command == "remove":
+            feed_urls = _parse_feed_url_args(rest, command="remove")
+            config, removed_feed_urls = remove_feed_urls(feed_urls)
+            return {
+                "status": "ok",
+                "source": cls.source,
+                "feed_urls": config.feed_urls,
+                "removed": removed_feed_urls,
+            }
+        if command == "import-opml":
+            options = _parse_import_opml_args(rest)
+            config, feeds, selected_feeds = import_opml_feeds(**options)
+            selected_feed_urls = [feed.feed_url for feed in selected_feeds]
+            return {
+                "status": "ok",
+                "source": cls.source,
+                "feed_urls": config.feed_urls,
+                "imported_feed_count": len(feeds),
+                "selected_feed_count": len(selected_feeds),
+                "added": selected_feed_urls,
+            }
+        raise ValueError(
+            f"Unknown rss connector command '{command}'. Available: add, remove, import-opml"
+        )
+
+    @classmethod
+    def cli_help(cls) -> str:
+        return _rss_help()
+
+    @classmethod
+    def format_cli_result(cls, result: dict[str, Any]) -> str:
+        return _format_rss_cli_result(result)
+
+    @classmethod
+    def command_effects(
+        cls,
+        args: list[str],
+        result: dict[str, Any],
+    ) -> ConnectorCommandEffects:
+        _ = result
+        command = args[0] if args else None
+        return ConnectorCommandEffects(poll=command in {"add", "import-opml"})
+
+    def can_handle(self, url: str) -> bool:
+        return self.resolve_url(url) is not None
+
+    def resolve_url(self, url: str) -> SourceReference | None:
+        try:
+            settings = load_rss_settings()
+        except RuntimeError:
+            return None
+        if url not in settings.feed_urls:
+            return None
+        return SourceReference(
+            source=self.source,
+            resource_type="document",
+            resource_id=url,
+        )
+
+    async def resolve_observation_url(self, url: str) -> SourceReference | None:
+        configured_feed = self.resolve_url(url)
+        if configured_feed is not None:
+            return configured_feed
+
+        normalised_url = normalise_article_url(url)
+        if normalised_url is None:
+            return None
+        try:
+            backend = get_backend()
+        except RuntimeError:
+            return None
+        entries = await backend.query_by_filter(
+            "Document",
+            {"platform": self.source, "web_url": normalised_url},
+            1,
+            "updated_at",
+            None,
+            None,
+        )
+        if not entries:
+            return None
+        entry = entries[0]
+        platform_entity_id = entry.get("platform_entity_id")
+        metadata = entry.get("metadata")
+        if not isinstance(platform_entity_id, str) or not isinstance(metadata, Mapping):
+            return None
+        return SourceReference(
+            source=self.source,
+            resource_type="document",
+            resource_id=platform_entity_id,
+            fetch_meta=_string_metadata(metadata),
+        )
+
+    async def observation_url_patterns(self) -> list[str]:
+        if self._observation_patterns is not None:
+            return self._observation_patterns
+        try:
+            settings = load_rss_settings()
+        except RuntimeError:
+            return []
+
+        links_by_feed: dict[str, list[str]] = {}
+        backend = get_backend()
+        for feed_url in settings.feed_urls:
+            entries = await backend.query_by_filter(
+                "Document",
+                {"platform": self.source, "feed_url": feed_url},
+                _MAX_OBSERVATION_ENTRIES_PER_FEED,
+                "updated_at",
+                None,
+                None,
+            )
+            links_by_feed[feed_url] = _entry_links(entries)
+        self._observation_patterns = derive_observation_url_patterns(links_by_feed)
+        return self._observation_patterns
+
+    async def ingest(self, account_id: str | None = None) -> EntityBatch:
+        settings = load_rss_settings(account_id)
+        combined = EntityBatch()
+        for feed_url in settings.feed_urls:
+            try:
+                batch = await _fetch_feed(feed_url, hydrate_documents=True)
+            except Exception as exc:
+                logger.warning(
+                    "Skipping RSS feed %s (%s: %s)",
+                    feed_url,
+                    type(exc).__name__,
+                    exc,
+                )
+                logger.debug("RSS feed fetch failure", exc_info=True)
+                continue
+            combined.entities.extend(batch.entities)
+            combined.edges.extend(batch.edges)
+            combined.persons.extend(batch.persons)
+        self._observation_patterns = derive_observation_url_patterns(
+            _entry_links_by_feed(combined.entities)
+        )
+        return combined
+
+    async def poll(
+        self,
+        cursor: dict[str, Any],
+        account_id: str | None = None,
+    ) -> tuple[EntityBatch, dict[str, Any]]:
+        _ = cursor
+        batch = await self.ingest(account_id=account_id)
+        return batch, {"last_polled_at": datetime.now(UTC).isoformat()}
+
+    async def preview_feed(self, feed_url: str, *, count: int = 3) -> dict[str, Any]:
+        return await preview_feed(feed_url, count=count)
+
+    async def fetch(
+        self,
+        resource_type: ResourceType,
+        resource_id: str,
+        meta: dict[str, str] | None = None,
+        account_id: str | None = None,
+    ) -> EntityBatch:
+        _ = account_id
+        feed_url = resource_id
+        if resource_type == "folder" and meta and meta.get("feed_url"):
+            feed_url = meta["feed_url"]
+        if feed_url.startswith(("http://", "https://")):
+            return await _fetch_feed(
+                feed_url,
+                hydrate_documents=True,
+                new_documents_only=True,
+            )
+        if resource_type == "document" and meta and meta.get("web_url"):
+            entity = await _fetch_entry_document(resource_id, meta)
+            return EntityBatch(entities=[entity])
+        return EntityBatch()
+
+
+async def _fetch_feed(
+    feed_url: str,
+    *,
+    hydrate_documents: bool = False,
+    new_documents_only: bool = False,
+) -> EntityBatch:
+    parsed = await _parse_feed(feed_url)
+    feed_title = str(cast(dict[str, Any], parsed.feed).get("title") or feed_url)
+    feed_id = _feed_id(feed_url)
+    feed_entity_id = f"feed/{feed_id}"
+    entities = [
+        EntityRecord(
+            entity_type="Folder",
+            platform="rss",
+            platform_entity_id=feed_entity_id,
+            title=feed_title,
+            content=f"RSS feed: {feed_title}\n{feed_url}",
+            metadata={"feed_url": feed_url, "web_url": feed_url},
+        )
+    ]
+    edges: list[EdgeRecord] = []
+    persons: dict[str, PersonRecord] = {}
+    batch = EntityBatch()
+    feed_authors = _parse_authors(cast(dict[str, Any], parsed.feed))
+    for author in feed_authors:
+        persons.setdefault(author.user_id, author.to_person())
+
+    for raw_entry in cast(list[Any], parsed.entries):
+        entry = cast(dict[str, Any], raw_entry)
+        # RFC 4287 §4.2.1: feed-level authors apply to entries that declare none.
+        authors = _parse_authors(entry) or feed_authors
+        entity = _entry_to_entity(feed_url, feed_entity_id, entry, authors)
+        include_entity = True
+        if new_documents_only:
+            existing = await get_backend().get_entity_by_platform(
+                "rss",
+                entity.platform_entity_id,
+            )
+            include_entity = existing is None
+        if include_entity:
+            if hydrate_documents:
+                entity = await _hydrate_entry_document(entity)
+            entities.append(entity)
+        edges.append(
+            EdgeRecord(
+                edge_type="posted_in",
+                source_platform_entity_id=entity.platform_entity_id,
+                target_platform_entity_id=feed_entity_id,
+                platform="rss",
+            )
+        )
+        for author in authors:
+            persons.setdefault(author.user_id, author.to_person())
+            edges.append(
+                EdgeRecord(
+                    edge_type="authored",
+                    source_platform_user_id=author.user_id,
+                    target_platform_entity_id=entity.platform_entity_id,
+                    platform="rss",
+                )
+            )
+        if include_entity:
+            batch.add_stubs_from(entity)
+
+    edges.extend(
+        EdgeRecord(
+            edge_type="authored",
+            source_platform_user_id=platform_user_id,
+            target_platform_entity_id=feed_entity_id,
+            platform="rss",
+        )
+        for platform_user_id in persons
+    )
+
+    batch.entities = [*entities, *batch.entities]
+    batch.edges = [*edges, *batch.edges]
+    batch.persons = [*persons.values(), *batch.persons]
+    return batch
+
+
+async def _parse_feed(feed_url: str) -> Any:
+    import asyncio
+
+    import feedparser  # type: ignore[import-untyped]
+
+    async with httpx.AsyncClient(
+        follow_redirects=True,
+        max_redirects=5,
+        timeout=_FEED_TIMEOUT,
+        headers={
+            "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"
+        },
+    ) as client:
+        response = await client.get(feed_url)
+        response.raise_for_status()
+        content = response.content
+        if len(content) > _MAX_FEED_BYTES:
+            raise ValueError(f"RSS feed response too large: limit is {_MAX_FEED_BYTES} bytes")
+        return await asyncio.to_thread(feedparser.parse, content)
+
+
+def _rss_usage() -> str:
+    return (
+        "Usage: agentgraph connector rss add <feed-url> [feed-url...]\n"
+        "   or: agentgraph connector rss remove <feed-url> [feed-url...]\n"
+        "   or: agentgraph connector rss import-opml <file.opml> [--all | --select <indexes>]"
+    )
+
+
+def _rss_help() -> str:
+    return "\n".join(
+        [
+            "RSS connector commands:",
+            "",
+            _rss_usage(),
+            "",
+            "Commands:",
+            "  add <feed-url> [feed-url...]",
+            "      Validate and add one or more RSS/Atom feeds, then queue an RSS poll.",
+            "  remove <feed-url> [feed-url...]",
+            "      Remove one or more exact RSS/Atom feed URLs from the configured feed list.",
+            "  import-opml <file.opml> [--all | --select <indexes>]",
+            "      Import RSS/Atom feed URLs from an OPML file. Omit flags for checkbox selection.",
+            "",
+            "Options:",
+            "  --all                   Import every feed from the OPML file.",
+            "  --select <indexes>      Import selected feed numbers, e.g. 1,3-5.",
+            "  --json                  Output command results as JSON.",
+        ]
+    )
+
+
+def _format_rss_cli_result(result: dict[str, Any]) -> str:
+    added = [str(item) for item in cast(list[object], result.get("added") or [])]
+    removed = [str(item) for item in cast(list[object], result.get("removed") or [])]
+    feed_urls = cast(list[object], result.get("feed_urls") or [])
+
+    if "imported_feed_count" in result:
+        imported_count = int(result.get("imported_feed_count") or 0)
+        selected_count = int(result.get("selected_feed_count") or len(added))
+        lines = [f"Imported {selected_count} of {imported_count} feed(s)."]
+    elif "removed" in result:
+        lines = [f"Removed {len(removed)} feed(s)."]
+    else:
+        lines = [f"Added {len(added)} feed(s)."]
+
+    if added:
+        lines.append("Added feeds:")
+        lines.extend(f"  - {feed_url}" for feed_url in added[:20])
+        if len(added) > 20:
+            lines.append(f"  ... {len(added) - 20} more")
+    if removed:
+        lines.append("Removed feeds:")
+        lines.extend(f"  - {feed_url}" for feed_url in removed[:20])
+        if len(removed) > 20:
+            lines.append(f"  ... {len(removed) - 20} more")
+    lines.append(f"Total configured feeds: {len(feed_urls)}")
+    poll = result.get("poll")
+    if isinstance(poll, Mapping):
+        status = str(poll.get("status") or "unknown").replace("_", " ")
+        lines.append(f"Poll: {status}.")
+    return "\n".join(lines)
+
+
+def _parse_feed_url_args(args: list[str], *, command: str) -> list[str]:
+    for arg in args:
+        if arg.startswith("--"):
+            raise ValueError(f"Unknown {command} option: {arg}")
+    return args
+
+
+def _parse_import_opml_args(args: list[str]) -> dict[str, Any]:
+    path: str | None = None
+    include_all = False
+    selection: str | None = None
+    index = 0
+    while index < len(args):
+        arg = args[index]
+        if arg == "--all":
+            include_all = True
+            index += 1
+            continue
+        if arg == "--select":
+            if index + 1 >= len(args):
+                raise ValueError("--select requires a value")
+            selection = args[index + 1]
+            index += 2
+            continue
+        if arg.startswith("--"):
+            raise ValueError(f"Unknown import-opml option: {arg}")
+        if path is not None:
+            raise ValueError(_rss_usage())
+        path = arg
+        index += 1
+    if path is None:
+        raise ValueError(_rss_usage())
+    if include_all and selection is not None:
+        raise ValueError("Use either --all or --select, not both")
+    return {
+        "path": path,
+        "include_all": include_all,
+        "selection": selection,
+    }
+
+
+@dataclass(frozen=True)
+class FeedAuthor:
+    """An author declared by a feed or one of its entries."""
+
+    user_id: str
+    display_name: str | None
+    email: str | None
+
+    def to_person(self) -> PersonRecord:
+        return PersonRecord(
+            platform="rss",
+            platform_user_id=self.user_id,
+            display_name=self.display_name,
+            canonical_email=self.email,
+        )
+
+    @property
+    def label(self) -> str:
+        return self.display_name or self.user_id
+
+
+def _author_details(container: Mapping[str, Any]) -> list[Mapping[str, Any]]:
+    """Return feedparser author dicts for a feed or entry, newest API first."""
+    raw_authors = container.get("authors")
+    if isinstance(raw_authors, list):
+        details = [item for item in cast(list[Any], raw_authors) if isinstance(item, Mapping)]
+        if details:
+            return cast(list[Mapping[str, Any]], details)
+    detail = container.get("author_detail")
+    if isinstance(detail, Mapping):
+        return [cast(Mapping[str, Any], detail)]
+    name = container.get("author")
+    if isinstance(name, str) and name.strip():
+        return [{"name": name}]
+    return []
+
+
+def _parse_authors(container: Mapping[str, Any]) -> list[FeedAuthor]:
+    """Map feedparser author metadata onto identity-bearing FeedAuthor records.
+
+    RSS ``<author>`` carries an email address, Atom ``<author>`` a name plus
+    optional email, and ``<dc:creator>`` a bare name; feedparser normalises all
+    three into ``{"name": ..., "email": ...}`` dicts with either key optional.
+    """
+    authors: list[FeedAuthor] = []
+    seen: set[str] = set()
+    for detail in _author_details(container):
+        name = _clean_author_field(detail.get("name"))
+        email = _clean_author_field(detail.get("email"))
+        if email is None and name is not None and "@" in name and " " not in name:
+            name, email = None, name
+        if email is not None:
+            email = email.lower()
+        user_id = email or name
+        if user_id is None or user_id in seen:
+            continue
+        seen.add(user_id)
+        authors.append(FeedAuthor(user_id=user_id, display_name=name, email=email))
+    return authors
+
+
+def _clean_author_field(value: object) -> str | None:
+    if not isinstance(value, str):
+        return None
+    return value.strip() or None
+
+
+def _entry_to_entity(
+    feed_url: str,
+    feed_entity_id: str,
+    entry: dict[str, Any],
+    authors: Sequence[FeedAuthor] = (),
+) -> EntityRecord:
+    link = normalise_article_url(str(entry.get("link") or "")) or ""
+    external_id = str(entry.get("id") or entry.get("guid") or link or entry.get("title") or "")
+    entity_id = f"entry/{_hash_ref(feed_url + ':' + external_id)}"
+    title = str(entry.get("title") or link or "(untitled)")
+    summary = _entry_text(entry)
+    published = _parse_entry_datetime(entry.get("published") or entry.get("updated"))
+    metadata: dict[str, str | int | float | bool | None] = {
+        "feed_url": feed_url,
+        "feed_entity_id": feed_entity_id,
+        "link": link or None,
+        "web_url": link or None,
+        "author": authors[0].label if authors else None,
+        "authors": ", ".join(author.label for author in authors) or None,
+    }
+    content_parts = [title]
+    if summary:
+        content_parts.append(summary)
+    if link:
+        content_parts.append(link)
+    return EntityRecord(
+        entity_type="Document",
+        platform="rss",
+        platform_entity_id=entity_id,
+        title=title,
+        content="\n\n".join(content_parts),
+        created_at=published,
+        updated_at=_parse_entry_datetime(entry.get("updated")) or published,
+        metadata=metadata,
+    )
+
+
+async def _fetch_entry_document(
+    platform_entity_id: str,
+    metadata: Mapping[str, object],
+    *,
+    fallback: EntityRecord | None = None,
+) -> EntityRecord:
+    web_url = _metadata_str(metadata, "web_url")
+    if web_url is None:
+        raise ValueError(f"RSS document {platform_entity_id} has no web_url")
+    existing = await get_backend().get_entity_by_platform("rss", platform_entity_id)
+    http_existing = _http_existing_entity(existing, web_url)
+    fetched = await _fetch_http_document(web_url, existing_entity=http_existing)
+    fetched_metadata = dict(fetched.metadata)
+    rss_metadata: dict[str, str | int | float | bool | None] = {
+        **_entity_record_metadata(metadata),
+        **fetched_metadata,
+        "web_url": str(fetched_metadata.get("web_url") or web_url),
+        "link": _metadata_str(metadata, "link") or web_url,
+    }
+    not_modified = fetched_metadata.get("status_code") == 304
+    return EntityRecord(
+        entity_type="Document",
+        platform="rss",
+        platform_entity_id=platform_entity_id,
+        title=fetched.title or (fallback.title if fallback else None),
+        content=fetched.content or (fallback.content if fallback else None),
+        created_at=fallback.created_at if fallback else None,
+        updated_at=(
+            None
+            if not_modified
+            else fetched.updated_at or (fallback.updated_at if fallback else None)
+        ),
+        metadata=rss_metadata,
+    )
+
+
+async def _hydrate_entry_document(entity: EntityRecord) -> EntityRecord:
+    if _metadata_str(entity.metadata, "web_url") is None:
+        return entity
+    try:
+        return await _fetch_entry_document(
+            entity.platform_entity_id,
+            entity.metadata,
+            fallback=entity,
+        )
+    except Exception as exc:
+        logger.warning(
+            "Skipping RSS article hydration for %s (%s: %s)",
+            entity.platform_entity_id,
+            type(exc).__name__,
+            exc,
+        )
+        logger.debug("RSS article hydration failure", exc_info=True)
+        return entity
+
+
+def _http_existing_entity(
+    existing: dict[str, Any] | None,
+    web_url: str,
+) -> dict[str, object] | None:
+    if existing is None:
+        return None
+    http_existing = dict(existing)
+    http_existing["platform_entity_id"] = web_url
+    return http_existing
+
+
+async def _fetch_http_document(
+    url: str,
+    *,
+    existing_entity: dict[str, object] | None,
+) -> EntityRecord:
+    module: Any = import_module("agentgraph_connector_web")
+    result = await module.fetch_http_document(url, existing_entity=existing_entity)
+    return cast(EntityRecord, result)
+
+
+def _entity_record_metadata(
+    metadata: Mapping[str, object],
+) -> dict[str, str | int | float | bool | None]:
+    result: dict[str, str | int | float | bool | None] = {}
+    for key, value in metadata.items():
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            result[key] = value
+    return result
+
+
+def _metadata_str(metadata: Mapping[str, object], key: str) -> str | None:
+    value = metadata.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _string_metadata(metadata: Mapping[str, object]) -> dict[str, str]:
+    return {key: value for key, value in metadata.items() if isinstance(value, str)}
+
+
+def normalise_article_url(url: str) -> str | None:
+    """Canonicalise an RSS entry URL for exact observation matching."""
+    parsed = urlsplit(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return None
+    query_items = sorted(
+        (key, value)
+        for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+        if not _is_tracking_query_key(key)
+    )
+    query = urlencode(query_items, doseq=True)
+    path = parsed.path.rstrip("/") or "/"
+    return urlunsplit((parsed.scheme.lower(), parsed.netloc.lower(), path, query, ""))
+
+
+def derive_observation_url_patterns(links_by_feed: Mapping[str, list[str]]) -> list[str]:
+    """Build bounded Chrome patterns from previously indexed article links."""
+    patterns: list[str] = []
+    seen: set[str] = set()
+    for links in links_by_feed.values():
+        candidates: dict[str, set[str]] = {}
+        for raw_link in links:
+            link = normalise_article_url(raw_link)
+            if link is None:
+                continue
+            parsed = urlsplit(link)
+            origin = f"{parsed.scheme}://{parsed.netloc}"
+            segments = [segment for segment in parsed.path.split("/") if segment]
+            for depth in range(1, len(segments)):
+                prefix = f"{origin}/{'/'.join(segments[:depth])}"
+                candidates.setdefault(prefix, set()).add(link)
+            candidates.setdefault(origin, set()).add(link)
+
+        shared = [
+            (prefix, urls)
+            for prefix, urls in candidates.items()
+            if len(urls) >= 2 and prefix.count("/") > 2
+        ]
+        useful = shared or [(prefix, urls) for prefix, urls in candidates.items() if len(urls) >= 1]
+        useful.sort(key=lambda item: (item[0].count("/"), -len(item[1]), item[0]))
+        selected_prefixes: list[str] = []
+        for prefix, _urls in useful:
+            if any(
+                prefix.startswith(f"{selected}/") or prefix == selected
+                for selected in selected_prefixes
+            ):
+                continue
+            selected_prefixes.append(prefix)
+            if len(selected_prefixes) == _MAX_OBSERVATION_PATTERNS_PER_FEED:
+                break
+
+        for prefix in selected_prefixes:
+            pattern = f"{prefix}/*"
+            if pattern not in seen:
+                patterns.append(pattern)
+                seen.add(pattern)
+    return patterns
+
+
+def _is_tracking_query_key(key: str) -> bool:
+    return key.lower().startswith("utm_") or key.lower() in _TRACKING_QUERY_KEYS
+
+
+def _entry_links(entries: Sequence[Mapping[str, object]]) -> list[str]:
+    links: list[str] = []
+    for entry in entries:
+        metadata = entry.get("metadata")
+        if not isinstance(metadata, Mapping):
+            continue
+        link = _metadata_str(metadata, "web_url") or _metadata_str(metadata, "link")
+        if link is not None:
+            links.append(link)
+    return links
+
+
+def _entry_links_by_feed(entities: list[EntityRecord]) -> dict[str, list[str]]:
+    links_by_feed: dict[str, list[str]] = {}
+    for entity in entities:
+        if entity.entity_type != "Document":
+            continue
+        feed_url = _metadata_str(entity.metadata, "feed_url")
+        link = _metadata_str(entity.metadata, "web_url") or _metadata_str(entity.metadata, "link")
+        if feed_url is not None and link is not None:
+            links_by_feed.setdefault(feed_url, []).append(link)
+    return links_by_feed
+
+
+def _entry_text(entry: dict[str, Any]) -> str:
+    content = entry.get("content")
+    if isinstance(content, list) and content:
+        first = content[0]
+        if isinstance(first, dict) and first.get("value"):
+            return str(first["value"])
+    return str(entry.get("summary") or entry.get("description") or "")
+
+
+def _parse_entry_datetime(value: object) -> datetime | None:
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        dt = parsedate_to_datetime(value)
+        return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+    except Exception:
+        return None
+
+
+def _feed_id(feed_url: str) -> str:
+    return _hash_ref(feed_url)
+
+
+def _hash_ref(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()[:24]

agentgraph_connector_rss/auth.py

@@ -0,0 +1,521 @@
+"""RSS feed configuration flow."""
+
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false
+# pyright: reportUnknownArgumentType=false
+
+from __future__ import annotations
+
+import json
+import tomllib
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import unquote, urlparse
+from xml.etree import ElementTree
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class RssConfig(BaseModel):
+    feed_urls: list[str] = Field(default_factory=list)
+
+
+class OpmlFeed(BaseModel):
+    title: str
+    feed_url: str
+    html_url: str | None = None
+
+
+_RSS_CONFIG_BEGIN = "# BEGIN AgentGraph managed RSS config"
+_RSS_CONFIG_END = "# END AgentGraph managed RSS config"
+
+
+def load_rss_settings(account_id: str | None = None) -> RssConfig:
+    _ = account_id
+    data = load_rss_config()
+    if data is None:
+        raise RuntimeError("RSS feeds not configured. Run: agentgraph connector rss add <feed-url>")
+    return RssConfig(**data)
+
+
+def load_rss_config() -> dict[str, Any] | None:
+    """Return RSS feed config."""
+    return _load_rss_config()
+
+
+def save_rss_config(data: Any) -> None:
+    payload = data.model_dump(mode="json") if hasattr(data, "model_dump") else dict(data)
+    _write_rss_config(_normalise_rss_config(payload))
+
+
+def _load_rss_config() -> dict[str, Any] | None:
+    from agentgraph.config import CONFIG_FILE, CONFIG_YAML_FILE
+
+    yaml_config = _load_rss_yaml_config(CONFIG_YAML_FILE)
+    if yaml_config is not None:
+        return yaml_config
+    toml_config = _load_rss_toml_config(CONFIG_FILE)
+    if toml_config is not None:
+        return toml_config
+    return None
+
+
+def _load_rss_toml_config(config_file: Path) -> dict[str, Any] | None:
+    if not config_file.exists():
+        return None
+    try:
+        raw = tomllib.loads(config_file.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    return _extract_rss_config(raw)
+
+
+def _load_rss_yaml_config(config_file: Path) -> dict[str, Any] | None:
+    if not config_file.exists():
+        return None
+    try:
+        raw = yaml.safe_load(config_file.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    if not isinstance(raw, dict):
+        return None
+    return _extract_rss_config(raw)
+
+
+def _extract_rss_config(raw: dict[Any, Any]) -> dict[str, Any] | None:
+    connectors = raw.get("connectors")
+    if not isinstance(connectors, dict):
+        return None
+    rss = connectors.get("rss")
+    if not isinstance(rss, dict):
+        return None
+    feed_urls = rss.get("feed_urls")
+    if isinstance(feed_urls, list):
+        return _normalise_rss_config({"feed_urls": feed_urls})
+    return None
+
+
+def _write_rss_config(config: dict[str, Any]) -> None:
+    from agentgraph.config import CONFIG_DIR, CONFIG_FILE, CONFIG_YAML_FILE
+
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    if CONFIG_YAML_FILE.exists():
+        _write_rss_yaml_config(CONFIG_YAML_FILE, config)
+        return
+    existing = CONFIG_FILE.read_text(encoding="utf-8") if CONFIG_FILE.exists() else ""
+    prefix = _strip_managed_rss_config(existing).rstrip()
+    block = _format_rss_config_block(config)
+    content = f"{prefix}\n\n{block}" if prefix else block
+    CONFIG_FILE.write_text(content, encoding="utf-8")
+
+
+def _rss_config_write_path() -> Path:
+    from agentgraph.config import CONFIG_FILE, CONFIG_YAML_FILE
+
+    return CONFIG_YAML_FILE if CONFIG_YAML_FILE.exists() else CONFIG_FILE
+
+
+def _write_rss_yaml_config(
+    config_file: Path,
+    config: dict[str, Any],
+) -> None:
+    raw: dict[str, Any]
+    if config_file.exists():
+        loaded = yaml.safe_load(config_file.read_text(encoding="utf-8"))
+        raw = loaded if isinstance(loaded, dict) else {}
+    else:
+        raw = {}
+    connectors = raw.get("connectors")
+    if not isinstance(connectors, dict):
+        connectors = {}
+    connectors["rss"] = _normalise_rss_config(config)
+    raw["connectors"] = connectors
+    config_file.write_text(
+        yaml.safe_dump(raw, sort_keys=False, allow_unicode=False),
+        encoding="utf-8",
+    )
+
+
+def _normalise_rss_config(config: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "feed_urls": [
+            str(feed_url)
+            for feed_url in config.get("feed_urls", [])
+            if isinstance(feed_url, str) and feed_url
+        ],
+    }
+
+
+def _strip_managed_rss_config(content: str) -> str:
+    begin = content.find(_RSS_CONFIG_BEGIN)
+    if begin == -1:
+        return content
+    end = content.find(_RSS_CONFIG_END, begin)
+    if end == -1:
+        return content[:begin]
+    return f"{content[:begin]}{content[end + len(_RSS_CONFIG_END) :]}"
+
+
+def _format_rss_config_block(config: dict[str, Any]) -> str:
+    lines = [
+        _RSS_CONFIG_BEGIN,
+        "[connectors.rss]",
+    ]
+    feed_urls = [
+        str(feed_url)
+        for feed_url in config.get("feed_urls", [])
+        if isinstance(feed_url, str) and feed_url
+    ]
+    lines.append(f"feed_urls = [{', '.join(_toml_string(url) for url in feed_urls)}]")
+    lines.append(_RSS_CONFIG_END)
+    return "\n".join(lines) + "\n"
+
+
+def _toml_string(value: str) -> str:
+    return json.dumps(value)
+
+
+def add_feed_urls(
+    feed_urls: list[str],
+    *,
+    validate: bool = True,
+) -> RssConfig:
+    """Add feed URLs to the configured RSS feed list and return the updated config."""
+    selected_feed_urls = [part.strip() for part in feed_urls if part.strip()]
+    if not selected_feed_urls:
+        raise ValueError("Usage: agentgraph connector rss add <feed-url> [feed-url...]")
+    if validate:
+        selected_feed_urls = resolve_feed_sources(selected_feed_urls)
+
+    try:
+        existing = load_rss_settings()
+        merged = [*existing.feed_urls]
+    except RuntimeError:
+        merged = []
+
+    for feed_url in selected_feed_urls:
+        if feed_url not in merged:
+            merged.append(feed_url)
+
+    config = RssConfig(feed_urls=merged)
+    save_rss_config(config)
+    return config
+
+
+def remove_feed_urls(
+    feed_urls: list[str],
+) -> tuple[RssConfig, list[str]]:
+    """Remove exact feed URLs from the configured RSS feed list."""
+    selected_feed_urls = [part.strip() for part in feed_urls if part.strip()]
+    if not selected_feed_urls:
+        raise ValueError("Usage: agentgraph connector rss remove <feed-url> [feed-url...]")
+
+    existing = load_rss_settings()
+    remove_set = set(selected_feed_urls)
+    updated_feed_urls = [feed_url for feed_url in existing.feed_urls if feed_url not in remove_set]
+    removed_feed_urls = [feed_url for feed_url in existing.feed_urls if feed_url in remove_set]
+    if not removed_feed_urls:
+        raise ValueError("No matching RSS feed URLs are configured for removal")
+
+    config = RssConfig(feed_urls=updated_feed_urls)
+    save_rss_config(config)
+    return config, removed_feed_urls
+
+
+def resolve_feed_sources(sources: list[str]) -> list[str]:
+    """Validate user-provided RSS/Atom feed URLs and deduplicate them."""
+    resolved: list[str] = []
+    seen: set[str] = set()
+    for source in sources:
+        feed_url = resolve_feed_source(source)
+        if feed_url not in seen:
+            seen.add(feed_url)
+            resolved.append(feed_url)
+    return resolved
+
+
+def resolve_feed_source(source: str) -> str:
+    """Return the supplied source if it is a valid RSS/Atom feed."""
+    candidate = source.strip()
+    if not candidate:
+        raise ValueError("RSS feed URL cannot be empty")
+
+    parsed = _parse_feed(candidate)
+    if _is_valid_feed(parsed):
+        return candidate
+    raise ValueError(f"Not a valid RSS/Atom feed: {candidate}")
+
+
+def _parse_feed(feed_url: str) -> Any:
+    import feedparser  # type: ignore[import-untyped]
+
+    return feedparser.parse(feed_url)
+
+
+def _is_valid_feed(parsed: Any) -> bool:
+    version = str(getattr(parsed, "version", "") or "")
+    if not version:
+        return False
+    feed = cast(dict[str, Any], getattr(parsed, "feed", {}) or {})
+    entries = cast(list[Any], getattr(parsed, "entries", []) or [])
+    return bool(feed or entries)
+
+
+def parse_opml_feeds(path: str | Path) -> list[OpmlFeed]:
+    """Parse feed outlines from an OPML file."""
+    source_path = _opml_source_path(path)
+    if not source_path.exists():
+        raise FileNotFoundError(f"OPML file not found: {source_path}")
+
+    try:
+        root = ElementTree.parse(source_path).getroot()
+    except ElementTree.ParseError as exc:
+        raise ValueError(f"Could not parse OPML file: {exc}") from exc
+
+    feeds: list[OpmlFeed] = []
+    seen: set[str] = set()
+    for outline in root.iter("outline"):
+        feed_url = (outline.attrib.get("xmlUrl") or outline.attrib.get("xmlurl") or "").strip()
+        if not feed_url or feed_url in seen:
+            continue
+        seen.add(feed_url)
+        title = (
+            outline.attrib.get("title")
+            or outline.attrib.get("text")
+            or outline.attrib.get("description")
+            or feed_url
+        )
+        html_url = (outline.attrib.get("htmlUrl") or outline.attrib.get("htmlurl") or "").strip()
+        feeds.append(OpmlFeed(title=title.strip(), feed_url=feed_url, html_url=html_url or None))
+    return feeds
+
+
+def _opml_source_path(path: str | Path) -> Path:
+    return _local_file_path(path, kind="OPML")
+
+
+def _local_file_path(path: str | Path, *, kind: str) -> Path:
+    raw_path = str(path)
+    parsed = urlparse(raw_path)
+    if parsed.scheme == "file":
+        if parsed.netloc not in ("", "localhost"):
+            raise ValueError(f"Unsupported {kind} file URI host: {parsed.netloc}")
+        return Path(unquote(parsed.path)).expanduser()
+    return Path(path).expanduser()
+
+
+def select_opml_feeds(
+    feeds: list[OpmlFeed],
+    *,
+    include_all: bool = False,
+    selection: str | None = None,
+    configured_feed_urls: list[str] | None = None,
+) -> list[OpmlFeed]:
+    if include_all:
+        return feeds
+    if selection is not None:
+        selected_indexes = _parse_feed_selection(selection, len(feeds))
+        return [feeds[index - 1] for index in selected_indexes]
+    return _prompt_for_opml_feeds(feeds, configured_feed_urls=configured_feed_urls or [])
+
+
+def import_opml_feeds(
+    path: str | Path,
+    *,
+    include_all: bool = False,
+    selection: str | None = None,
+) -> tuple[RssConfig, list[OpmlFeed], list[OpmlFeed]]:
+    feeds = parse_opml_feeds(path)
+    if not feeds:
+        raise ValueError("No RSS/Atom feeds found in OPML file")
+
+    configured_feed_urls: list[str] = []
+    try:
+        configured_feed_urls = load_rss_settings().feed_urls
+    except RuntimeError:
+        configured_feed_urls = []
+
+    selected_feeds = select_opml_feeds(
+        feeds,
+        include_all=include_all,
+        selection=selection,
+        configured_feed_urls=configured_feed_urls,
+    )
+    if not selected_feeds:
+        raise ValueError("No feeds selected")
+
+    config = add_feed_urls([feed.feed_url for feed in selected_feeds])
+    return config, feeds, selected_feeds
+
+
+def _parse_feed_selection(selection: str, feed_count: int) -> list[int]:
+    indexes: list[int] = []
+    seen: set[int] = set()
+    for raw_part in selection.split(","):
+        part = raw_part.strip()
+        if not part:
+            continue
+        if "-" in part:
+            start_raw, end_raw = part.split("-", 1)
+            try:
+                start = int(start_raw.strip())
+                end = int(end_raw.strip())
+            except ValueError as exc:
+                raise ValueError(f"Invalid feed selection: {selection}") from exc
+            if start > end:
+                raise ValueError(f"Invalid feed selection range: {part}")
+            candidates = range(start, end + 1)
+        else:
+            try:
+                candidates = range(int(part), int(part) + 1)
+            except ValueError as exc:
+                raise ValueError(f"Invalid feed selection: {selection}") from exc
+        for index in candidates:
+            if index < 1 or index > feed_count:
+                raise ValueError(f"Feed selection {index} is out of range 1-{feed_count}")
+            if index not in seen:
+                seen.add(index)
+                indexes.append(index)
+    return indexes
+
+
+def _prompt_for_opml_feeds(
+    feeds: list[OpmlFeed],
+    *,
+    configured_feed_urls: list[str],
+) -> list[OpmlFeed]:
+    import sys
+
+    if not sys.stdin.isatty():
+        raise ValueError("Use --all or --select <indexes> when importing OPML non-interactively")
+
+    try:
+        return _checkbox_select_opml_feeds(feeds, configured_feed_urls=configured_feed_urls)
+    except ImportError:
+        return _prompt_for_opml_feeds_numeric(feeds)
+
+
+def _checkbox_select_opml_feeds(
+    feeds: list[OpmlFeed],
+    *,
+    configured_feed_urls: list[str],
+) -> list[OpmlFeed]:
+    import questionary  # type: ignore[import-untyped]
+
+    configured = set(configured_feed_urls)
+    choices = [
+        questionary.Choice(
+            title=f"{feed.title} ({feed.feed_url})",
+            value=feed.feed_url,
+            checked=feed.feed_url in configured,
+        )
+        for feed in feeds
+    ]
+    selected_urls = questionary.checkbox(
+        f"Select feeds to add ({len(feeds)} found):",
+        choices=choices,
+    ).ask()
+    if selected_urls is None:
+        raise ValueError("No feeds selected")
+    selected = set(cast(list[str], selected_urls))
+    return [feed for feed in feeds if feed.feed_url in selected]
+
+
+def _prompt_for_opml_feeds_numeric(feeds: list[OpmlFeed]) -> list[OpmlFeed]:
+    import typer
+
+    typer.echo(f"Found {len(feeds)} feed(s) in OPML:")
+    for index, feed in enumerate(feeds, start=1):
+        typer.echo(f"  {index}. {feed.title} - {feed.feed_url}")
+    if typer.confirm("Add all feeds?", default=True):
+        return feeds
+    selection = typer.prompt("Feed numbers to add (comma-separated, ranges allowed)").strip()
+    return select_opml_feeds(feeds, selection=selection)
+
+
+async def preview_feed(feed_url: str, *, count: int = 3) -> dict[str, Any]:
+    """Fetch and parse a small preview from an RSS/Atom feed URL."""
+    import asyncio
+
+    import feedparser  # type: ignore[import-untyped]
+
+    parsed: Any = await asyncio.to_thread(feedparser.parse, feed_url)
+    entries = [
+        {
+            "title": str(
+                entry.get("title") or entry.get("id") or entry.get("link") or "(untitled)"
+            ),
+            "link": str(entry.get("link") or ""),
+        }
+        for entry in cast(list[Any], parsed.entries)[: max(1, min(count, 50))]
+    ]
+    feed = cast(dict[str, Any], parsed.feed)
+    return {
+        "title": str(feed.get("title") or feed_url),
+        "feed_url": feed_url,
+        "entries": entries,
+        "bozo": bool(parsed.bozo),
+        "bozo_exception": str(parsed.bozo_exception) if parsed.bozo else None,
+    }
+
+
+async def verify_rss_auth(account_id: str | None = None) -> tuple[str, str | None]:
+    try:
+        settings = load_rss_settings(account_id)
+    except RuntimeError:
+        return ("missing", None)
+
+    if not settings.feed_urls:
+        return ("invalid", "No RSS feed URLs configured")
+
+    try:
+        preview = await preview_feed(settings.feed_urls[0], count=1)
+    except Exception as exc:
+        return ("invalid", str(exc))
+
+    entry_count = len(cast(list[object], preview.get("entries") or []))
+    if preview.get("bozo") and entry_count == 0:
+        return ("invalid", str(preview.get("bozo_exception") or "Feed could not be parsed"))
+    return ("ok", f"{len(settings.feed_urls)} feed(s), sample returned {entry_count} article(s)")
+
+
+def run_rss_flow(
+    account_id: str | None = None,
+    add: bool = False,
+) -> None:
+    _ = (account_id, add)
+    import asyncio
+
+    import typer
+
+    typer.echo(
+        "\n"
+        "RSS setup does not need authentication. Provide one or more RSS/Atom feed URLs,\n"
+        "and AgentGraph will fetch those feeds directly.\n"
+    )
+    raw_feeds: str = typer.prompt("RSS/Atom feed URLs (comma-separated)").strip()
+    selected_feed_urls = resolve_feed_sources(
+        [part.strip() for part in raw_feeds.split(",") if part.strip()]
+    )
+
+    config = RssConfig(feed_urls=selected_feed_urls)
+    save_rss_config(config)
+
+    typer.echo(f"\nRSS feeds saved to {_rss_config_write_path()}")
+    if selected_feed_urls:
+        typer.echo("Checking configured feed(s)...")
+
+        async def _check() -> None:
+            status, detail = await verify_rss_auth()
+            if status == "ok":
+                typer.echo(f"RSS feed check passed: {detail}")
+                preview = await preview_feed(selected_feed_urls[0], count=3)
+                entries = cast(list[dict[str, str]], preview.get("entries") or [])
+                if entries:
+                    typer.echo("Sample articles from first feed:")
+                    for item in entries[:3]:
+                        typer.echo(f"  - {item['title']}")
+            else:
+                typer.echo(f"RSS feed check failed: {detail or status}")
+
+        asyncio.run(_check())

agentgraph_connector_rss-0.5.0.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: agentgraph-connector-rss
+Version: 0.5.0
+Summary: RSS and Atom connector for AgentGraph
+Requires-Python: >=3.12
+Requires-Dist: agentgraph-server<0.6,>=0.5.0
+Requires-Dist: agentgraph-connector-web<0.6,>=0.5.0
+Requires-Dist: feedparser>=6.0.11
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: questionary>=2.1.1

agentgraph_connector_rss-0.5.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+agentgraph_connector_rss/__init__.py,sha256=-7kXcAl5zmknz6zP031f-e1S4VDEHTbskCEJ4_8mwcE,28129
+agentgraph_connector_rss/auth.py,sha256=uk97Ka3fC_ognY3Sf-RQEREYGESB4g0VYJM_29vv8FI,16899
+agentgraph_connector_rss-0.5.0.dist-info/METADATA,sha256=_QVJjNaVR-FKlDnM5AiWuCIrAI8i_8iI2JaoRze7xFE,362
+agentgraph_connector_rss-0.5.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+agentgraph_connector_rss-0.5.0.dist-info/entry_points.txt,sha256=1lY1JBweReupuaRXl1iEo3as6fym1v5XjszuJ3vUXiA,68
+agentgraph_connector_rss-0.5.0.dist-info/top_level.txt,sha256=SYROHVGUSvNTdW_lVwjOun_MWBGkI-4iFg6Yc52f3wo,25
+agentgraph_connector_rss-0.5.0.dist-info/RECORD,,

agentgraph_connector_rss-0.5.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

agentgraph_connector_rss-0.5.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[agentgraph.connectors]
+rss = agentgraph_connector_rss:RssConnector

agentgraph_connector_rss-0.5.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+agentgraph_connector_rss