feed-all 0.1.0__py3-none-any.whl

feed_all-0.1.0.dist-info/METADATA

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: feed-all
+Version: 0.1.0
+Summary: All-in-one RSS/feed MCP Plugin — subscribe, fetch, browse, manage feeds, track webpage changes, health check, OPML import/export
+Author-email: EntroFeed Team <wuyuezhang1984@gmail.com>
+License: MIT
+Keywords: rss,feed,mcp,claude-code,cline
+Requires-Python: >=3.11
+Requires-Dist: feedparser>=6.0.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: mcp>=1.0.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0

feed_all-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+rss_plugin/__init__.py,sha256=MphEzKmeymA9Gl_lZYiiyQdtlR5PKVoX9-WPHhnoAeI,136
+rss_plugin/__main__.py,sha256=dubPYx1yCEDu-nHC8qYwUYyqDDZi4tnJpYIrXuLEd2M,116
+rss_plugin/discovery.py,sha256=QfdlX-Mk5onwTnTxCMo6TOBphop2RHu6CHaBgb9m0U8,5801
+rss_plugin/fetcher.py,sha256=5C41HVFFyxTT-4p-bP5F6HPRhSeybLaTXxlQEhiP05k,8125
+rss_plugin/models.py,sha256=IvcEroflRQYjnTgRtcP6G8lccOOyA8A90o4ntEledw4,3356
+rss_plugin/server.py,sha256=O0blSiyqko4SeksYE5bIylL2evDy_87y1sG6coMu4vs,27084
+rss_plugin/storage.py,sha256=r9QdntgOpjSQ-dJ6kJf0uwVjZCBG0HxSCqBvlE4LA9k,17461
+rss_plugin/utils.py,sha256=vot0d5Fk7PCw-L0uEvNpGUg2PDXAgsCdmJR2JvN4MHE,4354
+rss_plugin/webpage_parsers/__init__.py,sha256=bR7IY7xFfMrVrPM3stJgCQQ39PGfK8Iw4V7A2EVL0Ow,325
+rss_plugin/webpage_parsers/base.py,sha256=9MFaHPpw-hVCWU969ugttCxnOhJX_6Bq1EYWwvVEOcI,740
+rss_plugin/webpage_parsers/selector_parser.py,sha256=V5cn66mpi1KkKtrF_AqYCHBNys3o_SGSK4px0SFncAA,4819
+rss_plugin/webpage_parsers/site_registry.py,sha256=vIEwnmjQ2lhPcaSpDtz0HpmNCfib_-fL2a7HXnHvVIo,1985
+feed_all-0.1.0.dist-info/METADATA,sha256=UylSZaIlwvNCtLK-VFMT8Lfp1jcQSB_iAiIGue3-VPM,508
+feed_all-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+feed_all-0.1.0.dist-info/entry_points.txt,sha256=GJKXRSndZqEyYTPjNDkTnLTy4W-w3c3QZxPa1cKUSgw,58
+feed_all-0.1.0.dist-info/top_level.txt,sha256=7A1hA1z9pLknNfBOASpzYXzMTBrsAA_jimYjqurqUwk,11
+feed_all-0.1.0.dist-info/RECORD,,

feed_all-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

feed_all-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+feed-all = rss_plugin.server:run_server

feed_all-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+rss_plugin

rss_plugin/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+"""Standalone RSS MCP Plugin — subscribe, fetch, and manage RSS feeds via MCP tools."""
+
+__version__ = "0.1.0"

rss_plugin/__main__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""Run the RSS MCP plugin server."""
+
+from rss_plugin.server import run_server
+
+run_server()

rss_plugin/discovery.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+"""Standalone RSS Plugin — feed discovery.
+
+Supports three methods (from most to least preferred):
+1. Direct RSS check — URL is already an RSS/Atom feed
+2. Basic webpage scan — fetch HTML, extract <link> tags with feed MIME types
+3. Fuzzy URL check — common RSS path patterns
+"""
+
+import logging
+from typing import Any, Dict, Optional
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from rss_plugin.fetcher import check_feed as _check_feed
+from rss_plugin.models import FeedDiscoveryResult
+from rss_plugin.utils import extract_rss_links
+
+logger = logging.getLogger(__name__)
+
+_USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.6778.86 Safari/537.36"
+)
+
+# Common feed URL path segments
+FEED_PATH_SEGMENTS = {"feed", "rss", "atom", "feeds", "xml"}
+FEED_PATH_ENDINGS = (".rss", ".atom", "feed.xml", "rss.xml", "atom.xml", "index.xml")
+
+# Common feed-like URLs to try if direct scan fails
+COMMON_FEED_PATHS = [
+    "/feed",
+    "/feed/",
+    "/rss",
+    "/rss/",
+    "/atom.xml",
+    "/feed.xml",
+    "/rss.xml",
+    "/index.xml",
+    "/feeds/posts/default",  # Blogger
+    "/posts/default",  # Blogger
+]
+
+# Actually import the real check function
+async def _direct_rss_check(url: str) -> Optional[Dict[str, Any]]:
+    """Check if a URL is a valid RSS/Atom feed."""
+    try:
+        result = await _check_feed(url)
+        return result
+    except Exception as e:
+        logger.warning("Direct RSS check failed for %s: %s", url, e)
+        return None
+
+
+async def _webpage_scan(url: str) -> Optional[Dict[str, Any]]:
+    """Fetch HTML page and extract RSS/Atom feed links."""
+    async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client:
+        try:
+            response = await client.get(
+                url,
+                headers={"User-Agent": _USER_AGENT},
+            )
+            response.raise_for_status()
+        except Exception as e:
+            logger.warning("Webpage scan failed for %s: %s", url, e)
+            return None
+
+        html_content = response.text
+        if not html_content:
+            return None
+
+        rss_links = extract_rss_links(html_content, str(response.url))
+        if not rss_links:
+            return None
+
+        for feed_url in rss_links:
+            try:
+                result = await _check_feed(feed_url)
+                if result and result.get("is_valid"):
+                    result["feed_url"] = feed_url
+                    return result
+            except Exception:
+                continue
+
+        return None
+
+
+async def _try_common_feed_paths(base_url: str) -> Optional[Dict[str, Any]]:
+    """Try common RSS feed paths against a base URL."""
+    parsed = urlparse(base_url)
+    base = f"{parsed.scheme}://{parsed.netloc}"
+
+    for path in COMMON_FEED_PATHS:
+        feed_url = urljoin(base, path)
+        try:
+            result = await _check_feed(feed_url)
+            if result and result.get("is_valid"):
+                result["feed_url"] = feed_url
+                return result
+        except Exception:
+            continue
+
+    return None
+
+
+def _is_likely_feed_url(url: str) -> bool:
+    """Heuristic check if a URL looks like a feed URL."""
+    url_lower = url.lower().strip()
+    parsed = urlparse(url_lower)
+    path = parsed.path
+
+    if any(path.endswith(e) for e in FEED_PATH_ENDINGS):
+        return True
+
+    segments = [s for s in path.split("/") if s]
+    if segments and any(s in FEED_PATH_SEGMENTS for s in segments):
+        return True
+
+    if "rss" in parsed.query or "atom" in parsed.query:
+        return True
+
+    return False
+
+
+async def discover(url: str) -> FeedDiscoveryResult:
+    """Discover RSS feeds from a URL.
+
+    Priority:
+    1. Direct RSS check
+    2. Webpage scan (<link> tags)
+    3. Common feed paths
+    4. Fuzzy URL check (report if URL looks like a feed but wasn't validated)
+    """
+    # 1. Direct RSS check
+    if _is_likely_feed_url(url) or url.endswith(".xml"):
+        direct = await _direct_rss_check(url)
+        if direct and direct.get("is_valid"):
+            return FeedDiscoveryResult(
+                success=True,
+                feed_url=url,
+                source="direct_rss",
+                feed_title=direct.get("title", ""),
+                feed_type=direct.get("feed_type", "rss"),
+                article_count=direct.get("article_count", 0),
+            )
+
+    # 2. Try as webpage (extract <link> tags)
+    webpage = await _webpage_scan(url)
+    if webpage:
+        return FeedDiscoveryResult(
+            success=True,
+            feed_url=webpage.get("feed_url", url),
+            source="direct_rss",
+            feed_title=webpage.get("title", ""),
+            feed_type=webpage.get("feed_type", "rss"),
+            article_count=webpage.get("article_count", 0),
+        )
+
+    # 3. Try common feed paths
+    common = await _try_common_feed_paths(url)
+    if common:
+        return FeedDiscoveryResult(
+            success=True,
+            feed_url=common.get("feed_url", ""),
+            source="direct_rss",
+            feed_title=common.get("title", ""),
+            feed_type=common.get("feed_type", "rss"),
+            article_count=common.get("article_count", 0),
+        )
+
+    # 4. If URL looks like a feed but we couldn't validate, report it
+    if _is_likely_feed_url(url):
+        return FeedDiscoveryResult(
+            success=False,
+            feed_url=url,
+            source="unknown",
+            error_message="URL looks like a feed but could not be validated",
+        )
+
+    return FeedDiscoveryResult(
+        success=False,
+        error_message=(
+            "Could not discover an RSS feed. "
+            "Try providing a direct feed URL (e.g., .../feed.xml or .../rss)"
+        ),
+    )

rss_plugin/fetcher.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+"""Standalone RSS Plugin — feed fetching and parsing.
+
+Supports:
+- RSS/Atom feed fetching via feedparser
+- Webpage feed fetching (page_snapshot, list_extract, site_parser)
+"""
+
+import json
+from calendar import timegm
+from datetime import datetime, timezone
+from hashlib import md5
+from typing import Any, Dict, List, Optional, Tuple
+
+import feedparser
+import httpx
+
+from rss_plugin.utils import clean_html_text, extract_readable_content, timestamp_now
+
+_USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.6778.86 Safari/537.36"
+)
+
+
+def _get_entry_time_raw(entry: Any) -> int:
+    """Get entry timestamp from feedparser parsed entry, with fallbacks."""
+    if hasattr(entry, "published_parsed") and entry.published_parsed:
+        return timegm(entry.published_parsed)
+    if hasattr(entry, "updated_parsed") and entry.updated_parsed:
+        return timegm(entry.updated_parsed)
+    return timestamp_now()
+
+
+async def fetch_feed(url: str, timeout: int = 30) -> Dict[str, Any]:
+    """Fetch and parse an RSS/Atom feed URL.
+
+    Returns:
+        Dict with keys: success, title, description, feed_type, entry_count, entries, error
+    """
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+        try:
+            response = await client.get(
+                url, headers={"User-Agent": _USER_AGENT},
+            )
+            response.raise_for_status()
+        except httpx.TimeoutException:
+            return {"success": False, "error": f"Request timed out after {timeout}s",
+                    "title": "", "description": "", "feed_type": "", "entries": []}
+        except httpx.HTTPStatusError as e:
+            return {"success": False, "error": f"HTTP {e.response.status_code}: {e.response.reason_phrase}",
+                    "title": "", "description": "", "feed_type": "", "entries": []}
+        except httpx.RequestError as e:
+            return {"success": False, "error": f"Request failed: {e}",
+                    "title": "", "description": "", "feed_type": "", "entries": []}
+
+        content = response.text
+        if not content or len(content.strip()) < 50:
+            return {"success": False, "error": "Empty or too short response",
+                    "title": "", "description": "", "feed_type": "", "entries": []}
+
+    parsed = feedparser.parse(content)
+    if parsed.bozo and not parsed.entries:
+        bozo_exception = parsed.bozo_exception
+        return {"success": False, "error": f"Invalid feed: {bozo_exception}" if bozo_exception else "Invalid feed XML",
+                "title": "", "description": "", "feed_type": "", "entries": []}
+
+    feed_meta = parsed.feed
+    title = getattr(feed_meta, "title", "") or ""
+    description = getattr(feed_meta, "description", "") or ""
+
+    feed_type = "rss"
+    version = getattr(parsed, "version", "")
+    if version and "atom" in str(version).lower():
+        feed_type = "atom"
+
+    entries = []
+    for entry in parsed.entries:
+        entry_title = getattr(entry, "title", "") or ""
+        entry_link = getattr(entry, "link", "") or ""
+        published_time = _get_entry_time_raw(entry)
+
+        entry_content = ""
+        if hasattr(entry, "content") and entry.content:
+            entry_content = "".join(c.get("value", "") for c in entry.content if isinstance(c, dict))
+        elif hasattr(entry, "summary"):
+            entry_content = entry.summary or ""
+
+        summary = getattr(entry, "summary", "") or ""
+
+        authors = []
+        if hasattr(entry, "authors") and entry.authors:
+            authors = [a.get("name", "") for a in entry.authors if isinstance(a, dict)]
+        elif hasattr(entry, "author") and entry.author:
+            authors = [entry.author]
+
+        entries.append({
+            "title": entry_title,
+            "url": entry_link,
+            "published_at": published_time,
+            "updated_at": published_time,
+            "content": entry_content,
+            "preview": summary,
+            "authors": json.dumps(authors, ensure_ascii=False),
+        })
+
+    return {"success": True, "title": title, "description": description,
+            "feed_type": feed_type, "entry_count": len(entries),
+            "entries": entries, "error": ""}
+
+
+async def check_feed(url: str, timeout: int = 30) -> Dict[str, Any]:
+    """Quick check if a URL is a valid RSS/Atom feed."""
+    result = await fetch_feed(url, timeout=timeout)
+    if not result["success"]:
+        return {"is_valid": False, "title": "", "feed_type": "", "article_count": 0, "error": result["error"]}
+    return {"is_valid": True, "title": result["title"], "feed_type": result["feed_type"],
+            "article_count": result["entry_count"], "error": ""}
+
+
+# ── Webpage Feed Support ──
+
+
+async def fetch_webpage_html(url: str, timeout: int = 30) -> Optional[str]:
+    """Fetch raw HTML from a URL using httpx."""
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+        try:
+            response = await client.get(url, headers={"User-Agent": _USER_AGENT})
+            response.raise_for_status()
+            return response.text
+        except Exception:
+            return None
+
+
+def classify_webpage_response(html_content: Optional[str]) -> Optional[str]:
+    """Classify a webpage response. Returns None if OK, or a classification string."""
+    if not html_content:
+        return "empty"
+    lowered = html_content.lower()
+    challenge_signals = (
+        "captcha", "verify you are human", "access denied",
+        "cloudflare", "security check", "challenge",
+    )
+    if any(signal in lowered for signal in challenge_signals):
+        return "challenge"
+    cleaned = clean_html_text(extract_readable_content(html_content))
+    if len(cleaned) < 200 and len(html_content) > 5000:
+        return "challenge"
+    if len(html_content) < 500:
+        return "thin"
+    return None
+
+
+async def fetch_page_snapshot(url: str) -> Tuple[bool, Optional[str], Optional[str]]:
+    """Fetch a webpage and extract readable content as a snapshot.
+
+    Returns:
+        (changed, cleaned_text, entry_url)
+        changed: True if content differs from previous hash
+    """
+    html_content = await fetch_webpage_html(url)
+    if not html_content:
+        return False, None, None
+
+    readable = extract_readable_content(html_content)
+    cleaned = clean_html_text(readable)
+    if not cleaned:
+        return False, None, None
+
+    content_hash = md5(cleaned.encode("utf-8")).hexdigest()
+    entry_url = f"{url}#snapshot-{content_hash}"
+    return True, cleaned, entry_url
+
+
+async def fetch_list_extract(
+    url: str,
+    parser_config: Dict[str, Any],
+) -> List[Dict[str, Any]]:
+    """Fetch a webpage and extract a list of items using CSS selectors.
+
+    Returns:
+        List of dicts with title, url, published_at, preview
+    """
+    from rss_plugin.webpage_parsers import extract_with_selectors, get_site_parser
+
+    html_content = await fetch_webpage_html(url)
+    if not html_content:
+        return []
+
+    strategy = parser_config.get("strategy", "list_extract")
+    items = []
+
+    if strategy == "site_parser":
+        parser = get_site_parser(parser_config.get("parser_id"))
+        if parser:
+            parsed = parser(html_content, url, parser_config)
+        else:
+            parsed = []
+    elif parser_config.get("selectors"):
+        parsed = extract_with_selectors(html_content, url, parser_config)
+    else:
+        parser = get_site_parser(parser_config.get("parser_id") or "generic_news_list")
+        parsed = parser(html_content, url, parser_config) if parser else []
+
+    for item in parsed:
+        items.append({
+            "title": item.title,
+            "url": item.url,
+            "published_at": item.published_at or timestamp_now(),
+            "updated_at": item.updated_at or timestamp_now(),
+            "content": item.content or "",
+            "preview": item.summary or "",
+            "authors": json.dumps(item.authors, ensure_ascii=False),
+            "entry_type": "webpage_list",
+        })
+
+    return items

rss_plugin/models.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""Standalone RSS Plugin — Pydantic models.
+
+Feed model now includes health tracking fields (merged from poll_state).
+"""
+
+from hashlib import md5
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, field_validator
+
+
+class Feed(BaseModel):
+    """RSS/webpage feed model with embedded health tracking."""
+
+    name: str
+    url: str
+    category: str = "uncategorized"
+    type: str = "rss"  # "rss", "atom", "webpage", "crawl4ai"
+    refresh_enabled: bool = True
+    preview_only: bool = False
+    use_script: bool = False
+    retrieve_content: bool = True
+    check_interval_units: int = 1
+    parser_config: Optional[Dict[str, Any]] = None
+    authority_score: Optional[float] = None
+    source_category: Optional[str] = None
+    tags: List[str] = []
+    domains: List[str] = []
+    requires_vpn: bool = False
+    added_at: int = 0
+    # Health tracking (merged from poll_state)
+    last_polled_at: Optional[int] = None
+    start_ts: Optional[int] = None
+    consecutive_failures: int = 0
+    last_error: Optional[str] = None
+    auto_disabled: bool = False
+
+    @property
+    def id(self) -> str:
+        return md5(self.url.encode()).hexdigest()
+
+    def is_webpage(self) -> bool:
+        return self.type in {"webpage", "crawl4ai"}
+
+
+class FeedEntry(BaseModel):
+    """RSS feed entry model."""
+
+    feed_id: str
+    title: str
+    url: str
+    published_at: int
+    updated_at: int = 0
+    content: Optional[str] = None
+    preview: Optional[str] = None
+    authors: str = "[]"  # JSON-encoded list
+    is_read: bool = False
+    entry_type: str = "rss"  # "rss", "webpage_snapshot", "webpage_list"
+
+    @field_validator("title")
+    @classmethod
+    def _title_not_blank(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.strip():
+            raise ValueError("title must be a non-empty string")
+        return v
+
+    @field_validator("url")
+    @classmethod
+    def _url_must_be_http_like(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.strip():
+            raise ValueError("url must be a non-empty string")
+        if not (v.startswith("http://") or v.startswith("https://")):
+            raise ValueError("url must start with http:// or https://")
+        return v
+
+    @field_validator("published_at", "updated_at", mode="before")
+    @classmethod
+    def _ts_must_be_non_negative_int(cls, v: Any) -> int:
+        if isinstance(v, bool) or not isinstance(v, int) or v < 0:
+            raise ValueError(f"timestamp must be a non-negative int (got {v!r})")
+        return v
+
+    @property
+    def id(self) -> str:
+        return md5(self.url.encode()).hexdigest()
+
+
+class EntryContent(BaseModel):
+    """Full article content."""
+
+    entry_id: str
+    url: str
+    content: Optional[str] = None
+    summary: Optional[str] = None
+    unretrievable: bool = False
+
+    @property
+    def id(self) -> str:
+        return md5(self.url.encode()).hexdigest()
+
+
+class FeedDiscoveryResult(BaseModel):
+    """Feed discovery result model."""
+
+    success: bool
+    feed_url: Optional[str] = None
+    source: str = "unknown"  # "direct_rss", "webpage", "unknown"
+    feed_title: str = ""
+    feed_type: str = ""  # "rss", "atom", "unknown"
+    article_count: int = 0
+    latest_article_date: Optional[str] = None
+    error_message: str = ""
+    attempts: List[Dict[str, Any]] = []