fable-engine 1.3.1__py3-none-any.whl

fable_engine/scrapers/reddit.py

@@ -0,0 +1,154 @@
+"""
+Reddit threads and subreddits scraper implementation using Reddit's public JSON interface (.json).
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.parse
+from typing import Optional
+
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url
+
+
+class RedditScraper(ResearchSource):
+    source_type = "reddit"
+
+    def fetch(self, target: str, timeout: int = 15, max_content_length: int = 10000) -> ResearchResult:
+        target = target.strip()
+        if not target:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url="",
+                error="Target Reddit URL, subreddit, or search query cannot be empty."
+            )
+
+        if target.startswith("http://") or target.startswith("https://"):
+            clean_url = target.split("?")[0].rstrip("/")
+            url = clean_url + ".json"
+            canonical_url = clean_url
+        elif target.startswith("r/") or target.startswith("/r/"):
+            sub_path = target.lstrip("/").rstrip("/")
+            url = f"https://www.reddit.com/{sub_path}/hot.json?limit=10"
+            canonical_url = f"https://www.reddit.com/{sub_path}"
+        else:
+            query_enc = urllib.parse.quote(target)
+            url = f"https://www.reddit.com/search.json?q={query_enc}&limit=10"
+            canonical_url = f"https://www.reddit.com/search?q={query_enc}"
+
+        try:
+            json_str = fetch_url(url, timeout=timeout)
+            data = json.loads(json_str)
+
+            # Handle single thread post response (list of listings)
+            if isinstance(data, list) and len(data) >= 1 and isinstance(data[0], dict):
+                children = data[0].get("data", {}).get("children", [])
+                if not children or not isinstance(children[0], dict):
+                    return ResearchResult(
+                        ok=False,
+                        source_type=self.source_type,
+                        canonical_url=canonical_url,
+                        error="Reddit post data structure was empty or unexpected."
+                    )
+
+                post_data = children[0].get("data", {})
+                title = post_data.get("title", "Reddit Post")
+                subreddit = post_data.get("subreddit", "")
+                author = post_data.get("author", "[deleted]")
+                score = post_data.get("score", 0)
+                selftext = post_data.get("selftext", "")
+                permalink = post_data.get("permalink", "")
+                post_url = f"https://www.reddit.com{permalink}" if permalink else canonical_url
+
+                content_blocks = [
+                    f"**Subreddit**: r/{subreddit} | **Author**: u/{author} | **Score**: {score}\n"
+                ]
+                if selftext:
+                    content_blocks.append(f"## Post Text\n{selftext}\n")
+
+                if len(data) >= 2 and isinstance(data[1], dict):
+                    comments = data[1].get("data", {}).get("children", [])
+                    comment_lines = ["## Top Comments\n"]
+                    count = 0
+                    for c in comments:
+                        if count >= 10:
+                            break
+                        if isinstance(c, dict):
+                            cdata = c.get("data", {})
+                            c_author = cdata.get("author", "")
+                            c_body = cdata.get("body", "")
+                            c_score = cdata.get("score", 0)
+                            if c_body:
+                                comment_lines.append(f"### u/{c_author} ({c_score} points)")
+                                comment_lines.append(f"{c_body}\n")
+                                count += 1
+                    if count > 0:
+                        content_blocks.extend(comment_lines)
+
+                full_content = "\n".join(content_blocks)
+                if len(full_content) > max_content_length:
+                    full_content = full_content[:max_content_length] + "\n\n*(Content truncated)*"
+
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=post_url,
+                    title=title,
+                    author=f"u/{author}",
+                    content=full_content,
+                    metadata={"subreddit": subreddit, "score": score}
+                )
+
+            # Handle subreddit listing or search results
+            elif isinstance(data, dict) and "data" in data:
+                children = data.get("data", {}).get("children", [])
+                if not children:
+                    return ResearchResult(
+                        ok=True,
+                        source_type=self.source_type,
+                        canonical_url=canonical_url,
+                        title=f"Reddit Listing: '{target}'",
+                        content="*(No posts found for this subreddit or query)*"
+                    )
+
+                items = ["## Posts\n"]
+                for child in children[:10]:
+                    if isinstance(child, dict):
+                        p = child.get("data", {})
+                        p_title = p.get("title", "")
+                        p_sub = p.get("subreddit", "")
+                        p_author = p.get("author", "")
+                        p_score = p.get("score", 0)
+                        p_comments = p.get("num_comments", 0)
+                        p_permalink = f"https://www.reddit.com{p.get('permalink', '')}"
+                        p_text = p.get("selftext", "")[:250]
+                        text_str = f"\n  _{p_text.strip()}..._" if p_text else ""
+                        items.append(f"- **[{p_title}]({p_permalink})**")
+                        items.append(f"  *r/{p_sub} | u/{p_author} | Score: {p_score} | Comments: {p_comments}*{text_str}\n")
+
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=canonical_url,
+                    title=f"Reddit Listing: '{target}'",
+                    content="\n".join(items)
+                )
+
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                error="Unexpected JSON response format from Reddit API."
+            )
+        except Exception as e:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                error=f"Reddit fetch failed: {str(e)}"
+            )
+
+
+def scrape_reddit(target: str, timeout: int = 15) -> ResearchResult:
+    return RedditScraper().fetch(target, timeout=timeout)

fable_engine/scrapers/web.py

@@ -0,0 +1,120 @@
+"""
+Web page and search query scraper implementation.
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from typing import Optional
+
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, SimpleHTMLTextExtractor, fetch_url
+
+
+class WebScraper(ResearchSource):
+    source_type = "web"
+
+    def fetch(self, target: str, timeout: int = 15, max_content_length: int = 12000) -> ResearchResult:
+        target = target.strip()
+        if not target:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url="",
+                error="Target URL or query string cannot be empty."
+            )
+
+        if target.startswith("http://") or target.startswith("https://"):
+            try:
+                html = fetch_url(target, timeout=timeout)
+                parser = SimpleHTMLTextExtractor()
+                parser.feed(html)
+                title = parser.title.strip() or target
+                content = parser.get_markdown()
+                if len(content) > max_content_length:
+                    content = content[:max_content_length] + "\n\n*(Content truncated for size)*"
+
+                links_md = ""
+                if parser.links:
+                    seen = set()
+                    link_lines = []
+                    for text, href in parser.links[:15]:
+                        full_url = urllib.parse.urljoin(target, href)
+                        if full_url not in seen and text:
+                            seen.add(full_url)
+                            link_lines.append(f"- [{text}]({full_url})")
+                    if link_lines:
+                        links_md = "\n\n## Key Links\n" + "\n".join(link_lines)
+
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=target,
+                    title=title,
+                    content=f"## Web Page Content\n{content}{links_md}"
+                )
+            except Exception as e:
+                return ResearchResult(
+                    ok=False,
+                    source_type=self.source_type,
+                    canonical_url=target,
+                    error=f"Web page fetch failed: {str(e)}"
+                )
+        else:
+            # DuckDuckGo HTML Search
+            try:
+                query_enc = urllib.parse.quote(target)
+                ddg_url = f"https://html.duckduckgo.com/html/?q={query_enc}"
+                html = fetch_url(ddg_url, timeout=timeout)
+
+                results = []
+                result_blocks = re.findall(
+                    r'<a class="result__url" href="([^"]+)".*?</a>.*?<a class="result__snippet[^"]*">(.*?)</a>',
+                    html,
+                    re.DOTALL
+                )
+                if not result_blocks:
+                    urls = re.findall(r'class="result__a" href="([^"]+)">([^<]+)</a>', html)
+                    snippets = re.findall(r'class="result__snippet[^"]*">(.*?)</span>', html)
+                    for i in range(min(len(urls), 10)):
+                        raw_url, title = urls[i]
+                        parsed_url = urllib.parse.parse_qs(urllib.parse.urlparse(raw_url).query).get("uddg", [raw_url])[0]
+                        snip = snippets[i] if i < len(snippets) else ""
+                        snip_clean = re.sub(r"<[^>]+>", "", snip).strip()
+                        title_clean = re.sub(r"<[^>]+>", "", title).strip()
+                        results.append(f"### [{title_clean}]({parsed_url})\n**Snippet**: {snip_clean}\n")
+                else:
+                    for raw_url, snip in result_blocks[:10]:
+                        snip_clean = re.sub(r"<[^>]+>", "", snip).strip()
+                        results.append(f"### [Result]({raw_url})\n**Snippet**: {snip_clean}\n")
+
+                if not results:
+                    parser = SimpleHTMLTextExtractor()
+                    parser.feed(html)
+                    content = parser.get_markdown()[:max_content_length]
+                    return ResearchResult(
+                        ok=True,
+                        source_type=self.source_type,
+                        canonical_url=ddg_url,
+                        title=f"Web Search: '{target}'",
+                        content=content
+                    )
+
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=ddg_url,
+                    title=f"Web Search Results for: '{target}'",
+                    content="\n".join(results)
+                )
+            except Exception as e:
+                return ResearchResult(
+                    ok=False,
+                    source_type=self.source_type,
+                    canonical_url=f"https://duckduckgo.com/html/?q={urllib.parse.quote(target)}",
+                    error=f"Web search failed: {str(e)}"
+                )
+
+
+def scrape_web(target: str, timeout: int = 15) -> ResearchResult:
+    return WebScraper().fetch(target, timeout=timeout)

fable_engine/scrapers/x.py

@@ -0,0 +1,125 @@
+"""
+X (Twitter) tweet syndication and oembed lookup scraper implementation.
+Exact Tweet URLs utilize Twitter syndication API; handles/profiles use oEmbed embed summaries.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+from typing import Optional
+
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, SimpleHTMLTextExtractor, fetch_url
+
+
+class XScraper(ResearchSource):
+    source_type = "x"
+
+    def fetch(self, target: str, timeout: int = 15, max_content_length: int = 8000) -> ResearchResult:
+        target = target.strip()
+        if not target:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url="",
+                error="Target Tweet URL or handle cannot be empty."
+            )
+
+        tweet_id = None
+        tweet_id_match = re.search(r"status(?:es)?/(\d+)", target)
+        if tweet_id_match:
+            tweet_id = tweet_id_match.group(1)
+
+        if tweet_id:
+            # Use Twitter Syndication JSON API (free endpoint for individual tweets)
+            synd_url = f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}&token=x"
+            canonical_url = f"https://x.com/i/status/{tweet_id}"
+            try:
+                json_str = fetch_url(synd_url, timeout=timeout)
+                data = json.loads(json_str)
+
+                text = data.get("text", "")
+                user = data.get("user", {})
+                name = user.get("name", "User")
+                screen_name = user.get("screen_name", "unknown")
+                created_at = data.get("created_at", "")
+                likes = data.get("favorite_count", 0)
+                retweets = data.get("retweet_count", 0)
+                full_url = f"https://x.com/{screen_name}/status/{tweet_id}"
+
+                content = (
+                    f"**Date**: {created_at} | **Likes**: {likes} | **Retweets**: {retweets}\n\n"
+                    f"## Tweet\n{text}"
+                )
+
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=full_url,
+                    title=f"Tweet by @{screen_name} ({name})",
+                    author=f"@{screen_name}",
+                    content=content,
+                    metadata={"tweet_id": tweet_id, "likes": likes, "retweets": retweets}
+                )
+            except Exception as e:
+                # Fallback to Twitter oEmbed API
+                try:
+                    oembed_url = f"https://publish.twitter.com/oembed?url=https://twitter.com/i/status/{tweet_id}"
+                    json_str = fetch_url(oembed_url, timeout=timeout)
+                    data = json.loads(json_str)
+                    author = data.get("author_name", "User")
+                    html_embed = data.get("html", "")
+                    parser = SimpleHTMLTextExtractor()
+                    parser.feed(html_embed)
+                    text = parser.get_markdown()
+                    return ResearchResult(
+                        ok=True,
+                        source_type=self.source_type,
+                        canonical_url=canonical_url,
+                        title=f"Tweet by {author}",
+                        author=author,
+                        content=f"## Content\n{text}"
+                    )
+                except Exception:
+                    return ResearchResult(
+                        ok=False,
+                        source_type=self.source_type,
+                        canonical_url=canonical_url,
+                        error=f"X/Twitter tweet scrape failed for status ID {tweet_id}: {str(e)}"
+                    )
+
+        # Handle user profile / handle lookup via oembed
+        handle = target.lstrip("@").strip()
+        canonical_url = f"https://x.com/{handle}"
+        try:
+            oembed_url = f"https://publish.twitter.com/oembed?url=https://twitter.com/{handle}"
+            json_str = fetch_url(oembed_url, timeout=timeout)
+            data = json.loads(json_str)
+            author = data.get("author_name", handle)
+            html_embed = data.get("html", "")
+            parser = SimpleHTMLTextExtractor()
+            parser.feed(html_embed)
+            text = parser.get_markdown()
+            return ResearchResult(
+                ok=True,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                title=f"X Profile / Handle: @{handle}",
+                author=author,
+                content=f"## Summary\n{text}"
+            )
+        except Exception as e:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                error=(
+                    f"X handle/profile lookup failed for '@{handle}': {str(e)}. "
+                    "Note: For exact tweet contents, provide a direct Tweet status URL."
+                )
+            )
+
+
+def scrape_x(target: str, timeout: int = 15) -> ResearchResult:
+    return XScraper().fetch(target, timeout=timeout)

fable_engine/scrapers/youtube.py

@@ -0,0 +1,132 @@
+"""
+Best-effort YouTube metadata, search, and caption track scraper implementation.
+Uses heuristic page-structure parsing for public watch page metadata and automated caption tracks.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+import xml.etree.ElementTree as ET
+from typing import Optional
+
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url
+
+
+class YouTubeScraper(ResearchSource):
+    """
+    Best-effort scraper for YouTube public metadata and caption tracks.
+    Heuristic page parsing is subject to upstream YouTube markup variations.
+    """
+    source_type = "youtube"
+
+    def fetch(self, target: str, timeout: int = 15, max_content_length: int = 8000) -> ResearchResult:
+        target = target.strip()
+        video_id = None
+
+        if "youtube.com" in target or "youtu.be" in target:
+            parsed = urllib.parse.urlparse(target)
+            if "youtu.be" in target:
+                video_id = parsed.path.lstrip("/")
+            else:
+                qs = urllib.parse.parse_qs(parsed.query)
+                video_id = qs.get("v", [None])[0]
+        elif len(target) == 11 and re.match(r"^[A-Za-z0-9_-]{11}$", target):
+            video_id = target
+
+        if not video_id:
+            # Best-effort search resolution
+            try:
+                query_enc = urllib.parse.quote(target)
+                search_url = f"https://www.youtube.com/results?search_query={query_enc}"
+                html = fetch_url(search_url, timeout=timeout)
+                vids = re.findall(r'"videoId":"([A-Za-z0-9_-]{11})"', html)
+                if vids:
+                    video_id = vids[0]
+                else:
+                    return ResearchResult(
+                        ok=False,
+                        source_type=self.source_type,
+                        canonical_url=search_url,
+                        error=f"Best-effort YouTube search could not resolve video ID for '{target}'."
+                    )
+            except Exception as e:
+                return ResearchResult(
+                    ok=False,
+                    source_type=self.source_type,
+                    canonical_url=f"https://www.youtube.com/results?search_query={urllib.parse.quote(target)}",
+                    error=f"Best-effort YouTube search resolution failed: {str(e)}"
+                )
+
+        canonical_url = f"https://www.youtube.com/watch?v={video_id}"
+        try:
+            html = fetch_url(canonical_url, timeout=timeout)
+
+            title_match = re.search(r'<meta property="og:title" content="([^"]+)"', html) or re.search(r'"title":"([^"]+)"', html)
+            title = title_match.group(1) if title_match else f"YouTube Video ({video_id})"
+            title = re.sub(r"\\u0026", "&", title)
+
+            channel_match = re.search(r'<link itemprop="name" content="([^"]+)"', html) or re.search(r'"author":"([^"]+)"', html)
+            channel = channel_match.group(1) if channel_match else "Unknown Channel"
+
+            desc_match = re.search(r'<meta property="og:description" content="([^"]+)"', html)
+            description = desc_match.group(1) if desc_match else ""
+
+            transcript_text = ""
+            transcript_status = "not_found"
+            captions_match = re.search(r'"captionTracks":\[(.*?)\]', html)
+            if captions_match:
+                try:
+                    tracks_json = json.loads(f"[{captions_match.group(1)}]")
+                    if tracks_json and isinstance(tracks_json[0], dict) and "baseUrl" in tracks_json[0]:
+                        caption_url = tracks_json[0]["baseUrl"]
+                        caption_xml = fetch_url(caption_url, timeout=timeout)
+                        root = ET.fromstring(caption_xml)
+                        lines = []
+                        for child in root.findall("text"):
+                            txt = child.text or ""
+                            txt_clean = re.sub(r"&amp;", "&", txt)
+                            txt_clean = re.sub(r"&#39;", "'", txt_clean)
+                            txt_clean = re.sub(r"&quot;", '"', txt_clean)
+                            if txt_clean.strip():
+                                lines.append(txt_clean.strip())
+                        transcript_text = " ".join(lines)
+                        transcript_status = "retrieved" if transcript_text else "empty"
+                except Exception as exc:
+                    transcript_status = f"parse_error: {exc}"
+
+            content_blocks = [f"**Channel**: {channel}\n"]
+            if description:
+                content_blocks.append(f"## Description\n{description[:1000]}\n")
+            if transcript_text:
+                if len(transcript_text) > max_content_length:
+                    transcript_text = transcript_text[:max_content_length] + "...\n*(Transcript truncated)*"
+                content_blocks.append(f"## Transcript / Captions\n{transcript_text}")
+            else:
+                content_blocks.append(f"*(Note: Best-effort automated transcript track status: {transcript_status})*")
+
+            return ResearchResult(
+                ok=True,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                title=title,
+                author=channel,
+                content="\n".join(content_blocks),
+                metadata={
+                    "video_id": video_id,
+                    "extraction_mode": "best_effort_heuristic",
+                    "transcript_status": transcript_status,
+                }
+            )
+        except Exception as e:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                error=f"Best-effort YouTube video scrape failed: {str(e)}"
+            )
+
+
+def scrape_youtube(target: str, timeout: int = 15) -> ResearchResult:
+    return YouTubeScraper().fetch(target, timeout=timeout)