agent-recipes 0.0.5__py3-none-any.whl

agent_recipes/templates/ai-news-capture-pack/tools.py

@@ -0,0 +1,150 @@
+"""
+AI News Capture Pack Tools
+
+Bundle assets per news story:
+- Screenshots
+- Metadata
+- Source links
+"""
+
+import json
+import logging
+import os
+import shutil
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def capture_story_assets(
+    urls: List[str],
+    story_id: str,
+    output_dir: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Capture all assets for a news story.
+    
+    Args:
+        urls: List of URLs to capture
+        story_id: Unique story identifier
+        output_dir: Output directory
+        
+    Returns:
+        Dictionary with captured assets
+    """
+    output_dir = output_dir or f"./captures/{story_id}"
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    assets = []
+    
+    try:
+        from playwright.sync_api import sync_playwright
+        
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page(viewport={"width": 1920, "height": 1080})
+            
+            for i, url in enumerate(urls):
+                try:
+                    page.goto(url, wait_until="networkidle", timeout=30000)
+                    
+                    # Capture screenshot
+                    screenshot_path = os.path.join(output_dir, f"capture_{i}.png")
+                    page.screenshot(path=screenshot_path, full_page=True)
+                    
+                    # Extract metadata
+                    title = page.title()
+                    
+                    assets.append({
+                        "url": url,
+                        "screenshot": screenshot_path,
+                        "title": title,
+                        "captured_at": datetime.now(timezone.utc).isoformat(),
+                    })
+                    
+                except Exception as e:
+                    logger.warning(f"Error capturing {url}: {e}")
+                    assets.append({
+                        "url": url,
+                        "error": str(e),
+                    })
+            
+            browser.close()
+            
+    except ImportError:
+        logger.error("Playwright not installed")
+        return {"error": "Playwright not installed"}
+    
+    return {
+        "story_id": story_id,
+        "assets": assets,
+        "output_dir": output_dir,
+        "total_captured": len([a for a in assets if "screenshot" in a]),
+    }
+
+
+def create_bundle(
+    story_id: str,
+    assets: List[Dict[str, Any]],
+    output_dir: Optional[str] = None,
+    include_metadata: bool = True,
+) -> Dict[str, Any]:
+    """
+    Create a bundled pack of story assets.
+    
+    Args:
+        story_id: Story identifier
+        assets: List of asset dictionaries
+        output_dir: Output directory
+        include_metadata: Include metadata JSON
+        
+    Returns:
+        Bundle info
+    """
+    output_dir = output_dir or "./bundles"
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    
+    bundle_name = f"{story_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+    bundle_dir = os.path.join(output_dir, bundle_name)
+    Path(bundle_dir).mkdir(parents=True, exist_ok=True)
+    
+    bundled_files = []
+    
+    # Copy assets to bundle
+    for asset in assets:
+        if "screenshot" in asset and os.path.exists(asset["screenshot"]):
+            filename = os.path.basename(asset["screenshot"])
+            dest = os.path.join(bundle_dir, filename)
+            shutil.copy2(asset["screenshot"], dest)
+            bundled_files.append(filename)
+    
+    # Create metadata
+    if include_metadata:
+        metadata = {
+            "story_id": story_id,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "assets": assets,
+            "files": bundled_files,
+        }
+        metadata_path = os.path.join(bundle_dir, "metadata.json")
+        with open(metadata_path, "w") as f:
+            json.dump(metadata, f, indent=2)
+    
+    # Create zip
+    zip_path = f"{bundle_dir}.zip"
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+        for root, dirs, files in os.walk(bundle_dir):
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.relpath(file_path, bundle_dir)
+                zf.write(file_path, arcname)
+    
+    return {
+        "bundle_path": zip_path,
+        "bundle_dir": bundle_dir,
+        "files": bundled_files,
+        "size_bytes": os.path.getsize(zip_path),
+    }

agent_recipes/templates/ai-news-crawler/recipe.yaml

@@ -0,0 +1,99 @@
+name: ai-news-crawler
+version: 1.0.0
+description: Crawl AI news from multiple sources including AI labs blogs, arXiv, GitHub trending, HackerNews, Reddit, and X lists
+author: PraisonAI
+tags:
+  - news
+  - crawler
+  - ai
+  - research
+  - aggregation
+
+config:
+  sources:
+    - hackernews
+    - reddit
+    - arxiv
+    - github_trending
+    - ai_blogs
+  max_articles: 50
+  time_window_hours: 24
+  include_content: true
+  output_format: json
+
+input:
+  type: object
+  properties:
+    sources:
+      type: array
+      description: List of sources to crawl
+      items:
+        type: string
+        enum: [hackernews, reddit, arxiv, github_trending, ai_blogs, x_lists]
+    max_articles:
+      type: integer
+      description: Maximum number of articles to fetch
+      default: 50
+    time_window_hours:
+      type: integer
+      description: Only fetch articles from the last N hours
+      default: 24
+
+output:
+  type: object
+  properties:
+    articles:
+      type: array
+      items:
+        type: object
+        properties:
+          title:
+            type: string
+          url:
+            type: string
+          source:
+            type: string
+          published:
+            type: string
+          content:
+            type: string
+          score:
+            type: number
+    crawl_metadata:
+      type: object
+      properties:
+        total_fetched:
+          type: integer
+        sources_crawled:
+          type: array
+        crawl_time:
+          type: string
+
+requires:
+  env:
+    - OPENAI_API_KEY
+  optional_env:
+    - TAVILY_API_KEY
+    - REDDIT_CLIENT_ID
+    - REDDIT_CLIENT_SECRET
+  packages:
+    - requests
+    - feedparser
+    - beautifulsoup4
+
+workflow:
+  agents:
+    - name: news_crawler
+      role: AI News Aggregator
+      goal: Crawl and collect AI-related news from multiple sources
+      backstory: Expert at finding and aggregating AI news from various platforms
+      tools:
+        - crawl_hackernews
+        - crawl_reddit
+        - crawl_arxiv
+        - crawl_github_trending
+        - search_web
+      tasks:
+        - name: crawl_sources
+          description: Crawl all configured sources for AI news
+          expected_output: List of articles with metadata

agent_recipes/templates/ai-news-crawler/tools.py

@@ -0,0 +1,417 @@
+"""
+AI News Crawler Tools
+
+Tools for crawling AI news from multiple sources:
+- HackerNews
+- Reddit (r/MachineLearning, r/artificial, etc.)
+- arXiv (cs.AI, cs.LG, cs.CL)
+- GitHub Trending
+- AI Labs Blogs
+- Web Search (via Tavily)
+"""
+
+import json
+import os
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+from urllib.parse import urljoin
+
+logger = logging.getLogger(__name__)
+
+
+def crawl_hackernews(
+    max_articles: int = 20,
+    time_window_hours: int = 24,
+    keywords: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Crawl HackerNews for AI-related stories.
+    
+    Args:
+        max_articles: Maximum number of articles to fetch
+        time_window_hours: Only fetch articles from the last N hours
+        keywords: Filter by keywords (default: AI-related terms)
+        
+    Returns:
+        List of article dictionaries
+    """
+    import requests
+    
+    keywords = keywords or ["ai", "gpt", "llm", "machine learning", "openai", "anthropic", "google ai", "neural", "transformer"]
+    
+    articles = []
+    base_url = "https://hacker-news.firebaseio.com/v0"
+    
+    try:
+        # Get top stories
+        response = requests.get(f"{base_url}/topstories.json", timeout=10)
+        response.raise_for_status()
+        story_ids = response.json()[:100]  # Get top 100 to filter
+        
+        cutoff_time = datetime.now(timezone.utc) - timedelta(hours=time_window_hours)
+        
+        for story_id in story_ids:
+            if len(articles) >= max_articles:
+                break
+                
+            try:
+                story_response = requests.get(f"{base_url}/item/{story_id}.json", timeout=5)
+                story = story_response.json()
+                
+                if not story or story.get("type") != "story":
+                    continue
+                
+                title = story.get("title", "").lower()
+                
+                # Check if AI-related
+                if not any(kw in title for kw in keywords):
+                    continue
+                
+                # Check time window
+                story_time = datetime.fromtimestamp(story.get("time", 0), tz=timezone.utc)
+                if story_time < cutoff_time:
+                    continue
+                
+                articles.append({
+                    "title": story.get("title", ""),
+                    "url": story.get("url", f"https://news.ycombinator.com/item?id={story_id}"),
+                    "source": "hackernews",
+                    "published": story_time.isoformat(),
+                    "score": story.get("score", 0),
+                    "comments": story.get("descendants", 0),
+                    "author": story.get("by", ""),
+                    "content": "",  # HN doesn't provide content
+                })
+            except Exception as e:
+                logger.warning(f"Error fetching story {story_id}: {e}")
+                continue
+                
+    except Exception as e:
+        logger.error(f"Error crawling HackerNews: {e}")
+    
+    return articles
+
+
+def crawl_reddit(
+    subreddits: Optional[List[str]] = None,
+    max_articles: int = 20,
+    time_window_hours: int = 24,
+) -> List[Dict[str, Any]]:
+    """
+    Crawl Reddit for AI-related posts.
+    
+    Args:
+        subreddits: List of subreddits to crawl
+        max_articles: Maximum number of articles to fetch
+        time_window_hours: Only fetch articles from the last N hours
+        
+    Returns:
+        List of article dictionaries
+    """
+    import requests
+    
+    subreddits = subreddits or ["MachineLearning", "artificial", "LocalLLaMA", "OpenAI", "ClaudeAI"]
+    articles = []
+    
+    headers = {"User-Agent": "PraisonAI News Crawler 1.0"}
+    
+    for subreddit in subreddits:
+        if len(articles) >= max_articles:
+            break
+            
+        try:
+            url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit=25"
+            response = requests.get(url, headers=headers, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+            
+            cutoff_time = datetime.now(timezone.utc) - timedelta(hours=time_window_hours)
+            
+            for post in data.get("data", {}).get("children", []):
+                if len(articles) >= max_articles:
+                    break
+                    
+                post_data = post.get("data", {})
+                
+                # Check time window
+                created = datetime.fromtimestamp(post_data.get("created_utc", 0), tz=timezone.utc)
+                if created < cutoff_time:
+                    continue
+                
+                articles.append({
+                    "title": post_data.get("title", ""),
+                    "url": post_data.get("url", ""),
+                    "source": f"reddit/r/{subreddit}",
+                    "published": created.isoformat(),
+                    "score": post_data.get("score", 0),
+                    "comments": post_data.get("num_comments", 0),
+                    "author": post_data.get("author", ""),
+                    "content": post_data.get("selftext", "")[:500],
+                })
+                
+        except Exception as e:
+            logger.warning(f"Error crawling r/{subreddit}: {e}")
+            continue
+    
+    return articles
+
+
+def crawl_arxiv(
+    categories: Optional[List[str]] = None,
+    max_articles: int = 20,
+    time_window_hours: int = 48,
+) -> List[Dict[str, Any]]:
+    """
+    Crawl arXiv for AI research papers.
+    
+    Args:
+        categories: arXiv categories to search
+        max_articles: Maximum number of papers to fetch
+        time_window_hours: Only fetch papers from the last N hours
+        
+    Returns:
+        List of paper dictionaries
+    """
+    import requests
+    import xml.etree.ElementTree as ET
+    
+    categories = categories or ["cs.AI", "cs.LG", "cs.CL", "cs.CV", "cs.NE"]
+    articles = []
+    
+    try:
+        # Build query
+        cat_query = " OR ".join([f"cat:{cat}" for cat in categories])
+        url = f"http://export.arxiv.org/api/query?search_query={cat_query}&sortBy=submittedDate&sortOrder=descending&max_results={max_articles}"
+        
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        
+        # Parse XML
+        root = ET.fromstring(response.content)
+        ns = {"atom": "http://www.w3.org/2005/Atom"}
+        
+        for entry in root.findall("atom:entry", ns):
+            title = entry.find("atom:title", ns)
+            summary = entry.find("atom:summary", ns)
+            published = entry.find("atom:published", ns)
+            link = entry.find("atom:id", ns)
+            authors = entry.findall("atom:author/atom:name", ns)
+            
+            articles.append({
+                "title": title.text.strip() if title is not None else "",
+                "url": link.text if link is not None else "",
+                "source": "arxiv",
+                "published": published.text if published is not None else "",
+                "content": summary.text.strip()[:500] if summary is not None else "",
+                "authors": [a.text for a in authors],
+                "score": 0,
+            })
+            
+    except Exception as e:
+        logger.error(f"Error crawling arXiv: {e}")
+    
+    return articles
+
+
+def crawl_github_trending(
+    language: Optional[str] = None,
+    max_repos: int = 20,
+) -> List[Dict[str, Any]]:
+    """
+    Crawl GitHub trending repositories for AI projects.
+    
+    Args:
+        language: Filter by programming language
+        max_repos: Maximum number of repos to fetch
+        
+    Returns:
+        List of repository dictionaries
+    """
+    import requests
+    from bs4 import BeautifulSoup
+    
+    articles = []
+    
+    try:
+        url = "https://github.com/trending"
+        if language:
+            url += f"/{language}"
+        url += "?since=daily"
+        
+        headers = {"User-Agent": "PraisonAI News Crawler 1.0"}
+        response = requests.get(url, headers=headers, timeout=10)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+        
+        # AI-related keywords
+        ai_keywords = ["ai", "llm", "gpt", "transformer", "neural", "ml", "machine-learning", 
+                       "deep-learning", "nlp", "vision", "agent", "rag", "embedding"]
+        
+        for article in soup.select("article.Box-row")[:max_repos * 2]:
+            try:
+                repo_link = article.select_one("h2 a")
+                if not repo_link:
+                    continue
+                    
+                repo_name = repo_link.get_text(strip=True).replace("\n", "").replace(" ", "")
+                repo_url = "https://github.com" + repo_link.get("href", "")
+                
+                description_elem = article.select_one("p")
+                description = description_elem.get_text(strip=True) if description_elem else ""
+                
+                # Check if AI-related
+                text_to_check = (repo_name + " " + description).lower()
+                if not any(kw in text_to_check for kw in ai_keywords):
+                    continue
+                
+                stars_elem = article.select_one("a[href*='/stargazers']")
+                stars = stars_elem.get_text(strip=True).replace(",", "") if stars_elem else "0"
+                
+                articles.append({
+                    "title": repo_name,
+                    "url": repo_url,
+                    "source": "github_trending",
+                    "published": datetime.now(timezone.utc).isoformat(),
+                    "content": description,
+                    "score": int(stars) if stars.isdigit() else 0,
+                })
+                
+                if len(articles) >= max_repos:
+                    break
+                    
+            except Exception as e:
+                logger.warning(f"Error parsing repo: {e}")
+                continue
+                
+    except Exception as e:
+        logger.error(f"Error crawling GitHub trending: {e}")
+    
+    return articles
+
+
+def search_web(
+    query: str,
+    max_results: int = 10,
+) -> List[Dict[str, Any]]:
+    """
+    Search the web for AI news using Tavily API.
+    
+    Args:
+        query: Search query
+        max_results: Maximum number of results
+        
+    Returns:
+        List of search result dictionaries
+    """
+    api_key = os.environ.get("TAVILY_API_KEY")
+    if not api_key:
+        logger.warning("TAVILY_API_KEY not set, skipping web search")
+        return []
+    
+    import requests
+    
+    articles = []
+    
+    try:
+        response = requests.post(
+            "https://api.tavily.com/search",
+            json={
+                "api_key": api_key,
+                "query": query,
+                "search_depth": "advanced",
+                "max_results": max_results,
+                "include_answer": False,
+            },
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+        
+        for result in data.get("results", []):
+            articles.append({
+                "title": result.get("title", ""),
+                "url": result.get("url", ""),
+                "source": "web_search",
+                "published": datetime.now(timezone.utc).isoformat(),
+                "content": result.get("content", "")[:500],
+                "score": result.get("score", 0),
+            })
+            
+    except Exception as e:
+        logger.error(f"Error in web search: {e}")
+    
+    return articles
+
+
+def crawl_ai_news(
+    sources: Optional[List[str]] = None,
+    max_articles: int = 50,
+    time_window_hours: int = 24,
+    output_dir: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Main function to crawl AI news from all configured sources.
+    
+    Args:
+        sources: List of sources to crawl
+        max_articles: Maximum total articles
+        time_window_hours: Time window for articles
+        output_dir: Optional directory to save results
+        
+    Returns:
+        Dictionary with articles and metadata
+    """
+    sources = sources or ["hackernews", "reddit", "arxiv", "github_trending"]
+    all_articles = []
+    sources_crawled = []
+    
+    per_source_limit = max(5, max_articles // len(sources))
+    
+    for source in sources:
+        try:
+            if source == "hackernews":
+                articles = crawl_hackernews(max_articles=per_source_limit, time_window_hours=time_window_hours)
+            elif source == "reddit":
+                articles = crawl_reddit(max_articles=per_source_limit, time_window_hours=time_window_hours)
+            elif source == "arxiv":
+                articles = crawl_arxiv(max_articles=per_source_limit, time_window_hours=time_window_hours)
+            elif source == "github_trending":
+                articles = crawl_github_trending(max_repos=per_source_limit)
+            elif source == "web_search":
+                articles = search_web("AI news today", max_results=per_source_limit)
+            else:
+                logger.warning(f"Unknown source: {source}")
+                continue
+            
+            all_articles.extend(articles)
+            sources_crawled.append(source)
+            logger.info(f"Crawled {len(articles)} articles from {source}")
+            
+        except Exception as e:
+            logger.error(f"Error crawling {source}: {e}")
+    
+    # Sort by score and limit
+    all_articles.sort(key=lambda x: x.get("score", 0), reverse=True)
+    all_articles = all_articles[:max_articles]
+    
+    result = {
+        "articles": all_articles,
+        "crawl_metadata": {
+            "total_fetched": len(all_articles),
+            "sources_crawled": sources_crawled,
+            "crawl_time": datetime.now(timezone.utc).isoformat(),
+        }
+    }
+    
+    # Save to file if output_dir specified
+    if output_dir:
+        import os
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, "crawled_news.json")
+        with open(output_path, "w") as f:
+            json.dump(result, f, indent=2)
+        logger.info(f"Saved results to {output_path}")
+    
+    return result