drupal-news 0.1.0__py3-none-any.whl

drupal_news/__init__.py

@@ -0,0 +1 @@
+"""Drupal News Aggregator - Source package."""

drupal_news/ai_summarizer.py

@@ -0,0 +1,337 @@
+"""AI summarizer for Drupal Newsletter."""
+import importlib
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+import time
+from drupal_news.markdown_converter import items_to_text
+
+
+SUMMARIZER_PROMPT_TEMPLATE = """
+You are a technical writer for the Drupal community. Generate a summary of Drupal news and updates.
+
+**Requirements:**
+1. Focus on AI module and news on AI
+2. Each fact MUST include a [source](URL) link
+3. Use clear, factual language - no hype
+4. If no major updates: include "No significant core updates this week"
+5. Present RSS/new modules as a table with columns: URL, Name, Description
+6. Organize by sections: Core Updates, Modules, AI/Automation, Canvas/Admin UI, Planet, D.O. Blog
+
+**Timeframe:** Last {timeframe_days} days ({timezone})
+
+**Items to summarize:**
+
+{items_text}
+
+Generate the summary in Markdown format with proper sections and source links.
+"""
+
+
+def load_prompt_template(prompt_file: str = None) -> str:
+    """
+    Load prompt template from markdown file or use default.
+
+    Args:
+        prompt_file: Path to prompt.md file (default: prompt.md in project root)
+
+    Returns:
+        Prompt template string with placeholders
+    """
+    if prompt_file is None:
+        # Default to prompt.md in the project root
+        prompt_file = Path(__file__).parent.parent / 'prompt.md'
+
+    prompt_path = Path(prompt_file)
+
+    if prompt_path.exists():
+        try:
+            return prompt_path.read_text(encoding='utf-8')
+        except Exception as e:
+            print(f"Warning: Could not read {prompt_file}: {e}")
+            print("Using default hardcoded prompt template")
+
+    # Fall back to hardcoded default
+    return SUMMARIZER_PROMPT_TEMPLATE
+
+
+def get_provider_client(provider_name: str, client_name: str = None):
+    """
+    Dynamically load provider client module.
+
+    Args:
+        provider_name: Provider name (e.g., 'openai', 'anthropic')
+        client_name: Optional client name from config (e.g., 'generic_client')
+
+    Returns:
+        Provider client module
+    """
+    try:
+        # Use client_name if provided, otherwise use provider_name
+        module_name = client_name if client_name else f"{provider_name}_client"
+        # Remove _client suffix if already present
+        if not module_name.endswith('_client'):
+            module_name = f"{module_name}_client"
+        module = importlib.import_module(f"utils.providers.{module_name}")
+        return module
+    except ImportError as e:
+        raise ImportError(f"Provider '{provider_name}' (client: {client_name or provider_name}) not found: {e}")
+
+
+
+def calculate_cost(provider: str, provider_config: Dict[str, Any], summary_result: Dict[str, Any]) -> float:
+    """Estimate USD cost for a summary run based on token usage and configured pricing."""
+    if not provider_config:
+        return 0.0
+
+    pricing = provider_config.get("pricing")
+    if not pricing:
+        return 0.0
+
+    input_tokens = summary_result.get("input_tokens")
+    output_tokens = summary_result.get("output_tokens")
+    total_tokens = summary_result.get("tokens", 0) or 0
+
+    cost = 0.0
+
+    input_rate = pricing.get("input_cost_per_1k")
+    output_rate = pricing.get("output_cost_per_1k")
+    blended_rate = pricing.get("per_1k_tokens")
+
+    if input_tokens is not None and input_rate is not None:
+        cost += (input_tokens / 1000.0) * float(input_rate)
+
+    if output_tokens is not None and output_rate is not None:
+        cost += (output_tokens / 1000.0) * float(output_rate)
+
+    # Fallback to blended pricing when detailed usage is absent
+    if cost == 0.0 and blended_rate is not None:
+        cost += (total_tokens / 1000.0) * float(blended_rate)
+
+    return round(cost, 6)
+
+
+
+def summarize(
+    items: List[Dict[str, Any]],
+    provider: str,
+    model: str,
+    temperature: float,
+    timeframe_days: int,
+    timezone: str,
+    max_items: int = 200,
+    chunk_size: int = 200,
+    provider_config: Dict[str, Any] = None
+) -> Dict[str, Any]:
+    """
+    Generate AI summary of items.
+
+    Args:
+        items: List of news items
+        provider: AI provider name
+        model: Model name
+        temperature: Temperature setting
+        timeframe_days: Number of days covered
+        timezone: Timezone name
+        max_items: Maximum items to process
+        chunk_size: Chunk size for large inputs
+        provider_config: Full provider configuration (includes api_url, headers, etc.)
+
+    Returns:
+        Dictionary with 'text', 'tokens', 'model', 'provider', 'duration'
+    """
+    start_time = time.time()
+
+    # Load provider client
+    # Extract client name from provider_config if available
+    client_name = provider_config.get("client") if provider_config else None
+    client = get_provider_client(provider, client_name)
+
+    # Convert items to text
+    items_text = items_to_text(items, max_items)
+
+    # Load prompt template from file or use default
+    template = load_prompt_template()
+
+    def render_prompt(items_text_value: str) -> str:
+        return template.format(
+            timeframe_days=timeframe_days,
+            timezone=timezone,
+            items_text=items_text_value
+        )
+
+    prompt = render_prompt(items_text)
+
+    # Prepare kwargs for provider (prompt injected per request)
+    base_kwargs = {
+        "model": model,
+        "temperature": temperature
+    }
+
+    # Add optional provider-specific settings
+    if provider_config:
+        if "api_url" in provider_config:
+            base_kwargs["api_url"] = provider_config["api_url"]
+        if "headers" in provider_config:
+            base_kwargs["headers"] = provider_config["headers"]
+
+    # Check if chunking needed
+    if len(items) > chunk_size:
+        summary_result = _summarize_chunked(
+            client,
+            items,
+            render_prompt,
+            chunk_size,
+            base_kwargs
+        )
+    else:
+        request_kwargs = dict(base_kwargs)
+        request_kwargs["prompt"] = prompt
+        try:
+            summary_result = client.generate_summary(**request_kwargs)
+        except Exception as e:
+            raise RuntimeError(f"Summarization failed: {str(e)}")
+
+    # Add metadata
+    summary_result.setdefault("provider", provider)
+    summary_result.setdefault("model", model)
+    summary_result["duration"] = time.time() - start_time
+    summary_result["cost"] = calculate_cost(provider, provider_config or {}, summary_result)
+
+    return summary_result
+
+
+def _summarize_chunked(
+    client,
+    items: List[Dict[str, Any]],
+    render_prompt,
+    chunk_size: int,
+    base_kwargs: Dict[str, Any]
+) -> Dict[str, Any]:
+    """Summarize items in chunks for large datasets."""
+    summaries = []
+    total_tokens = 0
+    total_input_tokens = 0
+    total_output_tokens = 0
+
+    for i in range(0, len(items), chunk_size):
+        chunk = items[i:i + chunk_size]
+        chunk_kwargs = dict(base_kwargs)
+        chunk_kwargs["prompt"] = render_prompt(items_to_text(chunk))
+
+        try:
+            result = client.generate_summary(**chunk_kwargs)
+            summaries.append(result.get("text", ""))
+            total_tokens += result.get("tokens", 0) or 0
+            total_input_tokens += result.get("input_tokens") or 0
+            total_output_tokens += result.get("output_tokens") or 0
+        except Exception as e:
+            print(f"Warning: Chunk {i // chunk_size + 1} failed: {e}")
+
+    combined_text = "\n\n".join(filter(None, summaries))
+
+    summary = {
+        "text": combined_text,
+        "tokens": total_tokens,
+        "model": base_kwargs.get("model"),
+        "provider": client.__name__.split(".")[-1].replace("_client", ""),
+        "chunked": True
+    }
+
+    if total_input_tokens:
+        summary["input_tokens"] = total_input_tokens
+    if total_output_tokens:
+        summary["output_tokens"] = total_output_tokens
+
+    return summary
+
+
+def summarize_with_fallback(
+    items: List[Dict[str, Any]],
+    providers_config: Dict[str, Any],
+    default_provider: str,
+    timeframe_days: int,
+    timezone: str,
+    fallback_order: Optional[List[str]] = None
+) -> Dict[str, Any]:
+    """
+    Summarize with automatic fallback to other providers.
+
+    Args:
+        items: List of news items
+        providers_config: Providers configuration
+        default_provider: Default provider to try first
+        timeframe_days: Days covered
+        timezone: Timezone name
+        fallback_order: Optional custom fallback order
+
+    Returns:
+        Summary result with provider info
+    """
+    if fallback_order is None:
+        fallback_order = ["openai", "anthropic", "ollama", "qwen", "openrouter"]
+
+    # Try default provider first
+    provider_list = [default_provider] + [p for p in fallback_order if p != default_provider]
+
+    last_error = None
+
+    for provider_name in provider_list:
+        provider_config = providers_config.get("providers", {}).get(provider_name)
+
+        if not provider_config:
+            continue
+
+        try:
+            result = summarize(
+                items=items,
+                provider=provider_name,
+                model=provider_config["model"],
+                temperature=provider_config.get("temperature", 0.2),
+                timeframe_days=timeframe_days,
+                timezone=timezone,
+                provider_config=provider_config
+            )
+
+            return result
+
+        except Exception as e:
+            last_error = e
+            print(f"Provider {provider_name} failed: {e}")
+            continue
+
+    # All providers failed
+    raise RuntimeError(f"All providers failed. Last error: {last_error}")
+
+
+def generate_placeholder_summary(items: List[Dict[str, Any]], timeframe_days: int) -> str:
+    """
+    Generate placeholder summary for dry-run mode.
+
+    Args:
+        items: List of items
+        timeframe_days: Days covered
+
+    Returns:
+        Placeholder markdown summary
+    """
+    return f"""# Drupal Newesleter Summary (Dry Run)
+
+**Timeframe:** Last {timeframe_days} days
+
+## Summary
+
+This is a placeholder summary for dry-run mode.
+Total items collected: {len(items)}
+
+### Core Updates
+No significant core updates this week.
+
+### New Modules
+{len([i for i in items if i.get('source_type') == 'rss'])} new modules found.
+
+### AI and Automation
+No major AI updates this week.
+
+---
+*This is a dry-run summary. Enable AI provider for actual summaries.*
+"""

drupal_news/cache_manager.py

@@ -0,0 +1,174 @@
+"""Cache manager for Drupal Newsletter using SQLite."""
+import sqlite3
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime, timedelta
+import json
+import hashlib
+
+
+class CacheManager:
+    """Manages persistent caching with SQLite."""
+
+    def __init__(self, db_path: str = "./cache/cache.db", ttl_days: int = 21):
+        """
+        Initialize cache manager.
+
+        Args:
+            db_path: Path to SQLite database
+            ttl_days: Time-to-live for cache entries in days
+        """
+        self.db_path = Path(db_path)
+        self.ttl_days = ttl_days
+        self._init_db()
+
+    def _init_db(self):
+        """Initialize database schema."""
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS cache (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                expires_at TEXT NOT NULL
+            )
+        """)
+
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_expires_at ON cache(expires_at)
+        """)
+
+        conn.commit()
+        conn.close()
+
+    def _compute_key(self, url: str) -> str:
+        """Compute cache key from URL."""
+        return hashlib.sha256(url.encode()).hexdigest()
+
+    def get(self, url: str) -> Optional[Dict[str, Any]]:
+        """
+        Get cached value for URL.
+
+        Returns:
+            Cached value if found and not expired, None otherwise
+        """
+        key = self._compute_key(url)
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            SELECT value, expires_at FROM cache WHERE key = ?
+        """, (key,))
+
+        row = cursor.fetchone()
+        conn.close()
+
+        if not row:
+            return None
+
+        value_json, expires_at = row
+        expires_dt = datetime.fromisoformat(expires_at)
+
+        # Check expiration
+        if datetime.now() > expires_dt:
+            self.delete(url)
+            return None
+
+        try:
+            return json.loads(value_json)
+        except json.JSONDecodeError:
+            return None
+
+    def set(self, url: str, value: Dict[str, Any]):
+        """
+        Set cached value for URL.
+
+        Args:
+            url: URL to cache
+            value: Dictionary value to cache
+        """
+        key = self._compute_key(url)
+        created_at = datetime.now().isoformat()
+        expires_at = (datetime.now() + timedelta(days=self.ttl_days)).isoformat()
+
+        value_json = json.dumps(value, ensure_ascii=False)
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            INSERT OR REPLACE INTO cache (key, value, created_at, expires_at)
+            VALUES (?, ?, ?, ?)
+        """, (key, value_json, created_at, expires_at))
+
+        conn.commit()
+        conn.close()
+
+    def delete(self, url: str):
+        """Delete cached value for URL."""
+        key = self._compute_key(url)
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("DELETE FROM cache WHERE key = ?", (key,))
+
+        conn.commit()
+        conn.close()
+
+    def purge_expired(self) -> int:
+        """
+        Remove all expired cache entries.
+
+        Returns:
+            Number of entries removed
+        """
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        now = datetime.now().isoformat()
+        cursor.execute("DELETE FROM cache WHERE expires_at < ?", (now,))
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
+
+        return deleted_count
+
+    def clear_all(self):
+        """Clear all cache entries."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("DELETE FROM cache")
+
+        conn.commit()
+        conn.close()
+
+    def get_stats(self) -> Dict[str, int]:
+        """
+        Get cache statistics.
+
+        Returns:
+            Dictionary with total, expired, and valid counts
+        """
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        cursor.execute("SELECT COUNT(*) FROM cache")
+        total = cursor.fetchone()[0]
+
+        now = datetime.now().isoformat()
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE expires_at < ?", (now,))
+        expired = cursor.fetchone()[0]
+
+        conn.close()
+
+        return {
+            "total": total,
+            "expired": expired,
+            "valid": total - expired
+        }

drupal_news/cli.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+CLI entry points for drupal-news package.
+"""
+import sys
+
+
+def main():
+    """Main CLI entry point for drupal-news command."""
+    from drupal_news.index import main as index_main
+    sys.exit(index_main())
+
+
+def scheduler_main():
+    """Scheduler CLI entry point for drupal-news-scheduler command."""
+    from drupal_news.scheduler import main as scheduler_main_func
+    sys.exit(scheduler_main_func())
+
+
+def email_main():
+    """Email CLI entry point for drupal-news-email command."""
+    from drupal_news.email_sender import main as email_main_func
+    sys.exit(email_main_func())
+
+
+def viewer_main():
+    """Web viewer CLI entry point for drupal-news-viewer command."""
+    from drupal_news.viewer import app
+
+    print("=" * 60)
+    print("Drupal News Viewer")
+    print("=" * 60)
+    print("Starting server on http://localhost:5000")
+    print("Press Ctrl+C to stop")
+    print("=" * 60)
+
+    app.run(debug=False, host='0.0.0.0', port=5000)
+
+
+if __name__ == "__main__":
+    main()