realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/search/providers/serper.py

@@ -0,0 +1,209 @@
+"""
+Serper Google SERP Provider
+
+API: https://serper.dev
+Endpoint: https://google.serper.dev/{mode}
+
+Features:
+- Real-time Google search results (1-2 seconds)
+- Modes: search, scholar
+- Knowledge graph extraction
+- People Also Ask extraction
+- Related searches
+- Very cheap: $1/1000 queries at scale
+"""
+
+from datetime import datetime
+import json
+from typing import Any
+
+import requests
+
+from ..base import BaseSearchProvider
+from ..types import Citation, SearchResult, WebSearchResponse
+from . import register_provider
+
+
+class SerperAPIError(Exception):
+    """Serper API error"""
+
+    pass
+
+
+@register_provider("serper")
+class SerperProvider(BaseSearchProvider):
+    """Serper Google SERP provider"""
+
+    display_name = "Serper"
+    description = "Google SERP results"
+    supports_answer = False  # Raw SERP results, no LLM answer
+    BASE_URL = "https://google.serper.dev"
+
+    def search(
+        self,
+        query: str,
+        mode: str = "search",  # search, scholar
+        num: int = 10,
+        gl: str = "us",  # Country code
+        hl: str = "en",  # Language code
+        page: int = 1,
+        autocorrect: bool = True,
+        timeout: int = 30,
+        **kwargs: Any,
+    ) -> WebSearchResponse:
+        """
+        Perform Google SERP search using Serper API.
+
+        Args:
+            query: Search query.
+            mode: Search mode - "search" or "scholar".
+            num: Number of results (default 10, max 100).
+            gl: Country code (default "us").
+            hl: Language code (default "en").
+            page: Page number for pagination.
+            autocorrect: Enable autocorrect (default True).
+            timeout: Request timeout in seconds.
+            **kwargs: Additional options.
+
+        Returns:
+            WebSearchResponse: Standardized search response.
+        """
+        self.logger.debug(f"Calling Serper API mode={mode}, num={num}")
+        headers = {
+            "X-API-KEY": self.api_key,
+            "Content-Type": "application/json",
+        }
+
+        payload: dict[str, Any] = {
+            "q": query,
+            "num": num,
+            "gl": gl,
+            "hl": hl,
+            "page": page,
+            "autocorrect": autocorrect,
+        }
+
+        url = f"{self.BASE_URL}/{mode}"
+        response = requests.post(url, headers=headers, json=payload, timeout=timeout)
+
+        if response.status_code != 200:
+            try:
+                error_data = response.json()
+            except (json.JSONDecodeError, ValueError):
+                error_data = {"message": response.text}
+            self.logger.error(f"Serper API error: {response.status_code} - {error_data}")
+            raise SerperAPIError(
+                f"Serper API error: {response.status_code} - "
+                f"{error_data.get('message', response.text)}"
+            )
+
+        data = response.json()
+        self.logger.debug(f"Serper returned {len(data.get('organic', []))} results")
+
+        # Extract search results
+        citations: list[Citation] = []
+        search_results: list[SearchResult] = []
+
+        # Both search and scholar return results in "organic" key
+        results_key = "organic"
+
+        for i, result in enumerate(data.get(results_key, []), 1):
+            # Handle different result formats
+            title = result.get("title", "")
+            url_val = result.get("link", result.get("url", ""))
+            snippet = result.get("snippet", result.get("description", ""))
+            date = result.get("date", "")
+
+            # Extract sitelinks if available
+            sitelinks = []
+            if result.get("sitelinks"):
+                for sl in result["sitelinks"]:
+                    sitelinks.append({"title": sl.get("title", ""), "link": sl.get("link", "")})
+
+            # Build attributes dict with scholar-specific fields
+            attributes: dict[str, Any] = result.get("attributes", {})
+
+            # Scholar mode: extract publication info, citations, PDF URL, year
+            if mode == "scholar":
+                # publicationInfo is a string like "A Vaswani, N Shazeer... - Advances in neural..., 2017"
+                if result.get("publicationInfo"):
+                    attributes["publicationInfo"] = result["publicationInfo"]
+                # citedBy is a number
+                if result.get("citedBy") is not None:
+                    attributes["citedBy"] = result["citedBy"]
+                # pdfUrl is a direct link to PDF
+                if result.get("pdfUrl"):
+                    attributes["pdfUrl"] = result["pdfUrl"]
+                # year is a number
+                if result.get("year") is not None:
+                    attributes["year"] = result["year"]
+                # paper ID
+                if result.get("id"):
+                    attributes["paperId"] = result["id"]
+
+            sr = SearchResult(
+                title=title,
+                url=url_val,
+                snippet=snippet,
+                date=date,
+                source=result.get("source", ""),
+                sitelinks=sitelinks,
+                attributes=attributes,
+            )
+            search_results.append(sr)
+
+            citations.append(
+                Citation(
+                    id=i,
+                    reference=f"[{i}]",
+                    url=url_val,
+                    title=title,
+                    snippet=snippet,
+                    date=date,
+                    source=result.get("source", ""),
+                )
+            )
+
+        # Build metadata with rich SERP data
+        metadata: dict[str, Any] = {
+            "finish_reason": "stop",
+            "mode": mode,
+            "searchParameters": data.get("searchParameters", {}),
+        }
+
+        # Include knowledge graph if available
+        if data.get("knowledgeGraph"):
+            metadata["knowledgeGraph"] = data["knowledgeGraph"]
+
+        # Include answer box if available
+        if data.get("answerBox"):
+            metadata["answerBox"] = data["answerBox"]
+
+        # Include People Also Ask
+        if data.get("peopleAlsoAsk"):
+            metadata["peopleAlsoAsk"] = data["peopleAlsoAsk"]
+
+        # Include related searches
+        if data.get("relatedSearches"):
+            metadata["relatedSearches"] = data["relatedSearches"]
+
+        # Build answer from answer box or knowledge graph if available
+        answer = ""
+        if data.get("answerBox"):
+            ab = data["answerBox"]
+            answer = ab.get("answer", ab.get("snippet", ""))
+        elif data.get("knowledgeGraph"):
+            kg = data["knowledgeGraph"]
+            answer = kg.get("description", "")
+
+        return WebSearchResponse(
+            query=query,
+            answer=answer,
+            provider="serper_scholar" if mode == "scholar" else "serper",
+            timestamp=datetime.now().isoformat(),
+            model=f"serper-{mode}",
+            citations=citations,
+            search_results=search_results,
+            usage={},
+            metadata=metadata,
+        )

src/services/search/providers/tavily.py

@@ -0,0 +1,161 @@
+"""
+Tavily Search Provider
+
+API Docs: https://docs.tavily.com/documentation/api-reference/endpoint/search
+
+Features:
+- Research-focused search with relevance scoring
+- Optional LLM-generated answers (include_answer=true)
+- Full raw content extraction (include_raw_content=true)
+- Topic filtering (general, news, finance)
+- Time range filtering (day, week, month, year)
+- Domain include/exclude lists
+"""
+
+from datetime import datetime
+import json
+from typing import Any
+
+import requests
+
+from ..base import BaseSearchProvider
+from ..types import Citation, SearchResult, WebSearchResponse
+from . import register_provider
+
+
+@register_provider("tavily")
+class TavilyProvider(BaseSearchProvider):
+    """Tavily research-focused search provider"""
+
+    name = "tavily"
+    display_name = "Tavily"
+    description = "Research-focused search"
+    supports_answer = True
+    BASE_URL = "https://api.tavily.com/search"
+
+    def search(
+        self,
+        query: str,
+        search_depth: str = "basic",  # basic, advanced
+        topic: str = "general",  # general, news, finance
+        max_results: int = 10,
+        include_answer: bool = True,  # Get LLM-generated answer
+        include_raw_content: bool = False,  # Get full page content
+        include_images: bool = False,
+        days: int | None = None,  # Time filter (1-365)
+        include_domains: list[str] | None = None,
+        exclude_domains: list[str] | None = None,
+        timeout: int = 60,
+        **kwargs: Any,
+    ) -> WebSearchResponse:
+        """
+        Perform research-focused search using Tavily API.
+
+        Args:
+            query: Search query.
+            search_depth: Search depth - "basic" (faster) or "advanced" (more thorough).
+            topic: Topic category - "general", "news", or "finance".
+            max_results: Maximum number of results (1-20).
+            include_answer: Include LLM-generated answer.
+            include_raw_content: Include full raw content of pages.
+            include_images: Include images in results.
+            days: Filter results to last N days (1-365).
+            include_domains: List of domains to include.
+            exclude_domains: List of domains to exclude.
+            timeout: Request timeout in seconds.
+            **kwargs: Additional options.
+
+        Returns:
+            WebSearchResponse: Standardized search response.
+        """
+        self.logger.debug(f"Calling Tavily API depth={search_depth}, max_results={max_results}")
+        payload: dict[str, Any] = {
+            "api_key": self.api_key,
+            "query": query,
+            "search_depth": search_depth,
+            "topic": topic,
+            "max_results": max_results,
+            "include_answer": include_answer,
+            "include_raw_content": include_raw_content,
+            "include_images": include_images,
+        }
+
+        if days is not None:
+            payload["days"] = days
+        if include_domains:
+            payload["include_domains"] = include_domains
+        if exclude_domains:
+            payload["exclude_domains"] = exclude_domains
+
+        response = requests.post(self.BASE_URL, json=payload, timeout=timeout)
+
+        if response.status_code != 200:
+            try:
+                error_data = response.json()
+            except (json.JSONDecodeError, ValueError):
+                error_data = {"error": response.text}
+            self.logger.error(f"Tavily API error: {response.status_code} - {error_data}")
+            raise Exception(
+                f"Tavily API error: {response.status_code} - "
+                f"{error_data.get('error', response.text)}"
+            )
+
+        data = response.json()
+        self.logger.debug(f"Tavily returned {len(data.get('results', []))} results")
+
+        # Extract answer
+        answer = data.get("answer", "")
+
+        # Extract search results
+        citations: list[Citation] = []
+        search_results: list[SearchResult] = []
+
+        for i, result in enumerate(data.get("results", []), 1):
+            sr = SearchResult(
+                title=result.get("title", ""),
+                url=result.get("url", ""),
+                snippet=result.get("content", ""),
+                date=result.get("published_date", ""),
+                source=result.get("source", ""),
+                content=result.get("raw_content", ""),
+                score=result.get("score", 0.0),
+            )
+            search_results.append(sr)
+
+            citations.append(
+                Citation(
+                    id=i,
+                    reference=f"[{i}]",
+                    url=result.get("url", ""),
+                    title=result.get("title", ""),
+                    snippet=result.get("content", ""),
+                    source=result.get("source", ""),
+                    content=result.get("raw_content", ""),
+                )
+            )
+
+        # Build metadata
+        metadata: dict[str, Any] = {
+            "finish_reason": "stop",
+            "search_depth": search_depth,
+            "topic": topic,
+        }
+
+        if data.get("images"):
+            metadata["images"] = data["images"]
+        if data.get("response_time"):
+            metadata["response_time"] = data["response_time"]
+
+        response_obj = WebSearchResponse(
+            query=query,
+            answer=answer,
+            provider="tavily",
+            timestamp=datetime.now().isoformat(),
+            model=f"tavily-{search_depth}",
+            citations=citations,
+            search_results=search_results,
+            usage={},  # Tavily doesn't provide token usage
+            metadata=metadata,
+        )
+
+        return response_obj

src/services/search/types.py

@@ -0,0 +1,114 @@
+"""
+Web Search Types - Shared dataclasses and type definitions
+
+This module defines the standardized types used across all search providers.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Citation:
+    """Standardized citation from search results"""
+
+    id: int
+    reference: str  # e.g., "[1]"
+    url: str
+    title: str = ""
+    snippet: str = ""
+    date: str = ""
+    source: str = ""
+    content: str = ""  # Full content if available
+    # Additional fields for backward compatibility with legacy format
+    type: str = "web"  # Citation type (web, pdf, etc.)
+    icon: str = ""  # Source icon URL
+    website: str = ""  # Website name
+    web_anchor: str = ""  # Web anchor text
+
+
+@dataclass
+class SearchResult:
+    """Individual search result item"""
+
+    title: str
+    url: str
+    snippet: str
+    date: str = ""
+    source: str = ""
+    content: str = ""  # Full content if available (e.g., from Jina)
+    score: float = 0.0  # Relevance score if available
+    # Additional fields for rich results
+    sitelinks: list[dict[str, str]] = field(default_factory=list)
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class WebSearchResponse:
+    """Standardized response from any search provider"""
+
+    query: str
+    answer: str  # LLM-generated answer or empty for raw SERP providers
+    provider: str
+    timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
+    model: str = ""
+    citations: list[Citation] = field(default_factory=list)
+    search_results: list[SearchResult] = field(default_factory=list)
+    usage: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary (backward compatible format)"""
+        result = {
+            "timestamp": self.timestamp,
+            "query": self.query,
+            "model": self.model,
+            "provider": self.provider,
+            "answer": self.answer,
+            "response": {
+                "content": self.answer,
+                "role": "assistant",
+                "finish_reason": self.metadata.get("finish_reason", "stop"),
+            },
+            "usage": self.usage,
+            "citations": [
+                {
+                    "id": c.id,
+                    "reference": c.reference,
+                    "url": c.url,
+                    "title": c.title,
+                    "snippet": c.snippet,
+                    "date": c.date,
+                    "source": c.source,
+                    "content": c.content,
+                    "type": c.type,
+                    "icon": c.icon,
+                    "website": c.website,
+                    "web_anchor": c.web_anchor,
+                }
+                for c in self.citations
+            ],
+            "search_results": [
+                {
+                    "title": r.title,
+                    "url": r.url,
+                    "snippet": r.snippet,
+                    "date": r.date,
+                    "source": r.source,
+                    "content": r.content,
+                    "score": r.score,
+                    "sitelinks": r.sitelinks,
+                    "attributes": r.attributes,
+                }
+                for r in self.search_results
+            ],
+        }
+        # Add any extra metadata that isn't already in the result
+        for key, value in self.metadata.items():
+            if key not in result and key != "finish_reason":
+                result[key] = value
+        return result
+
+
+__all__ = ["Citation", "SearchResult", "WebSearchResponse"]

src/services/setup/__init__.py

@@ -0,0 +1,34 @@
+"""
+Setup Service
+=============
+
+System setup and initialization for DeepTutor.
+
+Port configuration is done via .env file:
+    BACKEND_PORT=8001   (default: 8001)
+    FRONTEND_PORT=3782  (default: 3782)
+
+Usage:
+    from src.services.setup import init_user_directories, get_backend_port
+
+    # Initialize user directories
+    init_user_directories()
+
+    # Get server ports (from .env)
+    backend_port = get_backend_port()
+    frontend_port = get_frontend_port()
+"""
+
+from .init import (
+    get_backend_port,
+    get_frontend_port,
+    get_ports,
+    init_user_directories,
+)
+
+__all__ = [
+    "init_user_directories",
+    "get_backend_port",
+    "get_frontend_port",
+    "get_ports",
+]