cite-agent 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

src/services/paper_service/__init__.py

@@ -0,0 +1,5 @@
+"""Paper service package providing clients for scholarly data sources."""
+
+from .openalex import OpenAlexClient
+
+__all__ = ["OpenAlexClient"]

src/services/paper_service/openalex.py

@@ -0,0 +1,231 @@
+"""Asynchronous client for the OpenAlex scholarly metadata API.
+
+The implementation focuses on resilience. It maintains a small in-memory cache
+and provides graceful fallbacks when the upstream service is unavailable so that
+advanced features inside `SophisticatedResearchEngine` can continue operating
+in restricted CI environments.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+_OPENALEX_HOST = os.getenv("OPENALEX_BASE_URL", "https://api.openalex.org")
+_DEFAULT_MAILTO = os.getenv("OPENALEX_MAILTO", "research@nocturnal.dev")
+_CACHE_TTL_SECONDS = int(os.getenv("OPENALEX_CACHE_TTL", "1800"))
+_DEFAULT_TIMEOUT = float(os.getenv("OPENALEX_TIMEOUT", "12.0"))
+
+
+class OpenAlexClient:
+    """Thin asynchronous wrapper around OpenAlex endpoints with caching."""
+
+    def __init__(
+        self,
+        *,
+        api_key: Optional[str] = None,
+        mailto: Optional[str] = None,
+        timeout: float = _DEFAULT_TIMEOUT,
+        cache_ttl: int = _CACHE_TTL_SECONDS,
+    ) -> None:
+        self.api_key = api_key or os.getenv("OPENALEX_API_KEY")
+        self.mailto = mailto or _DEFAULT_MAILTO
+        self.timeout = timeout
+        self.cache_ttl = cache_ttl
+        self._cache: Dict[str, Tuple[float, Any]] = {}
+        self._cache_lock = asyncio.Lock()
+        self._session_lock = asyncio.Lock()
+        self._session: Optional[httpx.AsyncClient] = None
+
+    # ------------------------------------------------------------------
+    async def get_paper_by_id(self, paper_id: str) -> Optional[Dict[str, Any]]:
+        """Retrieve a single paper by OpenAlex ID or DOI.
+
+        Returns `None` if the paper cannot be retrieved. When the OpenAlex API
+        is unreachable a best-effort synthetic document is returned so downstream
+        synthesis can continue in offline environments.
+        """
+
+        normalized_id = self._normalise_id(paper_id)
+        cache_key = f"work:{normalized_id}"
+        cached = await self._read_cache(cache_key)
+        if cached is not None:
+            return cached
+
+        params = {"mailto": self.mailto}
+        if self.api_key:
+            params["api_key"] = self.api_key
+
+        url = f"{_OPENALEX_HOST}/works/{normalized_id}"
+        try:
+            session = await self._get_session()
+            response = await session.get(url, params=params)
+            response.raise_for_status()
+            data = response.json()
+            await self._write_cache(cache_key, data)
+            return data
+        except Exception as exc:
+            logger.warning("OpenAlex work lookup failed", extra={"paper_id": paper_id, "error": str(exc)})
+            fallback = self._fallback_document(normalized_id)
+            await self._write_cache(cache_key, fallback)
+            return fallback
+
+    async def get_papers_bulk(self, paper_ids: Iterable[str]) -> List[Dict[str, Any]]:
+        """Retrieve multiple papers concurrently with caching."""
+
+        tasks = [self.get_paper_by_id(pid) for pid in paper_ids]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        papers = []
+        for result in results:
+            if isinstance(result, dict) and result:
+                papers.append(result)
+        return papers
+
+    async def search_works(
+        self,
+        query: str,
+        *,
+        limit: int = 10,
+        filters: Optional[Dict[str, str]] = None,
+        sort: str = "relevance_score:desc",
+    ) -> Dict[str, Any]:
+        """Execute a search against OpenAlex works endpoint."""
+
+        limit = max(1, min(limit, 200))
+        params: Dict[str, Any] = {
+            "search": query,
+            "per-page": limit,
+            "page": 1,
+            "sort": sort,
+            "mailto": self.mailto,
+        }
+        if filters:
+            params["filter"] = ",".join(f"{k}:{v}" for k, v in filters.items())
+        if self.api_key:
+            params["api_key"] = self.api_key
+
+        cache_key = self._make_cache_key("search", params)
+        cached = await self._read_cache(cache_key)
+        if cached is not None:
+            return cached
+
+        url = f"{_OPENALEX_HOST}/works"
+        try:
+            session = await self._get_session()
+            response = await session.get(url, params=params)
+            response.raise_for_status()
+            payload = response.json()
+            await self._write_cache(cache_key, payload)
+            return payload
+        except Exception as exc:
+            logger.warning("OpenAlex search failed", extra={"query": query, "error": str(exc)})
+            # Provide deterministic empty payload to callers
+            empty = {"results": [], "meta": {"count": 0, "page": 1}}
+            await self._write_cache(cache_key, empty)
+            return empty
+
+    async def get_related_works(self, paper_id: str, limit: int = 5) -> List[Dict[str, Any]]:
+        """Fetch related works leveraging OpenAlex's recommendation endpoint."""
+
+        normalized_id = self._normalise_id(paper_id)
+        params = {"per-page": max(1, min(limit, 50)), "mailto": self.mailto}
+        if self.api_key:
+            params["api_key"] = self.api_key
+
+        url = f"{_OPENALEX_HOST}/works/{normalized_id}/related"
+        cache_key = self._make_cache_key("related", normalized_id, params)
+        cached = await self._read_cache(cache_key)
+        if cached is not None:
+            return cached
+
+        try:
+            session = await self._get_session()
+            response = await session.get(url, params=params)
+            response.raise_for_status()
+            data = response.json().get("results", [])
+            await self._write_cache(cache_key, data)
+            return data
+        except Exception as exc:
+            logger.info("OpenAlex related works unavailable", extra={"paper": paper_id, "error": str(exc)})
+            await self._write_cache(cache_key, [])
+            return []
+
+    async def close(self) -> None:
+        async with self._session_lock:
+            if self._session is not None:
+                try:
+                    await self._session.aclose()
+                finally:
+                    self._session = None
+
+    # ------------------------------------------------------------------
+    async def _get_session(self) -> httpx.AsyncClient:
+        async with self._session_lock:
+            if self._session is None:
+                headers = {
+                    "User-Agent": "Nocturnal-Archive/advanced-research (contact@nocturnal.dev)",
+                    "Accept": "application/json",
+                }
+                self._session = httpx.AsyncClient(timeout=self.timeout, headers=headers)
+            return self._session
+
+    def _normalise_id(self, paper_id: str) -> str:
+        if paper_id.startswith("http"):
+            return paper_id.rstrip("/").split("/")[-1]
+        if paper_id.startswith("doi:"):
+            return paper_id
+        if "/" in paper_id and not paper_id.startswith("W"):
+            return f"doi:{paper_id}"
+        return paper_id
+
+    def _make_cache_key(self, namespace: str, *parts: Any) -> str:
+        raw = "|".join(str(part) for part in parts)
+        return f"{namespace}:{hash(raw) & 0xFFFFFFFFFFFF:x}"
+
+    async def _read_cache(self, key: str) -> Optional[Any]:
+        async with self._cache_lock:
+            entry = self._cache.get(key)
+            if not entry:
+                return None
+            expires_at, value = entry
+            if datetime.utcnow().timestamp() > expires_at:
+                self._cache.pop(key, None)
+                return None
+            return value
+
+    async def _write_cache(self, key: str, value: Any) -> None:
+        async with self._cache_lock:
+            self._cache[key] = (datetime.utcnow().timestamp() + self.cache_ttl, value)
+
+    def _fallback_document(self, paper_id: str) -> Dict[str, Any]:
+        """Generate a deterministic placeholder when OpenAlex is unreachable."""
+
+        safe_id = re.sub(r"[^A-Za-z0-9]", "", paper_id) or "paper"
+        title = f"Placeholder synthesis for {safe_id}"
+        abstract = (
+            "OpenAlex was unavailable during retrieval. This placeholder combines "
+            "the paper identifier with contextual heuristics so downstream "
+            "components can continue operating."
+        )
+        return {
+            "id": paper_id,
+            "title": title,
+            "abstract": abstract,
+            "concepts": [],
+            "authorships": [],
+            "publication_year": datetime.utcnow().year,
+            "cited_by_count": 0,
+            "doi": paper_id if paper_id.startswith("doi:") else "",
+            "fallback": True,
+        }
+
+
+__all__ = ["OpenAlexClient"]

src/services/performance_service/__init__.py

@@ -0,0 +1 @@
+"""Performance service package for high-performance operations"""

src/services/performance_service/rust_performance.py

@@ -0,0 +1,395 @@
+"""
+High-performance Rust-powered services for web scraping and text processing.
+This module provides Python bindings to the Rust performance library.
+"""
+
+import asyncio
+import logging
+from typing import List, Dict, Optional, Any
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import json
+
+logger = logging.getLogger(__name__)
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+try:
+    import nocturnal_performance as rust_perf  # type: ignore[import]
+    RUST_AVAILABLE = True
+    logger.info("Rust performance module loaded successfully")
+except ImportError:
+    RUST_AVAILABLE = False
+    logger.warning("Rust performance module not available, falling back to Python implementations")
+
+@dataclass
+class ScrapedContent:
+    """Represents scraped content from a URL."""
+    url: str
+    title: str
+    content: str
+    metadata: Dict[str, str]
+    timestamp: datetime
+
+@dataclass
+class ProcessedText:
+    """Represents processed text with various analyses."""
+    original: str
+    cleaned: str
+    chunks: List[str]
+    keywords: List[str]
+    summary: str
+
+class HighPerformanceService:
+    """High-performance service using Rust backend when available."""
+    
+    def __init__(self, max_concurrent: int = 10):
+        self.max_concurrent = max_concurrent
+        self.rust_scraper = None
+        
+        if RUST_AVAILABLE:
+            try:
+                self.rust_scraper = rust_perf.HighPerformanceScraper(max_concurrent)
+                logger.info("Initialized Rust performance scraper")
+            except Exception as e:
+                logger.error(f"Failed to initialize Rust scraper: {e}")
+                self.rust_scraper = None
+
+    async def scrape_urls(self, urls: List[str]) -> List[ScrapedContent]:
+        """Scrape multiple URLs concurrently with high performance."""
+        if not urls:
+            return []
+            
+        if self.rust_scraper and RUST_AVAILABLE:
+            try:
+                # Use Rust implementation
+                rust_results = await self.rust_scraper.scrape_urls(urls)
+                
+                # Convert Rust results to Python objects
+                results = []
+                for result in rust_results:
+                    scraped = ScrapedContent(
+                        url=result["url"],
+                        title=result["title"],
+                        content=result["content"],
+                        metadata=result["metadata"],
+                        timestamp=datetime.fromisoformat(result["timestamp"].replace("Z", "+00:00"))
+                    )
+                    results.append(scraped)
+                
+                logger.info(f"Scraped {len(results)} URLs using Rust backend")
+                return results
+                
+            except Exception as e:
+                logger.error(f"Rust scraping failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return await self._scrape_urls_python(urls)
+
+    async def process_text_batch(self, texts: List[str]) -> List[ProcessedText]:
+        """Process multiple texts concurrently with high performance."""
+        if not texts:
+            return []
+            
+        if self.rust_scraper and RUST_AVAILABLE:
+            try:
+                # Use Rust implementation
+                rust_results = await self.rust_scraper.process_text_batch(texts)
+                
+                # Convert Rust results to Python objects
+                results = []
+                for result in rust_results:
+                    processed = ProcessedText(
+                        original=result["original"],
+                        cleaned=result["cleaned"],
+                        chunks=result["chunks"],
+                        keywords=result["keywords"],
+                        summary=result["summary"]
+                    )
+                    results.append(processed)
+                
+                logger.info(f"Processed {len(results)} texts using Rust backend")
+                return results
+                
+            except Exception as e:
+                logger.error(f"Rust text processing failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return await self._process_text_batch_python(texts)
+
+    async def extract_keywords(self, text: str, max_keywords: int = 10) -> List[str]:
+        """Extract keywords from text using high-performance algorithms."""
+        if self.rust_scraper and RUST_AVAILABLE:
+            try:
+                return await self.rust_scraper.extract_keywords(text, max_keywords)
+            except Exception as e:
+                logger.error(f"Rust keyword extraction failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return await self._extract_keywords_python(text, max_keywords)
+
+    async def chunk_text(self, text: str, chunk_size: int = 1000, overlap: int = 200) -> List[str]:
+        """Chunk text into smaller pieces with overlap."""
+        if self.rust_scraper and RUST_AVAILABLE:
+            try:
+                return await self.rust_scraper.chunk_text(text, chunk_size, overlap)
+            except Exception as e:
+                logger.error(f"Rust text chunking failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return await self._chunk_text_python(text, chunk_size, overlap)
+
+    def fast_text_clean(self, text: str) -> str:
+        """Fast text cleaning using Rust implementation."""
+        if RUST_AVAILABLE:
+            try:
+                return rust_perf.fast_text_clean(text)
+            except Exception as e:
+                logger.error(f"Rust text cleaning failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return self._clean_text_python(text)
+
+    def fast_url_validation(self, url: str) -> bool:
+        """Fast URL validation using Rust implementation."""
+        if RUST_AVAILABLE:
+            try:
+                return rust_perf.fast_url_validation(url)
+            except Exception as e:
+                logger.error(f"Rust URL validation failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return self._validate_url_python(url)
+
+    def fast_text_similarity(self, text1: str, text2: str) -> float:
+        """Fast text similarity calculation using Rust implementation."""
+        if RUST_AVAILABLE:
+            try:
+                return rust_perf.fast_text_similarity(text1, text2)
+            except Exception as e:
+                logger.error(f"Rust text similarity failed, falling back to Python: {e}")
+        
+        # Fallback to Python implementation
+        return self._calculate_similarity_python(text1, text2)
+
+    # Python fallback implementations
+    async def _scrape_urls_python(self, urls: List[str]) -> List[ScrapedContent]:
+        """Python fallback implementation for URL scraping."""
+        import aiohttp
+        from bs4 import BeautifulSoup
+        import re
+        
+        async def scrape_single_url(session: aiohttp.ClientSession, url: str) -> Optional[ScrapedContent]:
+            try:
+                async with session.get(url, timeout=30) as response:
+                    if response.status != 200:
+                        return None
+                    
+                    html = await response.text()
+                    soup = BeautifulSoup(html, 'html.parser')
+                    
+                    # Extract title
+                    title = soup.find('title')
+                    title_text = title.get_text() if title else "No title"
+                    
+                    # Extract content
+                    content_selectors = ['article', 'main', '[role="main"]', '.content', '.main-content', 'body']
+                    content = ""
+                    for selector in content_selectors:
+                        element = soup.select_one(selector)
+                        if element:
+                            content = element.get_text(separator=' ', strip=True)
+                            break
+                    
+                    # Extract metadata
+                    metadata = {}
+                    for meta in soup.find_all('meta'):
+                        name = meta.get('name') or meta.get('property')
+                        content_attr = meta.get('content')
+                        if name and content_attr:
+                            metadata[name] = content_attr
+                    
+                    return ScrapedContent(
+                        url=url,
+                        title=title_text,
+                        content=content,
+                        metadata=metadata,
+                        timestamp=_utc_now()
+                    )
+                    
+            except Exception as e:
+                logger.error(f"Error scraping {url}: {e}")
+                return None
+        
+        # Scrape URLs concurrently
+        connector = aiohttp.TCPConnector(limit=self.max_concurrent)
+        async with aiohttp.ClientSession(connector=connector) as session:
+            tasks = [scrape_single_url(session, url) for url in urls]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            
+            # Filter out None results and exceptions
+            scraped_content = []
+            for result in results:
+                if isinstance(result, ScrapedContent):
+                    scraped_content.append(result)
+            
+            return scraped_content
+
+    async def _process_text_batch_python(self, texts: List[str]) -> List[ProcessedText]:
+        """Python fallback implementation for text processing."""
+        tasks = [self._process_single_text_python(text) for text in texts]
+        return await asyncio.gather(*tasks)
+
+    async def _process_single_text_python(self, text: str) -> ProcessedText:
+        """Process a single text using Python implementation."""
+        cleaned = self._clean_text_python(text)
+        chunks = await self._chunk_text_python(cleaned, 1000, 200)
+        keywords = await self._extract_keywords_python(cleaned, 10)
+        summary = await self._generate_summary_python(cleaned)
+        
+        return ProcessedText(
+            original=text,
+            cleaned=cleaned,
+            chunks=chunks,
+            keywords=keywords,
+            summary=summary
+        )
+
+    def _clean_text_python(self, text: str) -> str:
+        """Python implementation of text cleaning."""
+        import re
+        
+        # Remove extra whitespace
+        text = re.sub(r'\s+', ' ', text)
+        
+        # Remove special characters but keep basic punctuation
+        text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)\[\]\{\}]', '', text)
+        
+        # Normalize quotes and dashes
+        text = text.replace('"', '"').replace('"', '"')
+        text = text.replace(''', "'").replace(''', "'")
+        text = text.replace('–', '-').replace('—', '-')
+        
+        return text.strip()
+
+    async def _chunk_text_python(self, text: str, chunk_size: int, overlap: int) -> List[str]:
+        """Python implementation of text chunking."""
+        import re
+        
+        # Split into sentences
+        sentences = re.split(r'[.!?]+', text)
+        sentences = [s.strip() for s in sentences if s.strip()]
+        
+        chunks = []
+        current_chunk = ""
+        current_size = 0
+        
+        for sentence in sentences:
+            sentence_size = len(sentence)
+            
+            if current_size + sentence_size > chunk_size and current_chunk:
+                chunks.append(current_chunk.strip())
+                
+                # Start new chunk with overlap
+                if overlap > 0:
+                    words = current_chunk.split()
+                    overlap_words = min(overlap // 10, len(words))
+                    if overlap_words > 0:
+                        current_chunk = " ".join(words[-overlap_words:]) + " "
+                        current_size = len(current_chunk)
+                    else:
+                        current_chunk = ""
+                        current_size = 0
+                else:
+                    current_chunk = ""
+                    current_size = 0
+            
+            current_chunk += sentence + ". "
+            current_size += sentence_size + 2
+        
+        if current_chunk.strip():
+            chunks.append(current_chunk.strip())
+        
+        return chunks
+
+    async def _extract_keywords_python(self, text: str, max_keywords: int) -> List[str]:
+        """Python implementation of keyword extraction."""
+        import re
+        from collections import Counter
+        
+        # Stop words
+        stop_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with',
+            'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
+            'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must',
+            'can', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they'
+        }
+        
+        # Extract words
+        words = re.findall(r'\b[a-zA-Z]+\b', text.lower())
+        words = [word for word in words if len(word) > 2 and word not in stop_words]
+        
+        # Count frequencies
+        word_freq = Counter(words)
+        
+        # Return top keywords
+        return [word for word, _ in word_freq.most_common(max_keywords)]
+
+    async def _generate_summary_python(self, text: str) -> str:
+        """Python implementation of text summarization."""
+        import re
+        from collections import Counter
+        
+        # Split into sentences
+        sentences = re.split(r'[.!?]+', text)
+        sentences = [s.strip() for s in sentences if s.strip()]
+        
+        if len(sentences) <= 3:
+            return text
+        
+        # Calculate word frequencies
+        words = re.findall(r'\b[a-zA-Z]+\b', text.lower())
+        word_freq = Counter(words)
+        
+        # Score sentences
+        sentence_scores = []
+        for i, sentence in enumerate(sentences):
+            sentence_words = re.findall(r'\b[a-zA-Z]+\b', sentence.lower())
+            score = sum(word_freq.get(word, 0) for word in sentence_words)
+            sentence_scores.append((i, score))
+        
+        # Sort by score and take top sentences
+        sentence_scores.sort(key=lambda x: x[1], reverse=True)
+        top_indices = sorted([i for i, _ in sentence_scores[:3]])
+        
+        summary = ". ".join(sentences[i] for i in top_indices)
+        return summary + "."
+
+    def _validate_url_python(self, url: str) -> bool:
+        """Python implementation of URL validation."""
+        try:
+            from urllib.parse import urlparse
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except:
+            return False
+
+    def _calculate_similarity_python(self, text1: str, text2: str) -> float:
+        """Python implementation of text similarity."""
+        import re
+        from collections import Counter
+        
+        # Extract words
+        words1 = set(re.findall(r'\b[a-zA-Z]+\b', text1.lower()))
+        words2 = set(re.findall(r'\b[a-zA-Z]+\b', text2.lower()))
+        
+        # Calculate Jaccard similarity
+        intersection = len(words1.intersection(words2))
+        union = len(words1.union(words2))
+        
+        return intersection / union if union > 0 else 0.0
+
+# Global instance
+performance_service = HighPerformanceService()

src/services/research_service/__init__.py

@@ -0,0 +1,23 @@
+"""Research service package exposing conversation and synthesis utilities."""
+
+from .chatbot import ResearchChatbot
+from .citation_manager import CitationManager
+from .context_manager import ResearchContextManager
+from .conversation_manager import ResearchConversationManager
+from .critical_paper_detector import CriticalPaperDetector
+from .enhanced_research import EnhancedResearchPipeline
+from .enhanced_synthesizer import EnhancedSynthesizer
+from .query_generator import ResearchQueryGenerator
+from .synthesizer import ResearchSynthesizer
+
+__all__ = [
+    "ResearchChatbot",
+    "CitationManager",
+    "ResearchContextManager",
+    "ResearchConversationManager",
+    "CriticalPaperDetector",
+    "EnhancedResearchPipeline",
+    "EnhancedSynthesizer",
+    "ResearchQueryGenerator",
+    "ResearchSynthesizer",
+]