arionxiv 1.0.32__py3-none-any.whl

arionxiv/__init__.py

@@ -0,0 +1,40 @@
+"""
+ArionXiv - AI-Powered Research Paper Analysis and Management
+
+A comprehensive tool for discovering, analyzing, and managing research papers
+from arXiv with AI-powered insights and organizational features.
+"""
+
+__version__ = "1.0.29"
+__author__ = "Arion Das"
+__email__ = "ariondasad@gmail.com"
+__description__ = "AI-Powered Research Paper Analysis and Management"
+
+# Lazy imports to avoid requiring fastapi for CLI/GitHub Actions usage
+# Services are imported on-demand when accessed
+def __getattr__(name):
+    """Lazy import of services to avoid loading fastapi for CLI usage."""
+    if name == "config":
+        from .services.unified_config_service import config
+        return config
+    elif name == "database_service":
+        from .services.unified_database_service import database_service
+        return database_service
+    elif name == "paper_service":
+        from .services.unified_paper_service import paper_service
+        return paper_service
+    elif name == "analysis_service":
+        from .services.unified_analysis_service import analysis_service
+        return analysis_service
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+__all__ = [
+    "__version__",
+    "__author__", 
+    "__email__",
+    "__description__",
+    "config",
+    "database_service",
+    "paper_service", 
+    "analysis_service"
+]

arionxiv/__main__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+ArionXiv package main entry point
+"""
+
+import sys
+from .cli.main import cli
+
+if __name__ == "__main__":
+    cli()

arionxiv/arxiv_operations/client.py

@@ -0,0 +1,225 @@
+# Arxiv API client for fetching papers
+import arxiv
+import logging
+from typing import List, Dict, Any, Optional
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+class ArxivClient:
+    """Client for interacting with Arxiv API"""
+    
+    def __init__(self):
+        self.client = arxiv.Client()
+        self.default_page_size = 100
+        self.max_results = 100
+    
+    # Short words to skip in title searches (arXiv doesn't index these well)
+    SKIP_WORDS = {'a', 'an', 'the', 'is', 'are', 'be', 'to', 'of', 'in', 'on', 
+                  'at', 'by', 'for', 'and', 'or', 'but', 'not', 'all', 'you', 
+                  'it', 'its', 'as', 'so', 'if', 'do', 'no', 'up', 'we', 'my'}
+    
+    def search_papers(self, query: str, max_results: int = None, sort_by=arxiv.SortCriterion.Relevance) -> List[Dict[str, Any]]:
+        """Search for papers on Arxiv with relevance scoring"""
+        try:
+            max_results = max_results or self.default_page_size
+            
+            # If query already contains arXiv operators (cat:, au:, ti:, abs:, AND, OR), use as-is
+            has_operators = any(op in query for op in ['cat:', 'au:', 'ti:', 'abs:', ' AND ', ' OR '])
+            
+            if has_operators:
+                # Query already formatted with operators
+                search_query = query
+            else:
+                # Build title search - skip short common words that arXiv doesn't handle well
+                words = [w.strip() for w in query.split() if w.strip()]
+                content_words = [w for w in words if w.lower() not in self.SKIP_WORDS]
+                
+                if content_words:
+                    title_parts = [f"ti:{word.title()}" for word in content_words]
+                    search_query = " AND ".join(title_parts)
+                else:
+                    # All words were skipped, use plain query
+                    search_query = query
+            
+            # Fetch more results than requested so we can filter/sort better
+            fetch_count = min(max_results * 3, self.max_results) if not has_operators else max_results
+            
+            search = arxiv.Search(
+                query=search_query,
+                max_results=min(fetch_count, self.max_results),
+                sort_by=sort_by
+            )
+            
+            papers = []
+            for result in self.client.results(search):
+                paper_data = {
+                    "arxiv_id": result.entry_id.split('/')[-1],
+                    "title": result.title,
+                    "abstract": result.summary,
+                    "authors": [str(author) for author in result.authors],
+                    "published": result.published.isoformat() if result.published else None,
+                    "updated": result.updated.isoformat() if result.updated else None,
+                    "categories": result.categories,
+                    "primary_category": result.primary_category,
+                    "pdf_url": result.pdf_url,
+                    "entry_id": result.entry_id,
+                    "doi": result.doi,
+                    "journal_ref": result.journal_ref,
+                    "comment": result.comment,
+                    "links": [{"href": link.href, "title": link.title, "rel": link.rel} for link in result.links]
+                }
+                papers.append(paper_data)
+            
+            # Re-score and sort papers by title match quality, then limit to requested count
+            if not has_operators and papers:
+                papers = self._score_and_sort_papers(papers, query)[:max_results]
+            
+            logger.info(f"Found {len(papers)} papers for query: {query}")
+            return papers
+        except Exception as e:
+            logger.error(f"Error searching papers: {str(e)}")
+            return []
+    
+    def _score_and_sort_papers(self, papers: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
+        """Score papers by how well their title matches the query and sort by score"""
+        query_lower = query.lower().strip()
+        query_words = set(query_lower.split())
+        
+        scored_papers = []
+        for paper in papers:
+            title_lower = paper['title'].lower()
+            score = 0
+            
+            # Exact title match (highest priority)
+            if title_lower == query_lower:
+                score += 1000
+            # Title contains the exact query phrase
+            elif query_lower in title_lower:
+                score += 500
+            
+            # Count matching words in title
+            title_words = set(title_lower.split())
+            matching_words = query_words & title_words
+            score += len(matching_words) * 50
+            
+            # Bonus for shorter titles (more likely to be exact match)
+            if len(title_words) <= len(query_words) + 2:
+                score += 100
+            
+            # Bonus for title starting with query words
+            if title_lower.startswith(query_lower.split()[0]):
+                score += 75
+            
+            scored_papers.append((score, paper))
+        
+        # Sort by score descending
+        scored_papers.sort(key=lambda x: x[0], reverse=True)
+        return [paper for score, paper in scored_papers]
+    
+    def get_paper_by_id(self, arxiv_id: str) -> Optional[Dict[str, Any]]:
+        """Get a specific paper by its Arxiv ID"""
+        try:
+            search = arxiv.Search(id_list=[arxiv_id])
+            
+            for result in self.client.results(search):
+                paper_data = {
+                    "arxiv_id": result.entry_id.split('/')[-1],
+                    "title": result.title,
+                    "abstract": result.summary,
+                    "authors": [str(author) for author in result.authors],
+                    "published": result.published.isoformat() if result.published else None,
+                    "updated": result.updated.isoformat() if result.updated else None,
+                    "categories": result.categories,
+                    "primary_category": result.primary_category,
+                    "pdf_url": result.pdf_url,
+                    "entry_id": result.entry_id,
+                    "doi": result.doi,
+                    "journal_ref": result.journal_ref,
+                    "comment": result.comment,
+                    "links": [{"href": link.href, "title": link.title, "rel": link.rel} for link in result.links]
+                }
+                return paper_data
+            
+            return None
+        except Exception as e:
+            logger.error(f"Error fetching paper {arxiv_id}: {str(e)}")
+            return None
+    
+    def get_recent_papers(self, category: str = None, days: int = 7, max_results: int = 50) -> List[Dict[str, Any]]:
+        """Get recent papers from the last N days"""
+        try:
+            # Build query for recent papers
+            query_parts = []
+            
+            if category:
+                query_parts.append(f"cat:{category}")
+            
+            # Date filter (Arxiv doesn't support date ranges directly, so we'll filter results)
+            cutoff_date = datetime.now() - timedelta(days=days)
+            
+            query = " AND ".join(query_parts) if query_parts else "all:machine learning"
+            
+            search = arxiv.Search(
+                query=query,
+                max_results=max_results,
+                sort_by=arxiv.SortCriterion.SubmittedDate
+            )
+            
+            papers = []
+            for result in self.client.results(search):
+                # Filter by date
+                if result.published and result.published.replace(tzinfo=None) >= cutoff_date:
+                    paper_data = {
+                        "arxiv_id": result.entry_id.split('/')[-1],
+                        "title": result.title,
+                        "abstract": result.summary,
+                        "authors": [str(author) for author in result.authors],
+                        "published": result.published.isoformat() if result.published else None,
+                        "updated": result.updated.isoformat() if result.updated else None,
+                        "categories": result.categories,
+                        "primary_category": result.primary_category,
+                        "pdf_url": result.pdf_url,
+                        "entry_id": result.entry_id,
+                        "doi": result.doi,
+                        "journal_ref": result.journal_ref,
+                        "comment": result.comment
+                    }
+                    papers.append(paper_data)
+            
+            logger.info(f"Found {len(papers)} recent papers in category: {category}")
+            return papers
+        except Exception as e:
+            logger.error(f"Error fetching recent papers: {str(e)}")
+            return []
+    
+    def get_papers_by_category(self, category: str, max_results: int = 20) -> List[Dict[str, Any]]:
+        """Get papers by category"""
+        try:
+            query = f"cat:{category}"
+            return self.search_papers(query, max_results)
+        except Exception as e:
+            logger.error(f"Error fetching papers by category {category}: {str(e)}")
+            return []
+    
+    def get_papers_by_author(self, author: str, max_results: int = 20) -> List[Dict[str, Any]]:
+        """Get papers by author"""
+        try:
+            query = f"au:{author}"
+            return self.search_papers(query, max_results)
+        except Exception as e:
+            logger.error(f"Error fetching papers by author {author}: {str(e)}")
+            return []
+    
+    def get_trending_papers(self, category: str = None, days: int = 30) -> List[Dict[str, Any]]:
+        """Get trending papers (most recent with high engagement indicators)"""
+        try:
+            # For now, we'll use recent papers as a proxy for trending
+            # In a full implementation, this could consider download counts, citations, etc.
+            return self.get_recent_papers(category=category, days=days, max_results=30)
+        except Exception as e:
+            logger.error(f"Error fetching trending papers: {str(e)}")
+            return []
+
+# Global instance
+arxiv_client = ArxivClient()

arionxiv/arxiv_operations/fetcher.py

@@ -0,0 +1,173 @@
+import requests
+import aiohttp
+import asyncio
+import os
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+from ..services.unified_pdf_service import pdf_processor
+
+logger = logging.getLogger(__name__)
+
+class ArxivFetcher:
+    """Fetches and processes papers from Arxiv"""
+    
+    def __init__(self):
+        self.session = None
+        self.download_dir = "downloads"
+        os.makedirs(self.download_dir, exist_ok=True)
+    
+    async def _get_session(self):
+        """Get or create aiohttp session"""
+        if self.session is None:
+            self.session = aiohttp.ClientSession()
+        return self.session
+    
+    async def fetch_paper_pdf(self, arxiv_id: str, pdf_url: str) -> Optional[str]:
+        """Fetch PDF for a paper"""
+        try:
+            session = await self._get_session()
+            
+            # Clean arxiv_id for filename
+            safe_id = arxiv_id.replace('/', '_').replace(':', '_')
+            pdf_path = os.path.join(self.download_dir, f"{safe_id}.pdf")
+            
+            # Check if already downloaded
+            if os.path.exists(pdf_path):
+                logger.info(f"PDF already exists: {pdf_path}")
+                return pdf_path
+            
+            # Download PDF
+            async with session.get(pdf_url) as response:
+                if response.status == 200:
+                    content = await response.read()
+                    with open(pdf_path, 'wb') as f:
+                        f.write(content)
+                    logger.info(f"Downloaded PDF: {pdf_path}")
+                    return pdf_path
+                else:
+                    logger.error(f"Failed to download PDF: {response.status}")
+                    return None
+        except Exception as e:
+            logger.error(f"Error fetching PDF for {arxiv_id}: {str(e)}")
+            return None
+    
+    async def fetch_and_process_paper(self, paper_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Fetch and process a complete paper"""
+        try:
+            arxiv_id = paper_data.get("arxiv_id")
+            pdf_url = paper_data.get("pdf_url")
+            
+            logger.info(f"Processing paper: {arxiv_id}")
+            
+            if not arxiv_id or not pdf_url:
+                logger.warning(f"Missing arxiv_id or pdf_url for paper")
+                return {"error": "Missing arxiv_id or pdf_url"}
+            
+            # Fetch PDF
+            logger.debug(f"Fetching PDF from: {pdf_url}")
+            pdf_path = await self.fetch_paper_pdf(arxiv_id, pdf_url)
+            if not pdf_path:
+                logger.error(f"Failed to download PDF for {arxiv_id}")
+                return {"error": "Failed to download PDF"}
+            
+            # Process PDF
+            logger.debug(f"Processing PDF: {pdf_path}")
+            processing_result = await pdf_processor.process_pdf(pdf_path)
+            
+            logger.info(f"Successfully processed paper: {arxiv_id}")
+            
+            # Combine paper metadata with processed content
+            result = {
+                **paper_data,
+                "pdf_path": pdf_path,
+                "processed_content": processing_result,
+                "fetch_timestamp": asyncio.get_event_loop().time()
+            }
+            
+            return result
+        except Exception as e:
+            logger.error(f"Error processing paper: {str(e)}")
+            return {"error": str(e)}
+    
+    async def batch_fetch_papers(self, papers: list) -> list:
+        """Fetch multiple papers concurrently"""
+        try:
+            logger.info(f"Starting batch fetch for {len(papers)} papers")
+            tasks = []
+            for paper in papers:
+                task = self.fetch_and_process_paper(paper)
+                tasks.append(task)
+            
+            # Limit concurrent downloads
+            semaphore = asyncio.Semaphore(3)
+            
+            async def bounded_fetch(paper):
+                async with semaphore:
+                    return await self.fetch_and_process_paper(paper)
+            
+            bounded_tasks = [bounded_fetch(paper) for paper in papers]
+            results = await asyncio.gather(*bounded_tasks, return_exceptions=True)
+            
+            # Filter out exceptions
+            successful_results = []
+            for result in results:
+                if isinstance(result, Exception):
+                    logger.error(f"Batch fetch error: {str(result)}", exc_info=True)
+                else:
+                    successful_results.append(result)
+            
+            logger.info(f"Batch fetch completed: {len(successful_results)}/{len(papers)} successful")
+            return successful_results
+        except Exception as e:
+            logger.error(f"Batch fetch error: {str(e)}", exc_info=True)
+            return []
+    
+    def fetch_paper_sync(self, arxiv_id: str, pdf_url: str) -> Optional[str]:
+        """Synchronous version of PDF fetch"""
+        try:
+            safe_id = arxiv_id.replace('/', '_').replace(':', '_')
+            pdf_path = os.path.join(self.download_dir, f"{safe_id}.pdf")
+            
+            if os.path.exists(pdf_path):
+                return pdf_path
+            
+            response = requests.get(pdf_url, timeout=30)
+            if response.status_code == 200:
+                with open(pdf_path, 'wb') as f:
+                    f.write(response.content)
+                logger.info(f"Downloaded PDF: {pdf_path}")
+                return pdf_path
+            else:
+                logger.error(f"Failed to download PDF: {response.status_code}")
+                return None
+        except Exception as e:
+            logger.error(f"Error fetching PDF sync for {arxiv_id}: {str(e)}")
+            return None
+    
+    async def cleanup_downloads(self, max_age_days: int = 7):
+        """Clean up old downloaded files"""
+        try:
+            import time
+            current_time = time.time()
+            max_age_seconds = max_age_days * 24 * 60 * 60
+            
+            for filename in os.listdir(self.download_dir):
+                file_path = os.path.join(self.download_dir, filename)
+                if os.path.isfile(file_path):
+                    file_age = current_time - os.path.getmtime(file_path)
+                    if file_age > max_age_seconds:
+                        os.remove(file_path)
+                        logger.info(f"Cleaned up old file: {filename}")
+        except Exception as e:
+            logger.error(f"Error during cleanup: {str(e)}")
+    
+    async def close(self):
+        """Close the session"""
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+# Global instance
+arxiv_fetcher = ArxivFetcher()

arionxiv/arxiv_operations/searcher.py

@@ -0,0 +1,122 @@
+# Simple text-based search for arXiv papers
+from typing import List, Dict, Any, Optional
+import logging
+
+from .client import arxiv_client
+
+logger = logging.getLogger(__name__)
+
+
+class ArxivSearcher:
+    """Simple text-based search for arXiv papers"""
+    
+    def __init__(self):
+        self.client = arxiv_client
+        
+        # Common categories for reference
+        self.categories = {
+            "cs.AI": "Artificial Intelligence",
+            "cs.LG": "Machine Learning", 
+            "cs.CV": "Computer Vision",
+            "cs.CL": "Computation and Language",
+            "cs.RO": "Robotics",
+            "stat.ML": "Machine Learning (Statistics)",
+            "cs.DC": "Distributed Computing",
+            "cs.DB": "Databases",
+            "cs.IR": "Information Retrieval",
+            "math.OC": "Optimization and Control",
+        }
+    
+    async def search(self, query: str, max_results: int = 10) -> Dict[str, Any]:
+        """
+        Simple text search that returns the closest matching papers.
+        
+        Args:
+            query: Search text
+            max_results: Number of results to return (default 10)
+        
+        Returns:
+            Dict with success status and list of papers
+        """
+        try:
+            logger.info(f"Searching arXiv: query='{query}', max_results={max_results}")
+            # Direct search via arXiv API (uses relevance sorting by default)
+            papers = self.client.search_papers(query=query, max_results=max_results)
+            
+            logger.info(f"Search completed: found {len(papers)} papers")
+            return {
+                "success": True,
+                "papers": papers,
+                "count": len(papers),
+                "query": query
+            }
+            
+        except Exception as e:
+            logger.error(f"Search failed: {str(e)}", exc_info=True)
+            return {"success": False, "error": str(e), "papers": []}
+    
+    async def search_by_category(self, query: str, category: str, max_results: int = 10) -> Dict[str, Any]:
+        """
+        Search within a specific category.
+        
+        Args:
+            query: Search text
+            category: arXiv category (e.g., cs.LG, cs.AI)
+            max_results: Number of results to return
+        
+        Returns:
+            Dict with success status and list of papers
+        """
+        try:
+            # Combine query with category filter
+            full_query = f"{query} AND cat:{category}" if query else f"cat:{category}"
+            logger.info(f"Searching by category: query='{full_query}', category={category}")
+            papers = self.client.search_papers(query=full_query, max_results=max_results)
+            
+            logger.info(f"Category search completed: found {len(papers)} papers in {category}")
+            return {
+                "success": True,
+                "papers": papers,
+                "count": len(papers),
+                "query": query,
+                "category": category
+            }
+            
+        except Exception as e:
+            logger.error(f"Category search failed: {str(e)}", exc_info=True)
+            return {"success": False, "error": str(e), "papers": []}
+    
+    async def search_by_author(self, author: str, max_results: int = 10) -> Dict[str, Any]:
+        """
+        Search papers by author name.
+        
+        Args:
+            author: Author name
+            max_results: Number of results to return
+        
+        Returns:
+            Dict with success status and list of papers
+        """
+        try:
+            logger.info(f"Searching by author: author='{author}', max_results={max_results}")
+            papers = self.client.get_papers_by_author(author=author, max_results=max_results)
+            
+            logger.info(f"Author search completed: found {len(papers)} papers by {author}")
+            return {
+                "success": True,
+                "papers": papers,
+                "count": len(papers),
+                "author": author
+            }
+            
+        except Exception as e:
+            logger.error(f"Author search failed: {str(e)}", exc_info=True)
+            return {"success": False, "error": str(e), "papers": []}
+    
+    def get_available_categories(self) -> Dict[str, str]:
+        """Get available paper categories"""
+        return self.categories.copy()
+
+
+# Global instance
+arxiv_searcher = ArxivSearcher()