scholarx 0.6.0__py3-none-any.whl

scholarx/__init__.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+"""ScholarX — Universal Research Paper API.
+
+A single entry point for querying research papers from arXiv, PMC,
+bioRxiv, medRxiv, PsyArXiv, OSF, and Semantic Scholar.
+"""
+
+import importlib
+import inspect
+
+__all__: list[str] = []
+
+CORE_MODULES = [
+    "scholarx.models",
+    "scholarx.api_client",
+    "scholarx.deduplication",
+    "scholarx.paper_storage",
+    "scholarx.scanner",
+]
+
+OPTIONAL_MODULES = {
+    "scholarx.agent_server": "agent_server",
+    "scholarx.mcp_server": "mcp_server",
+}
+
+
+def _import_module_safely(module_name: str):
+    """Try to import a module and return it, or None if not available."""
+    try:
+        return importlib.import_module(module_name)
+    except ImportError:
+        return None
+
+
+def _expose_members(module):
+    """Expose public classes and functions from a module into globals and __all__."""
+    for name, obj in inspect.getmembers(module):
+        if (inspect.isclass(obj) or inspect.isfunction(obj)) and not name.startswith("_"):
+            globals()[name] = obj
+            __all__.append(name)
+
+
+for _module_name in CORE_MODULES:
+    _module = _import_module_safely(_module_name)
+    if _module is not None:
+        _expose_members(_module)
+
+for _module_name, _extra_name in OPTIONAL_MODULES.items():
+    _module = _import_module_safely(_module_name)
+    if _module is not None:
+        _expose_members(_module)
+        globals()[f"_{_extra_name.upper()}_AVAILABLE"] = True
+    else:
+        globals()[f"_{_extra_name.upper()}_AVAILABLE"] = False
+
+__all__.extend(["_MCP_AVAILABLE", "_AGENT_AVAILABLE"])
+
+_MCP_AVAILABLE = globals().get("_MCP_SERVER_AVAILABLE", False)
+_AGENT_AVAILABLE = globals().get("_AGENT_SERVER_AVAILABLE", False)

scholarx/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+"""ScholarX CLI entry point."""
+
+from scholarx.cli import cli
+
+if __name__ == "__main__":
+    cli()

scholarx/agent_server.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python
+"""ScholarX Agent Server.
+
+Standard graph agent server following the genius-agent pattern.
+Uses create_graph_agent_server from agent-utilities for graph orchestration.
+"""
+
+import logging
+import os
+import sys
+import warnings
+
+# Filter warnings early
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    try:
+        from requests.exceptions import RequestsDependencyWarning
+
+        warnings.filterwarnings("ignore", category=RequestsDependencyWarning)
+    except ImportError:
+        pass
+
+warnings.filterwarnings("ignore", message=".*urllib3.*or chardet.*")
+warnings.filterwarnings("ignore", message=".*urllib3.*or charset_normalizer.*")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="fastmcp")
+
+from agent_utilities import (
+    build_system_prompt_from_workspace,
+    create_agent_parser,
+    create_graph_agent_server,
+    initialize_workspace,
+    load_identity,
+)
+
+__version__ = "0.6.0"
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+initialize_workspace()
+meta = load_identity()
+
+DEFAULT_AGENT_NAME = os.getenv("DEFAULT_AGENT_NAME", meta.get("name", "ScholarX Agent"))
+DEFAULT_AGENT_DESCRIPTION = os.getenv(
+    "AGENT_DESCRIPTION",
+    meta.get("description", "ScholarX — Universal research paper discovery and analysis."),
+)
+DEFAULT_AGENT_SYSTEM_PROMPT = os.getenv(
+    "AGENT_SYSTEM_PROMPT",
+    meta.get("content") or build_system_prompt_from_workspace(),
+)
+
+
+def agent_server():
+    """Agent server entry point."""
+    print(f"{DEFAULT_AGENT_NAME} v{__version__}", file=sys.stderr)
+    logger.info("Application startup complete")
+
+    parser = create_agent_parser()
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+        logger.debug("Debug mode enabled")
+
+    create_graph_agent_server(
+        mcp_url=args.mcp_url,
+        mcp_config=args.mcp_config or "mcp_config.json",
+        host=args.host,
+        port=args.port,
+        provider=args.provider,
+        model_id=args.model_id,
+        router_model=args.model_id,
+        agent_model=args.model_id,
+        base_url=args.base_url,
+        api_key=args.api_key,
+        custom_skills_directory=args.custom_skills_directory,
+        enable_web_ui=args.web,
+        enable_otel=args.otel,
+        otel_endpoint=args.otel_endpoint,
+        otel_headers=args.otel_headers,
+        otel_public_key=args.otel_public_key,
+        otel_secret_key=args.otel_secret_key,
+        otel_protocol=args.otel_protocol,
+        debug=args.debug,
+    )
+
+
+if __name__ == "__main__":
+    agent_server()

scholarx/api_client.py

@@ -0,0 +1,253 @@
+#!/usr/bin/python
+"""ScholarX Unified API Client.
+
+The single entry point for all paper operations. Users interact with
+ScholarXClient — never with providers directly. The client handles
+fan-out to configured providers, cross-source deduplication, and
+result aggregation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from .deduplication import deduplicate_papers
+from .models import (
+    DEFAULT_SOURCE_CONFIGS,
+    Paper,
+    PaperSource,
+    SearchQuery,
+    SearchResult,
+    SourceConfig,
+    SourceStatus,
+)
+from .paper_storage import PaperStorage
+from .providers.base import PaperProvider
+
+logger = logging.getLogger(__name__)
+
+
+def _create_provider(source: PaperSource, config: SourceConfig) -> PaperProvider:
+    """Factory to create a provider instance for a given source."""
+    from .providers.arxiv import ArxivProvider
+    from .providers.biorxiv import BiorxivProvider, MedrxivProvider
+    from .providers.osf import OSFProvider, PsyarxivProvider
+    from .providers.pmc import PMCProvider
+    from .providers.semantic_scholar import SemanticScholarProvider
+
+    _PROVIDER_MAP: dict[PaperSource, type[PaperProvider]] = {
+        PaperSource.ARXIV: ArxivProvider,
+        PaperSource.PMC: PMCProvider,
+        PaperSource.BIORXIV: BiorxivProvider,
+        PaperSource.MEDRXIV: MedrxivProvider,
+        PaperSource.PSYARXIV: PsyarxivProvider,
+        PaperSource.OSF: OSFProvider,
+        PaperSource.SEMANTIC_SCHOLAR: SemanticScholarProvider,
+    }
+
+    provider_cls = _PROVIDER_MAP.get(source)
+    if not provider_cls:
+        raise ValueError(f"Unknown paper source: {source}")
+    return provider_cls(config)
+
+
+class ScholarXClient:
+    """Universal research paper client.
+
+    Fan-out queries to all configured sources, deduplicate results,
+    and present a unified view. Users never need to know about
+    individual provider implementations.
+
+    Usage::
+
+        client = ScholarXClient()
+        result = await client.search(SearchQuery(query="multi-agent systems"))
+        for paper in result.papers:
+            print(f"{paper.title} ({paper.source})")
+    """
+
+    def __init__(
+        self,
+        sources: list[PaperSource] | None = None,
+        configs: dict[PaperSource, SourceConfig] | None = None,
+        storage_dir: str | None = None,
+    ):
+        """Initialize the client.
+
+        Args:
+            sources: List of sources to enable (default: all).
+            configs: Optional per-source configuration overrides.
+            storage_dir: Optional custom paper storage directory.
+        """
+        self._configs = configs or dict(DEFAULT_SOURCE_CONFIGS)
+        enabled_sources = sources or list(PaperSource)
+
+        self._providers: dict[PaperSource, PaperProvider] = {}
+        for source in enabled_sources:
+            config = self._configs.get(source, DEFAULT_SOURCE_CONFIGS.get(source))
+            if config and config.enabled:
+                try:
+                    self._providers[source] = _create_provider(source, config)
+                except Exception as e:
+                    logger.warning(f"Failed to initialize provider {source}: {e}")
+
+        self.storage = PaperStorage(storage_dir)
+        logger.info(
+            f"ScholarX initialized with {len(self._providers)} sources: {', '.join(s.value for s in self._providers)}"
+        )
+
+    @property
+    def enabled_sources(self) -> list[PaperSource]:
+        """List of currently enabled sources."""
+        return list(self._providers.keys())
+
+    async def search(self, query: SearchQuery) -> SearchResult:
+        """Search across all configured sources with deduplication.
+
+        Args:
+            query: Unified search query.
+
+        Returns:
+            Aggregated, deduplicated SearchResult.
+        """
+        # Filter to requested sources
+        target_sources = [s for s in query.sources if s in self._providers]
+        if not target_sources:
+            target_sources = list(self._providers.keys())
+
+        # Fan out to all providers concurrently
+        all_papers: list[Paper] = []
+        sources_failed: list[str] = []
+
+        async def _query_source(source: PaperSource) -> list[Paper]:
+            try:
+                return await self._providers[source].search(query)
+            except Exception as e:
+                logger.error(f"Search failed for {source.value}: {e}")
+                sources_failed.append(f"{source.value}: {e}")
+                return []
+
+        results = await asyncio.gather(
+            *[_query_source(s) for s in target_sources],
+            return_exceptions=True,
+        )
+
+        for i, result in enumerate(results):
+            if isinstance(result, list):
+                all_papers.extend(result)
+            elif isinstance(result, Exception):
+                sources_failed.append(f"{target_sources[i].value}: {result}")
+
+        # Deduplicate
+        deduped, dup_count = deduplicate_papers(all_papers)
+
+        # Sort
+        if query.sort_by == "date":
+            deduped.sort(key=lambda p: p.published_date or "", reverse=True)
+
+        return SearchResult(
+            papers=deduped[: query.max_results],
+            total_count=len(deduped),
+            sources_queried=[s for s in target_sources],
+            sources_failed=sources_failed,
+            deduplicated_count=dup_count,
+            query=query.query,
+        )
+
+    async def get_paper(self, source: PaperSource, paper_id: str) -> Paper | None:
+        """Retrieve a single paper from a specific source.
+
+        Args:
+            source: The paper source to query.
+            paper_id: Source-specific paper identifier.
+
+        Returns:
+            Paper or None if not found.
+        """
+        provider = self._providers.get(source)
+        if not provider:
+            logger.error(f"Source not configured: {source}")
+            return None
+        return await provider.get_paper(paper_id)
+
+    async def get_recent_papers(
+        self,
+        categories: list[str] | None = None,
+        days: int = 1,
+        sources: list[PaperSource] | None = None,
+    ) -> SearchResult:
+        """Retrieve recently published papers.
+
+        Args:
+            categories: Optional category filters.
+            days: Number of days to look back.
+            sources: Optional source filter (default: all).
+
+        Returns:
+            Aggregated, deduplicated SearchResult.
+        """
+        target_sources = [s for s in (sources or list(self._providers.keys())) if s in self._providers]
+
+        all_papers: list[Paper] = []
+        sources_failed: list[str] = []
+
+        async def _fetch(source: PaperSource) -> list[Paper]:
+            try:
+                return await self._providers[source].get_recent(categories, days)
+            except Exception as e:
+                logger.error(f"get_recent failed for {source.value}: {e}")
+                sources_failed.append(f"{source.value}: {e}")
+                return []
+
+        results = await asyncio.gather(
+            *[_fetch(s) for s in target_sources],
+            return_exceptions=True,
+        )
+
+        for i, result in enumerate(results):
+            if isinstance(result, list):
+                all_papers.extend(result)
+            elif isinstance(result, Exception):
+                sources_failed.append(f"{target_sources[i].value}: {result}")
+
+        deduped, dup_count = deduplicate_papers(all_papers)
+        deduped.sort(key=lambda p: p.published_date or "", reverse=True)
+
+        return SearchResult(
+            papers=deduped,
+            total_count=len(deduped),
+            sources_queried=target_sources,
+            sources_failed=sources_failed,
+            deduplicated_count=dup_count,
+            query="recent papers",
+        )
+
+    async def download_paper(self, paper: Paper) -> str | None:
+        """Download a paper's full PDF.
+
+        Args:
+            paper: Paper to download.
+
+        Returns:
+            Local file path string, or None on failure.
+        """
+        path = await self.storage.download_paper(paper)
+        return str(path) if path else None
+
+    async def get_source_status(self) -> list[SourceStatus]:
+        """Get the status of all configured sources."""
+        return [SourceStatus(source=source, available=True) for source in self._providers]
+
+    async def list_categories(self, source: PaperSource | None = None) -> dict[str, list[dict]]:
+        """List available categories for each source."""
+        result: dict[str, list[dict]] = {}
+        targets = [source] if source and source in self._providers else list(self._providers.keys())
+        for s in targets:
+            try:
+                cats = await self._providers[s].get_categories()
+                result[s.value] = cats
+            except Exception as e:
+                logger.warning(f"Failed to get categories for {s.value}: {e}")
+                result[s.value] = []
+        return result