noesium 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

noesium/toolkits/wikipedia_toolkit.py

@@ -0,0 +1,420 @@
+"""
+Wikipedia toolkit for encyclopedia search and content retrieval.
+
+Provides tools for searching Wikipedia articles, retrieving content,
+and accessing Wikipedia's vast knowledge base through the MediaWiki API.
+"""
+
+import datetime
+from typing import Callable, Dict, List, Optional
+
+import aiohttp
+
+from noesium.core.toolify.base import AsyncBaseToolkit
+from noesium.core.toolify.config import ToolkitConfig
+from noesium.core.toolify.registry import register_toolkit
+from noesium.core.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+try:
+    import wikipediaapi
+
+    WIKIPEDIA_API_AVAILABLE = True
+except ImportError:
+    wikipediaapi = None
+    WIKIPEDIA_API_AVAILABLE = False
+
+
+@register_toolkit("wikipedia")
+class WikipediaToolkit(AsyncBaseToolkit):
+    """
+    Toolkit for Wikipedia search and content retrieval.
+
+    This toolkit provides comprehensive access to Wikipedia's content through
+    both the wikipedia-api library and direct MediaWiki API calls. It supports
+    multiple languages, different content formats, and various search modes.
+
+    Features:
+    - Multi-language Wikipedia support
+    - Full article content and summaries
+    - Page search and disambiguation
+    - Category and link information
+    - Recent changes and trending topics
+    - Configurable output formats (Wiki markup or HTML)
+    - Rate limiting and error handling
+
+    Required dependency: wikipedia-api
+    Install with: pip install wikipedia-api
+    """
+
+    def __init__(self, config: ToolkitConfig = None):
+        """
+        Initialize the Wikipedia toolkit.
+
+        Args:
+            config: Toolkit configuration
+
+        Raises:
+            ImportError: If wikipedia-api package is not installed
+        """
+        super().__init__(config)
+
+        if not WIKIPEDIA_API_AVAILABLE:
+            raise ImportError(
+                "wikipedia-api package is required for WikipediaToolkit. " "Install with: pip install wikipedia-api"
+            )
+
+        # Configuration
+        self.user_agent = self.config.config.get("user_agent", "noesium-wikipedia-toolkit")
+        self.language = self.config.config.get("language", "en")
+        self.content_type = self.config.config.get("content_type", "text")  # "text" or "summary"
+        self.extract_format = self.config.config.get("extract_format", "WIKI")  # "WIKI" or "HTML"
+
+        # Map string format to wikipediaapi.ExtractFormat
+        extract_format_map = {
+            "WIKI": wikipediaapi.ExtractFormat.WIKI,
+            "HTML": wikipediaapi.ExtractFormat.HTML,
+        }
+
+        if self.extract_format not in extract_format_map:
+            self.logger.warning(f"Invalid extract_format: {self.extract_format}, using WIKI")
+            self.extract_format = "WIKI"
+
+        # Initialize Wikipedia API client
+        self.wiki_client = wikipediaapi.Wikipedia(
+            user_agent=self.user_agent, language=self.language, extract_format=extract_format_map[self.extract_format]
+        )
+
+        # MediaWiki API configuration
+        self.api_base_url = f"https://{self.language}.wikipedia.org/w/api.php"
+
+        self.logger.info(f"Wikipedia toolkit initialized for language: {self.language}")
+
+    async def _make_api_request(self, params: Dict) -> Dict:
+        """
+        Make a request to the MediaWiki API.
+
+        Args:
+            params: API parameters
+
+        Returns:
+            API response as dictionary
+        """
+        default_params = {"format": "json", "formatversion": "2"}
+        params.update(default_params)
+
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(self.api_base_url, params=params) as response:
+                    response.raise_for_status()
+                    return await response.json()
+
+        except Exception as e:
+            self.logger.error(f"Wikipedia API request failed: {e}")
+            return {"error": f"API request failed: {str(e)}"}
+
+    async def search_wikipedia(self, query: str, num_results: int = 5) -> List[Dict]:
+        """
+        Search Wikipedia for articles matching the query.
+
+        This tool searches Wikipedia for articles related to your query and returns
+        a list of matching articles with basic information. It's useful for finding
+        relevant Wikipedia pages before retrieving detailed content.
+
+        Args:
+            query: Search query string
+            num_results: Maximum number of results to return (default: 5)
+
+        Returns:
+            List of dictionaries containing search results with:
+            - title: Article title
+            - pageid: Wikipedia page ID
+            - snippet: Brief text snippet with search terms highlighted
+            - wordcount: Number of words in the article
+            - size: Article size in bytes
+            - timestamp: Last modification timestamp
+
+        Example:
+            results = await search_wikipedia("artificial intelligence", 3)
+            for result in results:
+                print(f"Title: {result['title']}")
+                print(f"Snippet: {result['snippet']}")
+        """
+        self.logger.info(f"Searching Wikipedia for: {query}")
+
+        try:
+            params = {
+                "action": "query",
+                "list": "search",
+                "srsearch": query,
+                "srlimit": min(num_results, 50),  # API limit
+                "srprop": "snippet|titlesnippet|size|wordcount|timestamp",
+            }
+
+            response = await self._make_api_request(params)
+
+            if "error" in response:
+                return [{"error": response["error"]}]
+
+            search_results = response.get("query", {}).get("search", [])
+
+            results = []
+            for result in search_results:
+                results.append(
+                    {
+                        "title": result.get("title", ""),
+                        "pageid": result.get("pageid"),
+                        "snippet": result.get("snippet", "")
+                        .replace('<span class="searchmatch">', "")
+                        .replace("</span>", ""),
+                        "wordcount": result.get("wordcount", 0),
+                        "size": result.get("size", 0),
+                        "timestamp": result.get("timestamp", ""),
+                    }
+                )
+
+            self.logger.info(f"Found {len(results)} search results")
+            return results
+
+        except Exception as e:
+            error_msg = f"Wikipedia search failed: {str(e)}"
+            self.logger.error(error_msg)
+            return [{"error": error_msg}]
+
+    async def get_wikipedia_page(self, title: str, content_type: Optional[str] = None) -> Dict:
+        """
+        Retrieve a Wikipedia page by title.
+
+        This tool fetches the complete content of a Wikipedia article by its title.
+        It can return either the full article text or just a summary, depending
+        on the configuration.
+
+        Args:
+            title: Wikipedia article title
+            content_type: "text" for full article, "summary" for summary only
+
+        Returns:
+            Dictionary containing:
+            - title: Article title
+            - content: Article content (full text or summary)
+            - url: Wikipedia URL
+            - exists: Whether the page exists
+            - categories: List of categories
+            - links: List of internal links
+            - references: List of external references
+            - summary: Article summary (always included)
+
+        Example:
+            page = await get_wikipedia_page("Python (programming language)")
+            print(f"Title: {page['title']}")
+            print(f"Summary: {page['summary'][:200]}...")
+        """
+        content_type = content_type or self.content_type
+
+        self.logger.info(f"Retrieving Wikipedia page: {title}")
+
+        try:
+            # Get the page using wikipedia-api
+            page = self.wiki_client.page(title)
+
+            if not page.exists():
+                return {"title": title, "exists": False, "error": f"Wikipedia page '{title}' does not exist"}
+
+            # Extract content based on type
+            if content_type == "summary":
+                content = page.summary
+            else:
+                content = page.text
+
+            # Get additional information
+            categories = list(page.categories.keys()) if hasattr(page, "categories") else []
+            links = list(page.links.keys()) if hasattr(page, "links") else []
+
+            result = {
+                "title": page.title,
+                "content": content,
+                "summary": page.summary,
+                "url": page.fullurl,
+                "exists": True,
+                "categories": categories[:20],  # Limit to first 20
+                "links": links[:50],  # Limit to first 50
+                "page_id": getattr(page, "pageid", None),
+                "language": self.language,
+                "content_type": content_type,
+            }
+
+            # Get references using API
+            try:
+                refs_params = {"action": "query", "prop": "extlinks", "titles": title, "ellimit": 20}
+                refs_response = await self._make_api_request(refs_params)
+
+                pages = refs_response.get("query", {}).get("pages", [])
+                if pages:
+                    extlinks = pages[0].get("extlinks", [])
+                    result["references"] = [link.get("*", "") for link in extlinks]
+                else:
+                    result["references"] = []
+
+            except Exception:
+                result["references"] = []
+
+            self.logger.info(f"Retrieved page: {page.title} ({len(content)} characters)")
+            return result
+
+        except Exception as e:
+            error_msg = f"Failed to retrieve Wikipedia page '{title}': {str(e)}"
+            self.logger.error(error_msg)
+            return {"title": title, "exists": False, "error": error_msg}
+
+    async def get_wikipedia_summary(self, title: str, sentences: int = 3) -> str:
+        """
+        Get a concise summary of a Wikipedia article.
+
+        Args:
+            title: Wikipedia article title
+            sentences: Number of sentences to include in summary
+
+        Returns:
+            Article summary text
+        """
+        try:
+            page_data = await self.get_wikipedia_page(title, content_type="summary")
+
+            if not page_data.get("exists", False):
+                return page_data.get("error", "Page not found")
+
+            summary = page_data.get("summary", "")
+
+            # Limit to specified number of sentences
+            if sentences > 0:
+                sentences_list = summary.split(". ")
+                if len(sentences_list) > sentences:
+                    summary = ". ".join(sentences_list[:sentences]) + "."
+
+            return summary
+
+        except Exception as e:
+            return f"Failed to get summary: {str(e)}"
+
+    async def get_random_wikipedia_page(self) -> Dict:
+        """
+        Get a random Wikipedia article.
+
+        Returns:
+            Dictionary with random article information
+        """
+        self.logger.info("Getting random Wikipedia page")
+
+        try:
+            params = {"action": "query", "list": "random", "rnnamespace": 0, "rnlimit": 1}  # Main namespace only
+
+            response = await self._make_api_request(params)
+
+            if "error" in response:
+                return {"error": response["error"]}
+
+            random_pages = response.get("query", {}).get("random", [])
+
+            if not random_pages:
+                return {"error": "No random page found"}
+
+            random_title = random_pages[0].get("title")
+
+            # Get the full page content
+            return await self.get_wikipedia_page(random_title)
+
+        except Exception as e:
+            error_msg = f"Failed to get random page: {str(e)}"
+            self.logger.error(error_msg)
+            return {"error": error_msg}
+
+    async def get_wikipedia_categories(self, title: str) -> List[str]:
+        """
+        Get categories for a Wikipedia article.
+
+        Args:
+            title: Wikipedia article title
+
+        Returns:
+            List of category names
+        """
+        try:
+            params = {"action": "query", "prop": "categories", "titles": title, "cllimit": 50}
+
+            response = await self._make_api_request(params)
+
+            if "error" in response:
+                return [f"Error: {response['error']}"]
+
+            pages = response.get("query", {}).get("pages", [])
+
+            if not pages:
+                return ["No categories found"]
+
+            categories = pages[0].get("categories", [])
+            return [cat.get("title", "").replace("Category:", "") for cat in categories]
+
+        except Exception as e:
+            return [f"Error getting categories: {str(e)}"]
+
+    async def get_page_views(self, title: str, days: int = 30) -> Dict:
+        """
+        Get page view statistics for a Wikipedia article.
+
+        Args:
+            title: Wikipedia article title
+            days: Number of days to look back (default: 30)
+
+        Returns:
+            Dictionary with view statistics
+        """
+        try:
+            # Calculate date range
+            end_date = datetime.datetime.now()
+            start_date = end_date - datetime.timedelta(days=days)
+
+            # Format dates for API
+            start_str = start_date.strftime("%Y%m%d")
+            end_str = end_date.strftime("%Y%m%d")
+
+            # Use Wikimedia REST API for pageviews
+            url = f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/{self.language}.wikipedia/all-access/user/{title}/daily/{start_str}/{end_str}"
+
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as response:
+                    if response.status == 200:
+                        data = await response.json()
+                        items = data.get("items", [])
+
+                        total_views = sum(item.get("views", 0) for item in items)
+                        avg_daily_views = total_views / len(items) if items else 0
+
+                        return {
+                            "title": title,
+                            "total_views": total_views,
+                            "average_daily_views": round(avg_daily_views, 2),
+                            "days_analyzed": len(items),
+                            "date_range": f"{start_str} to {end_str}",
+                        }
+                    else:
+                        return {"error": f"Failed to get page views: HTTP {response.status}"}
+
+        except Exception as e:
+            return {"error": f"Failed to get page views: {str(e)}"}
+
+    async def get_tools_map(self) -> Dict[str, Callable]:
+        """
+        Get the mapping of tool names to their implementation functions.
+
+        Returns:
+            Dictionary mapping tool names to callable functions
+        """
+        return {
+            "search_wikipedia": self.search_wikipedia,
+            "get_wikipedia_page": self.get_wikipedia_page,
+            "get_wikipedia_summary": self.get_wikipedia_summary,
+            "get_random_wikipedia_page": self.get_random_wikipedia_page,
+            "get_wikipedia_categories": self.get_wikipedia_categories,
+            "get_page_views": self.get_page_views,
+        }

{noesium-0.1.0.dist-info → noesium-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: noesium
-Version: 0.1.0
+Version: 0.2.0
 Summary: Towards a cognitive agentic framework
 Author-email: Xiaming Chen <chenxm35@gmail.com>
 Maintainer-email: Xiaming Chen <chenxm35@gmail.com>
@@ -20,50 +20,79 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: requests>=2.31.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: aiohttp>=3.12.15
+Requires-Dist: aiofiles>=24.1.0
+Requires-Dist: anyio>=4.9.0
 Requires-Dist: python-dotenv>=1.1.1
 Requires-Dist: colorlog>=6.8.0
 Requires-Dist: typing-extensions>=4.8.0
 Requires-Dist: deprecated>=1.2.18
-Requires-Dist: bubus>=1.5.6
-Requires-Dist: openai>=1.0.0
-Requires-Dist: instructor>=1.10.0
-Requires-Dist: google-genai>=1.5.0
-Requires-Dist: litellm>=1.0.0
-Requires-Dist: ollama>=0.5.3
-Requires-Dist: huggingface-hub>=0.34.4
-Requires-Dist: llama-cpp-python>=0.3.16
-Requires-Dist: langchain-core>=0.3.72
-Requires-Dist: langchain-text-splitters>=0.3.0
-Requires-Dist: langchain-ollama>=0.2.0
-Requires-Dist: langgraph>=0.5.4
+Requires-Dist: psutil>=7.0.0
 Requires-Dist: networkx>=3.5
-Requires-Dist: mcp>=1.0.0
-Requires-Dist: protobuf<6,>=5
-Requires-Dist: weaviate-client<5,>=4
-Requires-Dist: psycopg2>=2.9.10
-Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: bubus>=1.5.6
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Requires-Dist: instructor>=1.10.0; extra == "openai"
+Provides-Extra: google
+Requires-Dist: google-genai>=1.5.0; extra == "google"
+Requires-Dist: google-api-python-client>=2.174.0; extra == "google"
+Requires-Dist: google-auth-oauthlib>=1.2.2; extra == "google"
+Requires-Dist: google-auth>=2.40.3; extra == "google"
+Provides-Extra: aliyun
+Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.13.1; extra == "aliyun"
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.0.0; extra == "litellm"
+Provides-Extra: local-llm
+Requires-Dist: ollama>=0.5.3; extra == "local-llm"
+Requires-Dist: llama-cpp-python>=0.3.16; extra == "local-llm"
+Requires-Dist: huggingface-hub>=0.34.4; extra == "local-llm"
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3.72; extra == "langchain"
+Requires-Dist: langchain-text-splitters>=0.3.0; extra == "langchain"
+Requires-Dist: langchain-ollama>=0.2.0; extra == "langchain"
+Requires-Dist: langgraph>=0.5.4; extra == "langchain"
+Provides-Extra: postgres
+Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgres"
+Requires-Dist: psycopg2>=2.9.10; extra == "postgres"
+Provides-Extra: weaviate
+Requires-Dist: weaviate-client<5,>=4; extra == "weaviate"
+Requires-Dist: protobuf<6,>=5; extra == "weaviate"
+Provides-Extra: datascience
+Requires-Dist: matplotlib>=3.8.0; extra == "datascience"
+Requires-Dist: pexpect>=4.9.0; extra == "datascience"
+Requires-Dist: ipython>=8.18.0; extra == "datascience"
+Requires-Dist: pandas>=2.0.0; extra == "datascience"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.0.0; extra == "mcp"
+Provides-Extra: tools
+Requires-Dist: noesium[aliyun,datascience,google,mcp]; extra == "tools"
+Requires-Dist: wizsearch<2.0.0,>=1.0.1; extra == "tools"
+Requires-Dist: arxiv>=2.2.0; extra == "tools"
+Requires-Dist: pillow<12.0,>=10.1.0; extra == "tools"
+Requires-Dist: pymupdf>=1.23.0; extra == "tools"
+Requires-Dist: openpyxl>=3.1.5; extra == "tools"
+Requires-Dist: wikipedia-api>=0.6.0; extra == "tools"
+Provides-Extra: all
+Requires-Dist: noesium[google,langchain,local-llm,openai,postgres,tools,weaviate]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest<9,>=8.2; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=1.1.0; extra == "dev"
-Requires-Dist: pytest-tornasync>=0.6.0.post2; extra == "dev"
-Requires-Dist: pytest-trio>=0.8.0; extra == "dev"
-Requires-Dist: pytest-twisted>=1.14.3; extra == "dev"
-Requires-Dist: twisted>=25.5.0; extra == "dev"
 Requires-Dist: black>=23.0.0; extra == "dev"
 Requires-Dist: isort>=5.12.0; extra == "dev"
-Requires-Dist: flake8>=7.3.0; extra == "dev"
+Requires-Dist: mypy>=1.10.0; extra == "dev"
 Requires-Dist: autoflake>=2.3.1; extra == "dev"
-Requires-Dist: pillow>=11.3.0; extra == "dev"
+Requires-Dist: flake8>=7.3.0; extra == "dev"
 Dynamic: license-file
 
-# Cogents-core
+# Noesium-core
 
 [![CI](https://github.com/mirasoth/noesium/actions/workflows/ci.yml/badge.svg)](https://github.com/mirasoth/noesium/actions/workflows/ci.yml)
 [![PyPI version](https://img.shields.io/pypi/v/noesium.svg)](https://pypi.org/project/noesium/)
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/mirasoth/noesium)
 
-This is part of [Project Cogents](https://www.xiaming.site/2025/08/30/project-cogents/), an initiative to develop a computation-driven, cognitive agentic system. This repo contains the foundational abstractions (Agent, Memory, Tool, Goal, Orchestration, and more) along with essential modules such as LLM clients, logging, message buses, model routing, and observability. For the underlying philosophy, refer to my talk on MAS ([link](https://github.com/mirasoth/mas-talk-2508/blob/master/mas-talk-xmingc.pdf)).
+Project Noesium is an initiative to develop a computation-driven, cognitive agentic system. This repo contains the foundational abstractions (Agent, Memory, Tool, Goal, Orchestration, and more) along with essential modules such as LLM clients, logging, message buses, model routing, and observability. For the underlying philosophy, refer to my talk on MAS ([link](https://github.com/caesar0301/mas-talk-2508/blob/master/mas-talk-xmingc.pdf)).
 
 ## Installation
 
@@ -117,27 +146,6 @@ Noesium offers a comprehensive set of modules for creating intelligent agent-bas
 - **Opik integration**: Production-ready observability
 - **LangGraph hooks**: Workflow tracing and debugging
 
-## Project Structure
-
-```
-noesium/core/
-├── agent/           # Base agent classes and models
-├── goalith/         # Goal management and planning system
-│   ├── decomposer/  # Goal decomposition strategies
-│   ├── goalgraph/   # Graph data structures
-│   ├── conflict/    # Conflict detection
-│   └── replanner/   # Dynamic replanning
-├── llm/             # LLM provider implementations
-├── memory/          # Memory management system
-│   └── memu/        # MemU memory agent integration
-├── toolify/         # Tool management and execution
-├── vector_store/    # Vector database integrations
-├── msgbus/          # Message bus system
-├── routing/         # LLM routing strategies
-├── tracing/         # Token tracking and observability
-└── utils/           # Utilities and logging
-```
-
 ## Quick Start
 
 ### 1. LLM Client Usage
@@ -386,7 +394,7 @@ Set these environment variables for different providers:
 
 ```bash
 # Default LLM provider
-export COGENTS_LLM_PROVIDER="openai"
+export NOESIUM_LLM_PROVIDER="openai"
 
 # OpenAI
 export OPENAI_API_KEY="sk-..."