entari-plugin-hyw 0.3.5__py3-none-any.whl → 4.0.0rc14__py3-none-any.whl

hyw_core/config.py

@@ -0,0 +1,154 @@
+"""
+hyw_core.config - Configuration Management
+
+Provides standalone configuration for hyw-core with optional passthrough from parent packages.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Any, Optional
+
+
+@dataclass
+class ModelConfig:
+    """Configuration for a specific model."""
+    model_name: Optional[str] = None
+    api_key: Optional[str] = None
+    base_url: Optional[str] = None
+    extra_body: Optional[Dict[str, Any]] = None
+    model_provider: Optional[str] = None
+    input_price: Optional[float] = None
+    output_price: Optional[float] = None
+    image_input: bool = True
+
+
+@dataclass
+class HywCoreConfig:
+    """
+    Core configuration for hyw-core.
+    
+    Can be used standalone or with passthrough from parent packages.
+    
+    Usage:
+        # Standalone from YAML
+        config = HywCoreConfig.from_yaml("config.yaml")
+        
+        # Passthrough from parent
+        config = HywCoreConfig.from_dict({
+            "model_name": parent_config.model_name,
+            "api_key": parent_config.api_key,
+            ...
+        })
+    """
+    
+    # LLM Configuration
+    models: List[Dict[str, Any]] = field(default_factory=list)
+    model_name: str = ""
+    api_key: str = ""
+    base_url: str = ""
+    temperature: float = 0.4
+    
+    # Stage-specific model overrides
+    instruct_model: Optional[str] = None
+    instruct_api_key: Optional[str] = None
+    instruct_base_url: Optional[str] = None
+    instruct_extra_body: Optional[Dict[str, Any]] = None
+    
+    summary_model: Optional[str] = None
+    summary_api_key: Optional[str] = None
+    summary_base_url: Optional[str] = None
+    summary_extra_body: Optional[Dict[str, Any]] = None
+    
+    # Search Configuration
+    search_engine: str = "duckduckgo"
+    search_limit: int = 10
+    blocked_domains: List[str] = field(default_factory=list)
+    
+    # Browser Configuration
+    headless: bool = True
+    fetch_timeout: float = 20.0
+    
+    # Output Configuration
+    language: str = "Simplified Chinese"
+    theme_color: str = "#ef4444"
+    
+    # Pricing (for cost estimation)
+    input_price: float = 0.0
+    output_price: float = 0.0
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "HywCoreConfig":
+        """
+        Create config from dictionary.
+        
+        Used for passthrough from parent packages.
+        Filters out unknown fields to allow flexible passthrough.
+        """
+        import dataclasses
+        field_names = {f.name for f in dataclasses.fields(cls)}
+        filtered_data = {k: v for k, v in data.items() if k in field_names}
+        return cls(**filtered_data)
+    
+    @classmethod
+    def from_yaml(cls, path: str) -> "HywCoreConfig":
+        """
+        Load config from YAML file.
+        
+        Used for standalone usage.
+        """
+        import yaml
+        with open(path, 'r', encoding='utf-8') as f:
+            data = yaml.safe_load(f) or {}
+        return cls.from_dict(data)
+    
+    def get_model_config(self, stage: str) -> ModelConfig:
+        """
+        Get resolved model config for a stage.
+        
+        Args:
+            stage: "instruct", "qa", or "main" (summary)
+        
+        Returns:
+            ModelConfig with resolved settings
+        """
+        # Determine primary and secondary stage config keys
+        if stage == "instruct":
+            primary_prefix = "instruct_"
+            secondary_prefix = None
+        elif stage == "qa":
+            primary_prefix = "qa_"
+            secondary_prefix = "instruct_"
+        else:  # "main" / summary
+            primary_prefix = "summary_"
+            secondary_prefix = None
+        
+        def resolve(field_name: str, is_essential: bool = True):
+            """Resolve a field with fallback: Primary -> Secondary -> Root."""
+            # Try primary
+            if primary_prefix:
+                val = getattr(self, f"{primary_prefix}{field_name}", None)
+                if val:
+                    return val
+            
+            # Try secondary
+            if secondary_prefix:
+                val = getattr(self, f"{secondary_prefix}{field_name}", None)
+                if val:
+                    return val
+            
+            # Fallback to root
+            return getattr(self, field_name, None)
+        
+        return ModelConfig(
+            model_name=resolve("model") or resolve("model_name") or self.model_name,
+            api_key=resolve("api_key") or self.api_key,
+            base_url=resolve("base_url") or self.base_url,
+            extra_body=resolve("extra_body"),
+            model_provider=resolve("model_provider"),
+            input_price=resolve("input_price") or self.input_price,
+            output_price=resolve("output_price") or self.output_price,
+        )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert config to dictionary."""
+        import dataclasses
+        return dataclasses.asdict(self)

hyw_core/core.py

@@ -0,0 +1,462 @@
+"""
+hyw_core.core - Main HywCore Class
+
+Provides the unified LLM query interface and search capabilities.
+"""
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from typing import Dict, List, Any, Optional, Callable, Awaitable
+
+from loguru import logger
+
+from .config import HywCoreConfig, ModelConfig
+from .pipeline import ModularPipeline
+from .agent import AgentPipeline
+from .search import SearchService
+from .stages.base import StageContext
+
+
+@dataclass
+class QueryRequest:
+    """Request for the query interface."""
+    user_input: str
+    images: List[str] = field(default_factory=list)  # base64 encoded images
+    conversation_history: List[Dict] = field(default_factory=list)
+    model_name: Optional[str] = None  # Override model
+    
+    # Optional callbacks
+    send_notification: Optional[Callable[[str], Awaitable[None]]] = None
+
+
+@dataclass
+class QueryResponse:
+    """Response from the query interface."""
+    success: bool
+    content: str  # Markdown response
+    image_path: Optional[str] = None  # Path to rendered image
+    
+    # Statistics
+    usage: Dict[str, int] = field(default_factory=dict)
+    cost: float = 0.0
+    total_time: float = 0.0
+    
+    # References
+    references: List[Dict[str, Any]] = field(default_factory=list)
+    page_references: List[Dict[str, Any]] = field(default_factory=list)
+    image_references: List[Dict[str, Any]] = field(default_factory=list)
+    
+    # Trace information
+    stages_trace: Dict[str, Any] = field(default_factory=dict)
+    
+    # Error handling
+    error: Optional[str] = None
+    should_refuse: bool = False
+    refuse_reason: str = ""
+    
+    # Debug/Save
+    web_results: List[Dict[str, Any]] = field(default_factory=list)
+    stages_used: List[Dict[str, Any]] = field(default_factory=list)
+
+
+
+class HywCore:
+    """
+    HYW Core Service.
+    
+    Provides the unified LLM query interface (/q command) and search capabilities.
+    
+    Usage:
+        from hyw_core import HywCore, HywCoreConfig, QueryRequest
+        
+        config = HywCoreConfig.from_yaml("config.yaml")
+        core = HywCore(config)
+        
+        response = await core.query(QueryRequest(
+            user_input="What is Python?",
+            images=[],
+            conversation_history=[]
+        ))
+    """
+    
+    def __init__(
+        self, 
+        config: HywCoreConfig,
+        send_func: Optional[Callable[[str], Awaitable[None]]] = None
+    ):
+        """
+        Initialize HywCore.
+        
+        Args:
+            config: HywCoreConfig instance
+            send_func: Optional callback for sending notifications
+        """
+        self.config = config
+        self._send_func = send_func
+        
+        # Create search service
+        self._search_service = SearchService(config)
+        
+        # Create pipeline (for non-agent mode)
+        self._pipeline = ModularPipeline(
+            config=config,
+            search_service=self._search_service,
+            send_func=send_func
+        )
+        
+        # Agent pipeline (lazy init)
+        self._agent_pipeline = None
+        
+        # Create renderer (lazy init)
+        self._renderer = None
+        
+        logger.info("HywCore initialized")
+    
+    async def _ensure_renderer(self):
+        """Lazy initialize renderer."""
+        if self._renderer is None:
+            from .browser_control import ContentRenderer
+            self._renderer = ContentRenderer(headless=self.config.headless)
+            await self._renderer.start()
+    
+    async def query(
+        self, 
+        request: QueryRequest,
+        output_path: Optional[str] = None
+    ) -> QueryResponse:
+        """
+        Unified query interface.
+        
+        This is the main entry point for /q commands.
+        
+        Args:
+            request: QueryRequest with user input, images, history
+            output_path: Optional path to save rendered image
+            
+        Returns:
+            QueryResponse with content, rendered image path, and metadata
+        """
+        start_time = time.time()
+        
+        try:
+            # Override model if specified
+            model_name = request.model_name or self.config.model_name
+            
+            # Use notification callback from request if provided
+            send_func = request.send_notification or self._send_func
+            if send_func and self._pipeline._send_func != send_func:
+                self._pipeline._send_func = send_func
+            
+            # Execute pipeline
+            result = await self._pipeline.execute(
+                user_input=request.user_input,
+                conversation_history=request.conversation_history,
+                model_name=model_name,
+                images=request.images if request.images else None
+            )
+            
+            total_time = time.time() - start_time
+            
+            # Check for refusal
+            if result.get("should_refuse"):
+                return QueryResponse(
+                    success=True,
+                    content="",
+                    should_refuse=True,
+                    refuse_reason=result.get("refuse_reason", ""),
+                    total_time=total_time
+                )
+            
+            # Extract response data
+            content = result.get("llm_response", "")
+            structured = result.get("structured_response", {})
+            billing = result.get("billing_info", {})
+            
+            usage = {
+                "input_tokens": billing.get("input_tokens", 0),
+                "output_tokens": billing.get("output_tokens", 0)
+            }
+            
+            # Calculate cost
+            model_cfg = self.config.get_model_config("main")
+            cost = (
+                usage["input_tokens"] * (model_cfg.input_price or 0) / 1_000_000 +
+                usage["output_tokens"] * (model_cfg.output_price or 0) / 1_000_000
+            )
+            
+            # Build response
+            response = QueryResponse(
+                success=True,
+                content=content,
+                usage=usage,
+                cost=cost,
+                total_time=total_time,
+                references=structured.get("references", []),
+                page_references=structured.get("page_references", []),
+                image_references=structured.get("image_references", []),
+                stages_trace=result.get("trace", {}),
+                web_results=result.get("web_results", []),
+                stages_used=result.get("stages_used", [])
+            )
+            
+            # Render image if output path provided
+            if output_path and content:
+                await self._ensure_renderer()
+                
+                render_success = await self._renderer.render(
+                    markdown_content=content,
+                    output_path=output_path,
+                    stats=result.get("stats", {}),
+                    references=result.get("references", []),
+                    page_references=result.get("page_references", []),
+                    image_references=result.get("image_references", []),
+                    stages_used=result.get("stages_used", []),
+                    theme_color=self.config.theme_color
+                )
+                
+                if render_success:
+                    response.image_path = output_path
+            
+            return response
+            
+        except Exception as e:
+            logger.error(f"HywCore query failed: {e}")
+            logger.exception("Query error details:")
+            return QueryResponse(
+                success=False,
+                content="",
+                error=str(e),
+                total_time=time.time() - start_time
+            )
+    
+    async def query_agent(
+        self, 
+        request: QueryRequest,
+        output_path: Optional[str] = None
+    ) -> QueryResponse:
+        """
+        Agent-mode query with tool-calling capability.
+        
+        Uses AgentPipeline which can autonomously call web_tool up to 2 times.
+        Each tool call triggers an IM notification via send_notification callback.
+        
+        Args:
+            request: QueryRequest with user input, images, history
+            output_path: Optional path to save rendered image
+            
+        Returns:
+            QueryResponse with content, rendered image path, and metadata
+        """
+        start_time = time.time()
+        
+        try:
+            # Get or create agent pipeline with current send_func
+            send_func = request.send_notification or self._send_func
+            
+            if self._agent_pipeline is None or self._agent_pipeline.send_func != send_func:
+                self._agent_pipeline = AgentPipeline(
+                    config=self.config,
+                    search_service=self._search_service,
+                    send_func=send_func
+                )
+            
+            # Execute agent pipeline
+            result = await self._agent_pipeline.execute(
+                user_input=request.user_input,
+                conversation_history=request.conversation_history,
+                images=request.images if request.images else None,
+                model_name=request.model_name
+            )
+            
+            total_time = time.time() - start_time
+            
+            # Check for refusal
+            if result.get("refuse_answer"):
+                return QueryResponse(
+                    success=True,
+                    content="",
+                    should_refuse=True,
+                    refuse_reason=result.get("refuse_reason", ""),
+                    total_time=total_time
+                )
+            
+            # Check for error
+            if not result.get("success", True):
+                return QueryResponse(
+                    success=False,
+                    content="",
+                    error=result.get("error", "Unknown error"),
+                    total_time=total_time
+                )
+            
+            # Extract response data
+            content = result.get("llm_response", "")
+            usage = result.get("usage", {})
+            
+            # Convert web_results to references format for frontend
+            # Only include references that are actually cited in the markdown
+            import re
+            web_results = result.get("web_results", [])
+            
+            # Build visible results list (excluding hidden items)
+            visible_results = [r for r in web_results if not r.get("_hidden")]
+            
+            # Parse markdown to find which citations are used (pattern: [number])
+            citation_pattern = re.compile(r'\[(\d+)\]')
+            cited_ids = set()
+            for match in citation_pattern.finditer(content):
+                cited_ids.add(int(match.group(1)))
+            
+            # Only include cited references, in order of first appearance
+            references = []
+            for idx in sorted(cited_ids):
+                # idx is 1-based in markdown
+                if 1 <= idx <= len(visible_results):
+                    r = visible_results[idx - 1]
+                    references.append({
+                        "title": r.get("title", ""),
+                        "url": r.get("url", ""),
+                        "snippet": r.get("content", "")[:300] if r.get("content") else "",
+                        "images": r.get("images", []),
+                        "is_fetched": r.get("_type") == "page",
+                        "raw_screenshot_b64": r.get("screenshot_b64"),
+                    })
+            
+            # Build response
+            response = QueryResponse(
+                success=True,
+                content=content,
+                usage=usage,
+                total_time=total_time,
+                references=references,
+                web_results=web_results,
+                stages_used=result.get("stages_used", [])
+            )
+            
+            # Render image if output path provided
+            if output_path and content:
+                await self._ensure_renderer()
+                
+                render_success = await self._renderer.render(
+                    markdown_content=content,
+                    output_path=output_path,
+                    stats=result.get("stats", {}),
+                    references=references,
+                    page_references=[],
+                    stages_used=result.get("stages_used", []),
+                    theme_color=self.config.theme_color
+                )
+                
+                if render_success:
+                    response.image_path = output_path
+            
+            return response
+            
+        except Exception as e:
+            logger.error(f"HywCore query_agent failed: {e}")
+            logger.exception("Agent query error details:")
+            return QueryResponse(
+                success=False,
+                content="",
+                error=str(e),
+                total_time=time.time() - start_time
+            )
+    
+    async def search(
+        self,
+        queries: List[str],
+        engine: Optional[str] = None,
+        limit: int = 10
+    ) -> List[List[Dict[str, Any]]]:
+        """
+        Independent search interface.
+        
+        For future step-by-step search functionality.
+        
+        Args:
+            queries: List of search queries
+            engine: Optional search engine override
+            limit: Results per query
+            
+        Returns:
+            List of search results for each query
+        """
+        # TODO: Support engine override per-call
+        return await self._search_service.search_batch(queries)
+
+    async def screenshot(self, url: str) -> Optional[str]:
+        """
+        Capture full page screenshot of a URL.
+        Returns: base64 string or None
+        """
+        # Default to full_page=True as requested for /w command
+        return await self._search_service.screenshot_url(url, full_page=True)
+
+    async def screenshot_with_content(self, url: str, max_content_length: int = 8000) -> Dict[str, Any]:
+        """
+        Capture screenshot and extract page content.
+        
+        Returns:
+            Dict with screenshot_b64, content (truncated), title, url
+        """
+        return await self._search_service.screenshot_with_content(url, max_content_length=max_content_length)
+    
+    async def screenshot_batch(self, urls: List[str]) -> List[Optional[str]]:
+        """
+        Capture full page screenshots of multiple URLs concurrently.
+        Returns: list of base64 strings (None for failed ones)
+        """
+        return await self._search_service.screenshot_urls_batch(urls, full_page=True)
+    
+    async def fetch_pages(
+        self,
+        urls: List[str],
+        include_screenshot: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        Fetch multiple pages.
+        
+        Args:
+            urls: List of URLs to fetch
+            include_screenshot: Whether to capture screenshots
+            
+        Returns:
+            List of page data dicts
+        """
+        return await self._search_service.fetch_pages_batch(
+            urls, 
+            include_screenshot=include_screenshot
+        )
+    
+    async def render(
+        self,
+        markdown_content: str,
+        output_path: str,
+        **kwargs
+    ) -> bool:
+        """
+        Render markdown to image.
+        
+        Args:
+            markdown_content: Markdown to render
+            output_path: Path to save image
+            **kwargs: Additional render options
+            
+        Returns:
+            True if successful
+        """
+        await self._ensure_renderer()
+        return await self._renderer.render(
+            markdown_content=markdown_content,
+            output_path=output_path,
+            theme_color=kwargs.pop("theme_color", self.config.theme_color),
+            **kwargs
+        )
+    
+    async def close(self):
+        """Close all resources."""
+        if self._renderer:
+            await self._renderer.close()
+        await self._pipeline.close()
+        logger.info("HywCore closed")

hyw_core/crawling/__init__.py

@@ -0,0 +1,18 @@
+"""
+hyw_core.crawling - Intelligent Web Crawling Module
+
+Provides Crawl4AI-inspired adaptive crawling with:
+- Page completeness guarantees (image loading verification)
+- Content quality scoring
+- Adaptive stop logic
+"""
+
+from .models import CrawlConfig, PageResult, CompletenessResult
+from .completeness import CompletenessChecker
+
+__all__ = [
+    "CrawlConfig",
+    "PageResult", 
+    "CompletenessResult",
+    "CompletenessChecker",
+]