pomera-ai-commander 1.2.8 → 1.2.10

package/tools/case_tool.py

@@ -188,7 +188,8 @@ class CaseTool:
         """Get default settings for the Case Tool.
         
         Uses the centralized Settings Defaults Registry if available,
-        otherwise falls back to hardcoded defaults.
+        otherwise falls back to minimal defaults. Full exclusions list
+        is maintained only in the registry.
         """
         try:
             from core.settings_defaults_registry import get_registry
@@ -199,11 +200,10 @@ class CaseTool:
         except Exception:
             pass
         
-        # Fallback to hardcoded defaults with updated exclusions
-        # Exclusions: a, an, the, and, but, or, for, nor, on, at, to, from, by, with, in, of
+        # Minimal fallback - registry has the full exclusions list
         return {
             "mode": "Sentence",
-            "exclusions": "a\nan\nthe\nand\nbut\nor\nfor\nnor\non\nat\nto\nfrom\nby\nwith\nin\nof"
+            "exclusions": ""
         }
 
 

package/tools/curl_settings.py

@@ -91,7 +91,18 @@ class CurlSettingsManager:
             # Version and metadata
             "settings_version": "1.0",
             "last_updated": None,
-            "created_date": None
+            "created_date": None,
+            
+            # UI State Persistence (NEW - persist between restarts)
+            "last_url": "",
+            "last_method": "GET",
+            "last_headers": "",
+            "last_body": "",
+            "last_body_type": "None",
+            "last_auth_type": "None",
+            "last_auth_data": {},  # Encrypted auth tokens stored here
+            "last_complex_options": "",
+            "persist_ui_state": True  # User preference to persist UI state
         }
         
         # Current settings (loaded from file or defaults)

package/tools/curl_tool.py

@@ -284,6 +284,9 @@ class CurlToolWidget:
         # Create the UI
         self.create_widgets()
         
+        # Restore saved UI state (URL, method, headers, body, auth)
+        self._restore_ui_state()
+        
         # Save settings when the window is closed
         if hasattr(self.parent, 'protocol'):
             self.parent.protocol("WM_DELETE_WINDOW", self._on_closing)
@@ -4232,17 +4235,8 @@ curl -X POST https://api.example.com/users \\
             if self.logger:
                 self.logger.error(f"Error saving settings: {e}")
     
-    def _on_closing(self):
-        """Handle application closing - save settings before exit."""
-        try:
-            self._save_current_settings()
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error saving settings on close: {e}")
-        
-        # Continue with normal closing
-        if hasattr(self.parent, 'destroy'):
-            self.parent.destroy()
+    # NOTE: _on_closing method has been moved to end of class (line ~5490)
+    # to consolidate with UI state persistence logic
     
     def _export_curl(self):
         """Export current request as cURL command."""
@@ -5483,6 +5477,177 @@ Timestamp: {timestamp}{additional_details}"""
         if item:
             self.history_tree.selection_set(item)
             self.history_context_menu.post(event.x_root, event.y_root)
+    
+    def _on_closing(self):
+        """Handle widget/window closing - save all settings and UI state."""
+        # Save original settings (from the original method at line 4238)
+        try:
+            self._save_current_settings()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error saving settings on close: {e}")
+        
+        # Also save UI state for full persistence (new functionality)
+        try:
+            self._save_ui_state()
+            self.logger.info("cURL Tool UI state saved on close")
+        except Exception as e:
+            self.logger.error(f"Error saving UI state on close: {e}")
+        
+        # If parent has destroy, call it
+        if hasattr(self.parent, 'destroy'):
+            self.parent.destroy()
+    
+    def _save_ui_state(self):
+        """Save current UI state to settings for persistence."""
+        if not self.settings_manager:
+            return
+        
+        # Check if UI state persistence is enabled
+        if not self.settings.get("persist_ui_state", True):
+            return
+        
+        try:
+            # Get current URL from text widget
+            url = ""
+            if self.url_text and self.url_text.winfo_exists():
+                url = self.url_text.get("1.0", tk.END).strip()
+            
+            # Get current method
+            method = self.method_var.get() if self.method_var else "GET"
+            
+            # Get headers from text widget
+            headers = ""
+            if hasattr(self, 'headers_text') and self.headers_text and self.headers_text.winfo_exists():
+                headers = self.headers_text.get("1.0", tk.END).strip()
+            
+            # Get body from text widget
+            body = ""
+            if hasattr(self, 'body_text') and self.body_text and self.body_text.winfo_exists():
+                body = self.body_text.get("1.0", tk.END).strip()
+            
+            # Get body type
+            body_type = self.body_type_var.get() if self.body_type_var else "None"
+            
+            # Get auth type
+            auth_type = self.auth_type_var.get() if self.auth_type_var else "None"
+            
+            # Get auth data (encrypted)
+            auth_data = {}
+            if auth_type == "Bearer":
+                token = self.bearer_token_var.get() if hasattr(self, 'bearer_token_var') else ""
+                if token:
+                    auth_data["bearer_token"] = encrypt_auth_value(token)
+            elif auth_type == "Basic":
+                username = self.basic_username_var.get() if hasattr(self, 'basic_username_var') else ""
+                password = self.basic_password_var.get() if hasattr(self, 'basic_password_var') else ""
+                if username or password:
+                    auth_data["basic_username"] = username
+                    auth_data["basic_password"] = encrypt_auth_value(password)
+            elif auth_type == "API Key":
+                key_name = self.api_key_name_var.get() if hasattr(self, 'api_key_name_var') else ""
+                key_value = self.api_key_value_var.get() if hasattr(self, 'api_key_value_var') else ""
+                key_location = self.api_key_location_var.get() if hasattr(self, 'api_key_location_var') else "header"
+                if key_value:
+                    auth_data["api_key_name"] = key_name
+                    auth_data["api_key_value"] = encrypt_auth_value(key_value)
+                    auth_data["api_key_location"] = key_location
+            
+            # Get complex options
+            complex_options = self.complex_options_var.get() if hasattr(self, 'complex_options_var') else ""
+            
+            # Update settings
+            self.settings_manager.set_setting("last_url", url)
+            self.settings_manager.set_setting("last_method", method)
+            self.settings_manager.set_setting("last_headers", headers)
+            self.settings_manager.set_setting("last_body", body)
+            self.settings_manager.set_setting("last_body_type", body_type)
+            self.settings_manager.set_setting("last_auth_type", auth_type)
+            self.settings_manager.set_setting("last_auth_data", auth_data)
+            self.settings_manager.set_setting("last_complex_options", complex_options)
+            
+            # Save to persistent storage
+            self.settings_manager.save_settings()
+            self.logger.debug("UI state saved successfully")
+            
+        except Exception as e:
+            self.logger.error(f"Error saving UI state: {e}")
+    
+    def _restore_ui_state(self):
+        """Restore UI state from saved settings."""
+        if not self.settings_manager:
+            return
+        
+        # Check if UI state persistence is enabled
+        if not self.settings.get("persist_ui_state", True):
+            return
+        
+        try:
+            # Restore URL
+            last_url = self.settings.get("last_url", "")
+            if last_url and self.url_text and self.url_text.winfo_exists():
+                self.url_text.delete("1.0", tk.END)
+                self.url_text.insert("1.0", last_url)
+            
+            # Restore method
+            last_method = self.settings.get("last_method", "GET")
+            if self.method_var:
+                self.method_var.set(last_method)
+            
+            # Restore headers
+            last_headers = self.settings.get("last_headers", "")
+            if last_headers and hasattr(self, 'headers_text') and self.headers_text:
+                if self.headers_text.winfo_exists():
+                    self.headers_text.delete("1.0", tk.END)
+                    self.headers_text.insert("1.0", last_headers)
+            
+            # Restore body
+            last_body = self.settings.get("last_body", "")
+            if last_body and hasattr(self, 'body_text') and self.body_text:
+                if self.body_text.winfo_exists():
+                    self.body_text.delete("1.0", tk.END)
+                    self.body_text.insert("1.0", last_body)
+            
+            # Restore body type
+            last_body_type = self.settings.get("last_body_type", "None")
+            if self.body_type_var:
+                self.body_type_var.set(last_body_type)
+            
+            # Restore auth type
+            last_auth_type = self.settings.get("last_auth_type", "None")
+            if self.auth_type_var:
+                self.auth_type_var.set(last_auth_type)
+            
+            # Restore auth data (decrypted)
+            last_auth_data = self.settings.get("last_auth_data", {})
+            if last_auth_data:
+                if last_auth_type == "Bearer" and hasattr(self, 'bearer_token_var'):
+                    token = last_auth_data.get("bearer_token", "")
+                    self.bearer_token_var.set(decrypt_auth_value(token))
+                elif last_auth_type == "Basic":
+                    if hasattr(self, 'basic_username_var'):
+                        self.basic_username_var.set(last_auth_data.get("basic_username", ""))
+                    if hasattr(self, 'basic_password_var'):
+                        password = last_auth_data.get("basic_password", "")
+                        self.basic_password_var.set(decrypt_auth_value(password))
+                elif last_auth_type == "API Key":
+                    if hasattr(self, 'api_key_name_var'):
+                        self.api_key_name_var.set(last_auth_data.get("api_key_name", ""))
+                    if hasattr(self, 'api_key_value_var'):
+                        key_value = last_auth_data.get("api_key_value", "")
+                        self.api_key_value_var.set(decrypt_auth_value(key_value))
+                    if hasattr(self, 'api_key_location_var'):
+                        self.api_key_location_var.set(last_auth_data.get("api_key_location", "header"))
+            
+            # Restore complex options
+            last_complex_options = self.settings.get("last_complex_options", "")
+            if last_complex_options and hasattr(self, 'complex_options_var'):
+                self.complex_options_var.set(last_complex_options)
+            
+            self.logger.debug("UI state restored successfully")
+            
+        except Exception as e:
+            self.logger.error(f"Error restoring UI state: {e}")
 
 
 # For standalone testing

package/tools/tool_loader.py

@@ -366,6 +366,24 @@ TOOL_SPECS: Dict[str, ToolSpec] = {
         description="Model Context Protocol server management",
         available_flag="MCP_WIDGET_MODULE_AVAILABLE"
     ),
+    
+    # Web Tools - handled inline in pomera.py (tabbed interface like AI Tools)
+    "Web Search": ToolSpec(
+        name="Web Search",
+        module_path="tools.web_search",  # Core module, UI created inline
+        class_name="search",  # Function, not class
+        category=ToolCategory.UTILITY,
+        description="Search the web using DuckDuckGo, Tavily, Google, Brave, SerpApi, Serper",
+        available_flag=""  # Always available
+    ),
+    "URL Reader": ToolSpec(
+        name="URL Reader",
+        module_path="tools.url_content_reader",
+        class_name="URLContentReader",
+        category=ToolCategory.UTILITY,
+        description="Fetch URL content and convert to HTML, JSON, or Markdown",
+        available_flag=""  # Always available
+    ),
 }
 
 # These sub-tools appear as tabs within their parent tool

package/tools/url_content_reader.py

@@ -0,0 +1,402 @@
+"""
+URL Content Reader Module for Pomera AI Commander
+
+Fetches web content and converts HTML to Markdown.
+Features:
+- HTTP/HTTPS URL fetching
+- Main content extraction (skips nav, header, footer)
+- HTML to Markdown conversion
+- Proper error handling and timeout support
+
+Author: Pomera AI Commander
+"""
+
+import re
+import urllib.request
+import urllib.error
+from typing import Optional, List, Tuple
+from html.parser import HTMLParser
+from html import unescape
+import logging
+
+
+class HTMLToMarkdownConverter(HTMLParser):
+    """Convert HTML to Markdown format."""
+    
+    # Tags to completely skip (including content)
+    SKIP_TAGS = {'script', 'style', 'noscript', 'iframe', 'svg', 'canvas', 
+                 'nav', 'header', 'footer', 'aside', 'form', 'button'}
+    
+    # Block-level tags that need newlines
+    BLOCK_TAGS = {'p', 'div', 'section', 'article', 'main', 'h1', 'h2', 'h3', 
+                  'h4', 'h5', 'h6', 'blockquote', 'pre', 'li', 'tr', 'td', 'th'}
+    
+    def __init__(self):
+        super().__init__()
+        self.output: List[str] = []
+        self.tag_stack: List[str] = []
+        self.skip_depth = 0
+        self.list_depth = 0
+        self.in_pre = False
+        self.in_code = False
+        self.current_link_url = ""
+        self.current_link_text = ""
+        self.in_link = False
+    
+    def handle_starttag(self, tag, attrs):
+        tag = tag.lower()
+        
+        # Track skip depth for nested skip tags
+        if tag in self.SKIP_TAGS:
+            self.skip_depth += 1
+            return
+        
+        if self.skip_depth > 0:
+            return
+        
+        self.tag_stack.append(tag)
+        attrs_dict = dict(attrs)
+        
+        # Headings
+        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
+            level = int(tag[1])
+            self.output.append('\n\n' + '#' * level + ' ')
+        
+        # Paragraphs and divs
+        elif tag in ('p', 'div', 'section', 'article', 'main'):
+            self.output.append('\n\n')
+        
+        # Line break
+        elif tag == 'br':
+            self.output.append('\n')
+        
+        # Horizontal rule
+        elif tag == 'hr':
+            self.output.append('\n\n---\n\n')
+        
+        # Bold
+        elif tag in ('strong', 'b'):
+            self.output.append('**')
+        
+        # Italic
+        elif tag in ('em', 'i'):
+            self.output.append('*')
+        
+        # Code
+        elif tag == 'code':
+            if not self.in_pre:
+                self.output.append('`')
+            self.in_code = True
+        
+        # Preformatted
+        elif tag == 'pre':
+            self.output.append('\n\n```\n')
+            self.in_pre = True
+        
+        # Links
+        elif tag == 'a':
+            href = attrs_dict.get('href', '')
+            if href and not href.startswith('#') and not href.startswith('javascript:'):
+                self.current_link_url = href
+                self.current_link_text = ""
+                self.in_link = True
+                self.output.append('[')
+        
+        # Images
+        elif tag == 'img':
+            src = attrs_dict.get('src', '')
+            alt = attrs_dict.get('alt', 'image')
+            if src:
+                self.output.append(f'\n![{alt}]({src})\n')
+        
+        # Unordered list
+        elif tag == 'ul':
+            self.list_depth += 1
+            self.output.append('\n')
+        
+        # Ordered list
+        elif tag == 'ol':
+            self.list_depth += 1
+            self.output.append('\n')
+        
+        # List item
+        elif tag == 'li':
+            indent = '  ' * (self.list_depth - 1)
+            parent = self.tag_stack[-2] if len(self.tag_stack) > 1 else 'ul'
+            if parent == 'ol':
+                self.output.append(f'\n{indent}1. ')
+            else:
+                self.output.append(f'\n{indent}- ')
+        
+        # Blockquote
+        elif tag == 'blockquote':
+            self.output.append('\n\n> ')
+        
+        # Table elements
+        elif tag == 'table':
+            self.output.append('\n\n')
+        elif tag == 'tr':
+            self.output.append('\n')
+        elif tag in ('td', 'th'):
+            self.output.append(' | ')
+    
+    def handle_endtag(self, tag):
+        tag = tag.lower()
+        
+        if tag in self.SKIP_TAGS:
+            self.skip_depth = max(0, self.skip_depth - 1)
+            return
+        
+        if self.skip_depth > 0:
+            return
+        
+        if self.tag_stack and self.tag_stack[-1] == tag:
+            self.tag_stack.pop()
+        
+        # Bold
+        if tag in ('strong', 'b'):
+            self.output.append('**')
+        
+        # Italic
+        elif tag in ('em', 'i'):
+            self.output.append('*')
+        
+        # Code
+        elif tag == 'code':
+            if not self.in_pre:
+                self.output.append('`')
+            self.in_code = False
+        
+        # Preformatted
+        elif tag == 'pre':
+            self.output.append('\n```\n\n')
+            self.in_pre = False
+        
+        # Links
+        elif tag == 'a' and self.in_link:
+            self.output.append(f']({self.current_link_url})')
+            self.in_link = False
+            self.current_link_url = ""
+        
+        # Lists
+        elif tag in ('ul', 'ol'):
+            self.list_depth = max(0, self.list_depth - 1)
+            if self.list_depth == 0:
+                self.output.append('\n')
+        
+        # Block elements
+        elif tag in self.BLOCK_TAGS:
+            self.output.append('\n')
+    
+    def handle_data(self, data):
+        if self.skip_depth > 0:
+            return
+        
+        # Preserve whitespace in pre/code blocks
+        if self.in_pre:
+            self.output.append(data)
+        else:
+            # Normalize whitespace
+            text = re.sub(r'\s+', ' ', data)
+            if text.strip():
+                self.output.append(text)
+    
+    def handle_entityref(self, name):
+        if self.skip_depth > 0:
+            return
+        char = unescape(f'&{name};')
+        self.output.append(char)
+    
+    def handle_charref(self, name):
+        if self.skip_depth > 0:
+            return
+        char = unescape(f'&#{name};')
+        self.output.append(char)
+    
+    def get_markdown(self) -> str:
+        """Get the converted markdown text."""
+        text = ''.join(self.output)
+        
+        # Clean up excessive newlines
+        text = re.sub(r'\n{3,}', '\n\n', text)
+        
+        # Clean up spaces around markdown elements
+        text = re.sub(r'\*\* +', '**', text)
+        text = re.sub(r' +\*\*', '**', text)
+        text = re.sub(r'\* +', '*', text)
+        text = re.sub(r' +\*', '*', text)
+        
+        # Clean up empty list items
+        text = re.sub(r'\n- \n', '\n', text)
+        text = re.sub(r'\n1\. \n', '\n', text)
+        
+        return text.strip()
+
+
+class URLContentReader:
+    """Fetch URLs and convert content to Markdown."""
+    
+    USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    
+    def __init__(self, logger=None):
+        self.logger = logger or logging.getLogger(__name__)
+    
+    def fetch_url(self, url: str, timeout: int = 30) -> str:
+        """
+        Fetch content from a URL.
+        
+        Args:
+            url: URL to fetch
+            timeout: Request timeout in seconds
+            
+        Returns:
+            HTML content as string
+        """
+        # Validate URL
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+        
+        try:
+            req = urllib.request.Request(url)
+            req.add_header('User-Agent', self.USER_AGENT)
+            req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
+            req.add_header('Accept-Language', 'en-US,en;q=0.5')
+            
+            with urllib.request.urlopen(req, timeout=timeout) as response:
+                # Detect encoding
+                charset = response.headers.get_content_charset()
+                if not charset:
+                    charset = 'utf-8'
+                
+                content = response.read()
+                try:
+                    return content.decode(charset, errors='replace')
+                except (UnicodeDecodeError, LookupError):
+                    return content.decode('utf-8', errors='replace')
+        
+        except urllib.error.HTTPError as e:
+            raise Exception(f"HTTP Error {e.code}: {e.reason}")
+        except urllib.error.URLError as e:
+            raise Exception(f"URL Error: {e.reason}")
+        except Exception as e:
+            raise Exception(f"Fetch error: {str(e)}")
+    
+    def html_to_markdown(self, html: str, extract_main_content: bool = True) -> str:
+        """
+        Convert HTML to Markdown.
+        
+        Args:
+            html: HTML content
+            extract_main_content: If True, try to extract main content area
+            
+        Returns:
+            Markdown formatted text
+        """
+        if extract_main_content:
+            html = self._extract_main_content(html)
+        
+        converter = HTMLToMarkdownConverter()
+        try:
+            converter.feed(html)
+            return converter.get_markdown()
+        except Exception as e:
+            self.logger.error(f"HTML parsing error: {e}")
+            # Fallback: simple text extraction
+            return self._simple_text_extraction(html)
+    
+    def _extract_main_content(self, html: str) -> str:
+        """Try to extract main content area from HTML."""
+        # Try to find main content containers
+        patterns = [
+            r'<main[^>]*>(.*?)</main>',
+            r'<article[^>]*>(.*?)</article>',
+            r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
+            r'<div[^>]*id="[^"]*content[^"]*"[^>]*>(.*?)</div>',
+            r'<div[^>]*class="[^"]*main[^"]*"[^>]*>(.*?)</div>',
+            r'<body[^>]*>(.*?)</body>',
+        ]
+        
+        for pattern in patterns:
+            match = re.search(pattern, html, re.IGNORECASE | re.DOTALL)
+            if match:
+                return match.group(1)
+        
+        return html
+    
+    def _simple_text_extraction(self, html: str) -> str:
+        """Simple fallback text extraction."""
+        # Remove script and style
+        text = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.IGNORECASE | re.DOTALL)
+        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.IGNORECASE | re.DOTALL)
+        
+        # Remove HTML tags
+        text = re.sub(r'<[^>]+>', ' ', text)
+        
+        # Decode entities
+        text = unescape(text)
+        
+        # Normalize whitespace
+        text = re.sub(r'\s+', ' ', text)
+        
+        return text.strip()
+    
+    def fetch_and_convert(self, url: str, timeout: int = 30, 
+                         extract_main_content: bool = True) -> str:
+        """
+        Fetch URL and convert to Markdown in one step.
+        
+        Args:
+            url: URL to fetch
+            timeout: Request timeout in seconds
+            extract_main_content: If True, extract main content only
+            
+        Returns:
+            Markdown formatted content
+        """
+        html = self.fetch_url(url, timeout)
+        return self.html_to_markdown(html, extract_main_content)
+
+
+# CLI support
+def main():
+    """CLI entry point for URL content reading."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Fetch URLs and convert to Markdown")
+    parser.add_argument("url", nargs="?", help="URL to fetch")
+    parser.add_argument("--timeout", "-t", type=int, default=30, help="Timeout in seconds")
+    parser.add_argument("--no-extract", action="store_true", 
+                       help="Don't try to extract main content")
+    parser.add_argument("--output", "-o", type=str, help="Output file path")
+    
+    args = parser.parse_args()
+    
+    if not args.url:
+        parser.print_help()
+        return
+    
+    reader = URLContentReader()
+    
+    try:
+        markdown = reader.fetch_and_convert(
+            args.url, 
+            timeout=args.timeout,
+            extract_main_content=not args.no_extract
+        )
+        
+        if args.output:
+            with open(args.output, 'w', encoding='utf-8') as f:
+                f.write(markdown)
+            print(f"Saved to: {args.output}")
+        else:
+            print(markdown)
+            
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main() or 0)