ngpt 3.4.4__py3-none-any.whl → 3.5.0__py3-none-any.whl

ngpt/cli/modes/chat.py

@@ -70,17 +70,21 @@ def chat_mode(client, args, logger=None):
             print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
             # Continue with the original prompt if web search fails
         
-    # Create messages array with preprompt if available
-    messages = None
-    if args.preprompt:
-        # Log the system message if logging is enabled
-        if logger:
-            logger.log("system", args.preprompt)
-            
-        messages = [
-            {"role": "system", "content": args.preprompt},
-            {"role": "user", "content": prompt}
-        ]
+    # Create messages array with system prompt
+    default_system_prompt = "You are a helpful assistant."
+    if args.prettify or args.stream_prettify:
+        default_system_prompt += " You can use markdown formatting in your responses where appropriate."
+    
+    system_prompt = args.preprompt if args.preprompt else default_system_prompt
+    
+    # Log the system message if logging is enabled
+    if logger:
+        logger.log("system", system_prompt)
+        
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": prompt}
+    ]
     
     # Set default streaming behavior based on --no-stream and --prettify arguments
     should_stream = not args.no_stream and not args.prettify

ngpt/utils/web_search.py

@@ -1,5 +1,5 @@
 """
-Web search utilities for nGPT using duckduckgo-search and trafilatura.
+Web search utilities for nGPT using duckduckgo-search and BeautifulSoup4.
 
 This module provides functionality to search the web and extract
 information from search results to enhance AI prompts.
@@ -12,6 +12,9 @@ from urllib.parse import urlparse
 import requests
 import sys
 import datetime
+from bs4 import BeautifulSoup
+from bs4.element import Comment, Declaration, Doctype, ProcessingInstruction
+import json
 
 # Get actual logger from global context instead of using standard logging
 from . import log
@@ -58,9 +61,10 @@ def perform_web_search(query: str, max_results: int = 5) -> List[Dict[str, Any]]
         logger.info("Web search encountered an issue, but will continue with available results")
         return []
 
-def extract_article_content(url: str, max_chars: int = 2000) -> Optional[str]:
+def extract_article_content(url: str, max_chars: int = 5000) -> Optional[str]:
     """
-    Extract and clean content from a webpage URL.
+    Extract and clean content from a webpage URL using a hybrid approach
+    inspired by trafilatura and readability algorithms.
     
     Args:
         url: The URL to extract content from
@@ -81,82 +85,336 @@ def extract_article_content(url: str, max_chars: int = 2000) -> Optional[str]:
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
             'Accept-Language': 'en-US,en;q=0.5',
+            'DNT': '1',  # Do Not Track
             'Connection': 'keep-alive',
             'Upgrade-Insecure-Requests': '1',
+            'Cache-Control': 'max-age=0',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Pragma': 'no-cache',
         }
         
         logger.info(f"Fetching content from {url}")
         
         try:
-            # Try using trafilatura
-            import trafilatura
+            # Fetch the page content
+            response = requests.get(url, headers=headers, timeout=15)
             
-            # Download with correct parameters
-            # trafilatura handles user-agent internally
-            downloaded = trafilatura.fetch_url(url)
-            
-            if downloaded:
-                # Extract main content
-                content = trafilatura.extract(downloaded, include_comments=False, 
-                                             include_tables=False, 
-                                             no_fallback=False)
+            if response.status_code == 200:
+                # Try to detect the encoding if not properly specified
+                if response.encoding == 'ISO-8859-1':
+                    # Try to detect encoding from content
+                    possible_encoding = re.search(r'charset=["\'](.*?)["\']', response.text)
+                    if possible_encoding:
+                        response.encoding = possible_encoding.group(1)
+                    else:
+                        # Default to UTF-8 if we can't detect
+                        response.encoding = 'utf-8'
+                
+                # Parse with BeautifulSoup using lxml parser for better results if available
+                try:
+                    soup = BeautifulSoup(response.text, 'lxml')
+                except ImportError:
+                    soup = BeautifulSoup(response.text, 'html.parser')
+                
+                # Extract main content using multiple strategies
+                extracted_content = None
+                
+                # ---------- PREPROCESSING ----------
+                # Clone the soup before preprocessing
+                processed_soup = BeautifulSoup(str(soup), 'lxml' if 'lxml' in str(type(soup.parser)) else 'html.parser')
+                
+                # Remove all script, style tags and comments
+                for element in processed_soup.find_all(['script', 'style', 'noscript']):
+                    element.decompose()
+                
+                # Remove HTML comments
+                for comment in processed_soup.find_all(text=lambda text: isinstance(text, Comment)):
+                    comment.extract()
+                
+                # Remove hidden elements
+                for hidden in processed_soup.find_all(style=lambda s: s and isinstance(s, str) and ('display:none' in s.lower() or 'visibility:hidden' in s.lower())):
+                    hidden.decompose()
+                for hidden in processed_soup.find_all(hidden=True):
+                    hidden.decompose()
+                for hidden in processed_soup.find_all(class_=lambda c: c and isinstance(c, str) and any(x in c.lower() for x in ['hidden', 'invisible'])):
+                    hidden.decompose()
+                
+                # Handle iframes and frames
+                for frame in processed_soup.find_all(['iframe', 'frame']):
+                    frame.decompose()
+                
+                # ---------- SITE-SPECIFIC HANDLING ----------
+                domain = parsed_url.netloc.lower()
+                
+                # Wikipedia-specific extraction
+                if 'wikipedia.org' in domain:
+                    content_div = processed_soup.select_one('#mw-content-text')
+                    if content_div:
+                        # Remove tables, references, navigation elements
+                        for unwanted in content_div.select('table, .reference, .reflist, .navbox, .vertical-navbox, .thumbcaption, .mw-editsection, .mw-headline, .toc, #toc'):
+                            unwanted.decompose()
+                        extracted_content = content_div.get_text(separator=' ', strip=True)
+                
+                # News site specific handling
+                news_sites = {
+                    'cnn.com': ['article', '.article__content', '.l-container', '.body-text', '#body-text'],
+                    'bbc.com': ['.article__body-content', '.story-body__inner', '[data-component="text-block"]'],
+                    'nytimes.com': ['article', '.meteredContent', '.StoryBodyCompanionColumn', '.article-body'],
+                    'reuters.com': ['article', '.ArticleBody__content___3MtHP', '.article-body'],
+                    'theguardian.com': ['.article-body-commercial-selector', '.content__article-body', '.dcr-1cas96z'],
+                    'washingtonpost.com': ['.article-body', '.teaser-content'],
+                    'apnews.com': ['.Article', '.RichTextStoryBody'],
+                    'indiatimes.com': ['.article-body', '.article_content', '.article-desc', '.Normal'],
+                    'cnbc.com': ['.ArticleBody-articleBody', '.group-article-body', '.article-body'],
+                    'thehindu.com': ['.article-body', '.article-text', '#content-body-14269002']
+                }
+                
+                if not extracted_content:
+                    # Check if we're on a known news site
+                    for site, selectors in news_sites.items():
+                        if site in domain:
+                            for selector in selectors:
+                                content_element = processed_soup.select_one(selector)
+                                if content_element:
+                                    # Clean the news content
+                                    for unwanted in content_element.select('aside, figure, .ad, .ads, .advertisement, .social, .share, .related, .newsletter, .more-on, .read-more, .promotions'):
+                                        unwanted.decompose()
+                                    extracted_content = content_element.get_text(separator=' ', strip=True)
+                                    break
+                            if extracted_content:
+                                break
+                
+                # ---------- JSON-LD EXTRACTION ----------
+                if not extracted_content:
+                    # Look for structured data in JSON-LD format
+                    json_ld = processed_soup.find_all('script', type='application/ld+json')
+                    for script in json_ld:
+                        try:
+                            script_content = script.string
+                            if not script_content:  # Skip empty scripts
+                                continue
+                                
+                            # Clean the JSON string (some sites have invalid JSON)
+                            script_content = re.sub(r'[\n\t\r]', '', script_content)
+                            script_content = script_content.strip()
+                            
+                            data = json.loads(script_content)
+                            # Handle both single objects and arrays of objects
+                            if isinstance(data, list):
+                                data_list = data
+                            else:
+                                data_list = [data]
+                            
+                            for item in data_list:
+                                article_body = None
+                                # Try to find articleBody or various content fields
+                                if isinstance(item, dict):
+                                    # Check for common content fields directly
+                                    for field in ['articleBody', 'description', 'text', 'mainEntityOfPage']:
+                                        if field in item and isinstance(item[field], str) and len(item[field]) > 200:
+                                            article_body = item[field]
+                                            break
+                                    
+                                    # Check in nested objects
+                                    if not article_body and '@graph' in item and isinstance(item['@graph'], list):
+                                        for graph_item in item['@graph']:
+                                            if isinstance(graph_item, dict):
+                                                for field in ['articleBody', 'description', 'text']:
+                                                    if field in graph_item and isinstance(graph_item[field], str) and len(graph_item[field]) > 200:
+                                                        article_body = graph_item[field]
+                                                        break
+                                                if article_body:
+                                                    break
+                                
+                                if article_body:
+                                    extracted_content = article_body
+                                    break
+                            
+                            if extracted_content:
+                                break
+                        except (json.JSONDecodeError, TypeError, AttributeError, ValueError) as e:
+                            logger.debug(f"Error parsing JSON-LD: {str(e)}")
+                            continue
+                
+                # ---------- META DESCRIPTION EXTRACTION ----------
+                meta_description = None
+                meta_tag = processed_soup.find('meta', attrs={'name': 'description'}) or processed_soup.find('meta', attrs={'property': 'og:description'})
+                if meta_tag and meta_tag.get('content'):
+                    meta_description = meta_tag.get('content')
                 
-                if content:
-                    # Clean up content if needed
-                    content = content.strip()
+                # ---------- CONTENT ANALYSIS ----------
+                if not extracted_content:
+                    # Get all content blocks (divs, sections, articles)
+                    content_blocks = []
                     
-                    # Truncate if needed
-                    if len(content) > max_chars:
-                        content = content[:max_chars] + "..."
+                    # Prioritize semantic tags
+                    for tag in ['article', 'main', 'section', 'div']:
+                        blocks = processed_soup.find_all(tag)
                         
-                    return content
-            
-            # If trafilatura failed, try direct requests
-            logger.info(f"Trafilatura extraction failed for {url}, trying fallback method")
-            response = requests.get(url, headers=headers, timeout=10)
-            
-            if response.status_code == 200:
-                # Very basic HTML cleaning
-                html_content = response.text
-                # Remove HTML tags
-                text = re.sub(r'<[^>]+>', ' ', html_content)
-                # Remove excess whitespace
-                text = re.sub(r'\s+', ' ', text).strip()
+                        for block in blocks:
+                            # Skip if too small
+                            text = block.get_text(strip=True)
+                            if len(text) < 200:
+                                continue
+                                
+                            # Calculate content metrics
+                            char_count = len(text)
+                            link_density = calculate_link_density(block)
+                            p_count = len(block.find_all('p'))
+                            p_text_length = sum(len(p.get_text(strip=True)) for p in block.find_all('p'))
+                            p_density = p_text_length / char_count if char_count > 0 else 0
+                            
+                            # Skip blocks with high link density (likely navigation)
+                            if link_density > 0.5:
+                                continue
+                            
+                            # Calculate readability scores
+                            text_density = char_count / (len(str(block)) + 1)  # Text to HTML ratio
+                            
+                            # Score content blocks
+                            score = 0
+                            
+                            # Prefer blocks with many paragraphs
+                            score += min(p_count * 5, 50)  # Max 50 points for paragraphs
+                            
+                            # Prefer blocks with high paragraph text density
+                            score += min(int(p_density * 100), 50)  # Max 50 points for paragraph density
+                            
+                            # Penalize high link density
+                            score -= int(link_density * 100)
+                            
+                            # Boost for high text density
+                            score += min(int(text_density * 30), 30)  # Max 30 points for text density
+                            
+                            # Boost for certain attributes and classes
+                            content_indicators = ['content', 'article', 'story', 'post', 'text', 'body', 'entry']
+                            
+                            # Check class and id attributes
+                            for attr in ['class', 'id']:
+                                attr_val = block.get(attr, '')
+                                if attr_val:
+                                    if isinstance(attr_val, list):
+                                        attr_val = ' '.join(attr_val)
+                                    for indicator in content_indicators:
+                                        if indicator in attr_val.lower():
+                                            score += 30
+                                            break
+                            
+                            # Penalty for boilerplate indicators
+                            boilerplate_indicators = ['sidebar', 'menu', 'nav', 'banner', 'ad', 'footer', 'header', 'comment', 'share', 'related']
+                            for attr in ['class', 'id']:
+                                attr_val = block.get(attr, '')
+                                if attr_val:
+                                    if isinstance(attr_val, list):
+                                        attr_val = ' '.join(attr_val)
+                                    for indicator in boilerplate_indicators:
+                                        if indicator in attr_val.lower():
+                                            score -= 50
+                                            break
+                            
+                            # Add to content blocks if score is positive
+                            if score > 0:
+                                content_blocks.append({
+                                    'element': block,
+                                    'score': score,
+                                    'char_count': char_count,
+                                    'text': text
+                                })
+                    
+                    # Sort content blocks by score
+                    if content_blocks:
+                        content_blocks.sort(key=lambda x: x['score'], reverse=True)
+                        best_block = content_blocks[0]['element']
+                        
+                        # Clean up the best block
+                        for unwanted in best_block.find_all(['aside', 'nav', 'footer', 'header']):
+                            unwanted.decompose()
+                            
+                        extracted_content = best_block.get_text(separator=' ', strip=True)
                 
-                if text:
-                    if len(text) > max_chars:
-                        text = text[:max_chars] + "..."
-                    return text
+                # ---------- PARAGRAPH EXTRACTION FALLBACK ----------
+                if not extracted_content:
+                    # Get all paragraphs with substantial content
+                    paragraphs = []
+                    for p in processed_soup.find_all('p'):
+                        text = p.get_text(strip=True)
+                        if len(text) > 40:  # Only consider substantial paragraphs
+                            # Calculate link density
+                            link_density = calculate_link_density(p)
+                            if link_density < 0.25:  # Skip if too many links
+                                paragraphs.append(text)
+                    
+                    if paragraphs:
+                        extracted_content = ' '.join(paragraphs)
                 
+                # If we have content, clean it up
+                if extracted_content:
+                    # Clean whitespace
+                    extracted_content = re.sub(r'\s+', ' ', extracted_content).strip()
+                    
+                    # Remove URLs
+                    extracted_content = re.sub(r'https?://\S+', '', extracted_content)
+                    
+                    # Remove email addresses
+                    extracted_content = re.sub(r'\S+@\S+', '', extracted_content)
+                    
+                    # Remove social media handles
+                    extracted_content = re.sub(r'@\w+', '', extracted_content)
+                    
+                    # Replace multiple spaces with single space
+                    extracted_content = re.sub(r' +', ' ', extracted_content)
+                    
+                    # Normalize quotes and apostrophes
+                    extracted_content = extracted_content.replace('"', '"').replace('"', '"')
+                    extracted_content = extracted_content.replace("'", "'").replace("'", "'")
+                    
+                    # Remove any remaining HTML entities
+                    extracted_content = re.sub(r'&[a-zA-Z]+;', ' ', extracted_content)
+                    
+                    # Remove short lines that are likely navigation/menu items
+                    lines = extracted_content.split('\n')
+                    extracted_content = ' '.join([line for line in lines if len(line) > 40 or '.' in line])
+                    
+                    # Combine with meta description if available and content is short
+                    if meta_description and len(extracted_content) < 500:
+                        extracted_content = meta_description + " " + extracted_content
+                    
+                    # Truncate if needed
+                    if len(extracted_content) > max_chars:
+                        # Try to break at a sentence boundary
+                        cutoff_point = max_chars
+                        for i in range(max_chars - 1, max_chars - 300, -1):
+                            if i < len(extracted_content) and extracted_content[i] in ['.', '!', '?']:
+                                cutoff_point = i + 1
+                                break
+                        
+                        extracted_content = extracted_content[:cutoff_point]
+                    
+                    return extracted_content
+                else:
+                    # Return meta description if nothing else was found
+                    if meta_description:
+                        return meta_description
+                    
+                    logger.error(f"No content extracted from {url}")
+                    return None
             else:
                 logger.error(f"Request to {url} returned status code {response.status_code}")
-                
-        except ImportError:
-            logger.error("Trafilatura not installed. Install with 'pip install trafilatura'")
-            # Try direct requests only
-            try:
-                response = requests.get(url, headers=headers, timeout=10)
-                if response.status_code == 200:
-                    # Very basic HTML cleaning
-                    html_content = response.text
-                    text = re.sub(r'<[^>]+>', ' ', html_content)
-                    text = re.sub(r'\s+', ' ', text).strip()
-                    
-                    if text:
-                        if len(text) > max_chars:
-                            text = text[:max_chars] + "..."
-                        return text
-            except Exception as req_error:
-                logger.error(f"Direct request fallback failed: {str(req_error)}")
+                return None
                 
         except Exception as e:
-            logger.error(f"Error extracting content with trafilatura: {str(e)}")
-            # Try the requests fallback
+            logger.error(f"Error extracting content with hybrid approach: {str(e)}")
+            # Try a basic fallback
             try:
                 response = requests.get(url, headers=headers, timeout=10)
                 if response.status_code == 200:
-                    html_content = response.text
-                    text = re.sub(r'<[^>]+>', ' ', html_content)
+                    soup = BeautifulSoup(response.text, 'html.parser')
+                    # Just get the text without images, scripts, styles, etc.
+                    for tag in soup(['script', 'style', 'img', 'nav', 'footer', 'header']):
+                        tag.decompose()
+                    text = soup.get_text(separator=' ', strip=True)
                     text = re.sub(r'\s+', ' ', text).strip()
                     
                     if text:
@@ -164,14 +422,40 @@ def extract_article_content(url: str, max_chars: int = 2000) -> Optional[str]:
                             text = text[:max_chars] + "..."
                         return text
             except Exception as req_error:
-                logger.error(f"Direct request fallback failed: {str(req_error)}")
+                logger.error(f"Basic fallback failed: {str(req_error)}")
             
         return None
     except Exception as e:
         logger.error(f"Error extracting content from {url}: {str(e)}")
         return None
 
-def get_web_search_results(query: str, max_results: int = 3, max_chars_per_result: int = 2000) -> Dict[str, Any]:
+def calculate_link_density(element):
+    """
+    Calculate the ratio of link text to all text in an element.
+    Used to identify navigation-heavy areas.
+    
+    Args:
+        element: BeautifulSoup element
+        
+    Returns:
+        Float between 0 and 1 indicating link density
+    """
+    try:
+        if element is None:
+            return 0
+            
+        text_length = len(element.get_text(strip=True))
+        if text_length == 0:
+            return 0
+            
+        links = element.find_all('a')
+        link_text_length = sum(len(a.get_text(strip=True)) for a in links)
+        
+        return link_text_length / text_length
+    except Exception:
+        return 0
+
+def get_web_search_results(query: str, max_results: int = 5, max_chars_per_result: int = 5000) -> Dict[str, Any]:
     """
     Get formatted web search results ready to be included in AI prompts.
     

{ngpt-3.4.4.dist-info → ngpt-3.5.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: ngpt
-Version: 3.4.4
-Summary: Swiss army knife for LLMs: powerful CLI, interactive chatbot, and flexible Python library. Works with OpenAI, Ollama, Groq, Claude, Gemini, and any OpenAI-compatible API.
+Version: 3.5.0
+Summary: Swiss army knife for LLMs: powerful CLI and interactive chatbot in one package. Seamlessly work with OpenAI, Ollama, Groq, Claude, Gemini, or any OpenAI-compatible API to generate code, craft git commits, rewrite text, and execute shell commands.
 Project-URL: Homepage, https://github.com/nazdridoy/ngpt
 Project-URL: Repository, https://github.com/nazdridoy/ngpt
 Project-URL: Bug Tracker, https://github.com/nazdridoy/ngpt/issues
@@ -28,12 +28,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Requires-Python: >=3.8
+Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: duckduckgo-search>=3.0.0
 Requires-Dist: prompt-toolkit>=3.0.0
 Requires-Dist: pyperclip>=1.8.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=10.0.0
-Requires-Dist: trafilatura>=1.6.0
 Description-Content-Type: text/markdown
 
 # nGPT
@@ -46,13 +46,13 @@ Description-Content-Type: text/markdown
 </p>
 
 <p align="center">
-  <a href="https://nazdridoy.github.io/ngpt/installation.html"><img src="https://img.shields.io/badge/Linux-support-blue?logo=linux" alt="Linux"></a>
-  <a href="https://nazdridoy.github.io/ngpt/installation.html"><img src="https://img.shields.io/badge/Windows-support-blue?logo=windows" alt="Windows"></a>
-  <a href="https://nazdridoy.github.io/ngpt/installation.html"><img src="https://img.shields.io/badge/macOS-support-blue?logo=apple" alt="macOS"></a>
-  <a href="https://nazdridoy.github.io/ngpt/installation.html"><img src="https://img.shields.io/badge/Android-Termux-blue?logo=android" alt="Android"></a>
+  <a href="https://nazdridoy.github.io/ngpt/installation/"><img src="https://img.shields.io/badge/Linux-support-blue?logo=linux" alt="Linux"></a>
+  <a href="https://nazdridoy.github.io/ngpt/installation/"><img src="https://img.shields.io/badge/Windows-support-blue?logo=windows" alt="Windows"></a>
+  <a href="https://nazdridoy.github.io/ngpt/installation/"><img src="https://img.shields.io/badge/macOS-support-blue?logo=apple" alt="macOS"></a>
+  <a href="https://nazdridoy.github.io/ngpt/installation/"><img src="https://img.shields.io/badge/Android-Termux-blue?logo=android" alt="Android"></a>
 </p> 
 
-🤖 nGPT: A Swiss army knife for LLMs: powerful CLI, interactive chatbot, and flexible library all in one package. Seamlessly work with OpenAI, Ollama, Groq, Claude, Gemini, or any OpenAI-compatible API to generate code, craft git commits, rewrite text, and execute shell commands. Fast, lightweight, and designed for both casual users and developers.
+🤖 nGPT: A Swiss army knife for LLMs: powerful CLI and interactive chatbot in one package. Seamlessly work with OpenAI, Ollama, Groq, Claude, Gemini, or any OpenAI-compatible API to generate code, craft git commits, rewrite text, and execute shell commands. Fast, lightweight, and designed for both casual users and developers.
 
 
 ![2025-04-23_16-18-01](https://github.com/user-attachments/assets/b8e58926-5165-4352-b48b-9f4a982da86e)
@@ -60,12 +60,12 @@ Description-Content-Type: text/markdown
 
 ## Features
 
-- ✅ **Versatile**: Use as a CLI tool, Python library, or CLI framework for building custom tools
+- ✅ **Versatile**: Powerful and easy-to-use CLI tool for various AI tasks
 - 🪶 **Lightweight**: Minimal dependencies with everything you need included
-- 🔄 **API Flexibility**: Works with OpenAI, Ollama, Groq, Claude, Gemini, and any compatible endpoint
+- 🔄 **API Flexibility**: Works with OpenAI, Ollama, Groq, Claude, Gemini, and any OpenAI-compatible endpoint
 - 💬 **Interactive Chat**: Continuous conversation with memory in modern UI
 - 📊 **Streaming Responses**: Real-time output for better user experience
-- 🔍 **Web Search**: Enhance any model with contextual information from the web
+- 🔍 **Web Search**: Enhance any model with contextual information from the web, using advanced content extraction to identify the most relevant information from web pages
 - 📥 **Stdin Processing**: Process piped content by using `{}` placeholder in prompts
 - 🎨 **Markdown Rendering**: Beautiful formatting of markdown and code with syntax highlighting
 - ⚡ **Real-time Markdown**: Stream responses with live updating syntax highlighting and formatting
@@ -77,48 +77,55 @@ Description-Content-Type: text/markdown
 - 📑 **Git Commit Messages**: AI-powered generation of conventional, detailed commit messages from git diffs
 - 🎭 **System Prompts**: Customize model behavior with custom system prompts
 - 📃 **Conversation Logging**: Save your conversations to text files for later reference
-- 🧰 **CLI Components**: Reusable components for building custom AI-powered command-line tools
 - 🔌 **Modular Architecture**: Well-structured codebase with clean separation of concerns
 - 🔄 **Provider Switching**: Easily switch between different LLM providers with a single parameter
 - 🚀 **Performance Optimized**: Fast response times and minimal resource usage
 
-See the [Feature Overview](https://nazdridoy.github.io/ngpt/overview.html) for more details.
+See the [Feature Overview](https://nazdridoy.github.io/ngpt/overview/) for more details.
 
 
 ## Table of Contents
-- [Quick Start](#quick-start)
 - [Features](#features)
-- [Documentation](#documentation)
 - [Installation](#installation)
+- [Quick Start](#quick-start)
 - [Usage](#usage)
+  - [Command Line Options](#command-line-options)
   - [Documentation](https://nazdridoy.github.io/ngpt/)
-  - [CLI Tool](#as-a-cli-tool)
-  - [Python Library](#as-a-library)
-  - [CLI Framework](#as-a-cli-framework)
+- [Documentation](#documentation)
 - [Configuration](#configuration)
-  - [Command Line Options](#command-line-options)
+  - [API Key Setup](#api-key-setup)
+  - [OpenAI API Key](#openai-api-key)
+  - [Google Gemini API Key](#google-gemini-api-key)
   - [CLI Configuration](#cli-configuration)
   - [Interactive Configuration](#interactive-configuration)
   - [Configuration File](#configuration-file)
   - [Configuration Priority](#configuration-priority)
-  - [API Key Setup](#api-key-setup)
-  - [OpenAI API Key](#openai-api-key)
-  - [Google Gemini API Key](#google-gemini-api-key)
 - [Contributing](#contributing)
 - [License](#license)
 
-## Quick Start
+## Installation
 
 ```bash
-# Install with pip
+# Installation with pip
 pip install ngpt
 
-# Or install with uv (faster)
+# Or install with uv (faster installation)
 uv pip install ngpt
 
-# Or install globally as a CLI tool (recommended)
+# Or install globally as a CLI tool (recommended for command-line usage)
 uv tool install ngpt
 
+# Arch Linux: install from AUR
+paru -S ngpt
+```
+
+Requires Python 3.8 or newer.
+
+For detailed installation instructions, see the [Installation Guide](https://nazdridoy.github.io/ngpt/installation/).
+
+## Quick Start
+
+```bash
 # Chat with default settings
 ngpt "Tell me about quantum computing"
 
@@ -203,60 +210,6 @@ ngpt --provider Groq "Explain quantum computing"
 # Compare outputs from different providers
 ngpt --provider OpenAI "Explain quantum physics" > openai_response.txt
 ngpt --provider Ollama "Explain quantum physics" > ollama_response.txt
-```
-
-For more examples and detailed usage, visit the [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage.html).
-
-## Documentation
-
-Comprehensive documentation, including API reference, usage guides, and examples, is available at:
-
-**[https://nazdridoy.github.io/ngpt/](https://nazdridoy.github.io/ngpt/)**
-
-Key documentation sections:
-- [Installation Guide](https://nazdridoy.github.io/ngpt/installation.html)
-- [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage.html)
-- [Library Usage Guide](https://nazdridoy.github.io/ngpt/usage/library_usage.html)
-- [CLI Framework Guide](https://nazdridoy.github.io/ngpt/usage/cli_framework.html)
-- [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration.html)
-- [Examples & Tutorials](https://nazdridoy.github.io/ngpt/examples/basic.html)
-
-## Installation
-
-```bash
-# Installation with pip
-pip install ngpt
-
-# Or install with uv (faster installation)
-uv pip install ngpt
-
-# Or install globally as a CLI tool (recommended for command-line usage)
-uv tool install ngpt
-
-# Arch Linux: install from AUR
-paru -S ngpt
-```
-
-Requires Python 3.8 or newer.
-
-For detailed installation instructions, see the [Installation Guide](https://nazdridoy.github.io/ngpt/installation.html).
-
-## Usage
-
-### As a CLI Tool
-
-```bash
-# Basic chat (default mode)
-ngpt "Hello, how are you?"
-
-# Interactive chat session with conversation history
-ngpt -i
-
-# Log conversation to a file
-ngpt --interactive --log conversation.log
-
-# Use custom system prompt to guide AI behavior
-ngpt --preprompt "You are a Python programming tutor" "Explain decorators"
 
 # Show all API configurations
 ngpt --show-config --all
@@ -282,165 +235,16 @@ ngpt -s "list all files in current directory"
 # On Windows generates: dir
 # On Linux/macOS generates: ls -la
 
-# Generate clean code (using -c or --code flag)
-# Returns only code without markdown formatting or explanations
+# Generate code (using -c or --code flag)
 ngpt -c "create a python function that calculates fibonacci numbers"
 
 # Use multiline text editor for complex prompts (using -t or --text flag)
-# Opens an interactive editor with syntax highlighting and intuitive controls
 ngpt -t
 ```
 
-For more CLI examples and detailed usage information, see the [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage.html).
-
-### As a Library
-
-```python
-from ngpt import NGPTClient
-from ngpt.utils.config import load_config
-
-# Load the first configuration (index 0) from config file
-config = load_config(config_index=0)
-
-# Initialize the client with config
-client = NGPTClient(**config)
-
-# Or initialize with custom parameters
-client = NGPTClient(
-    api_key="your-key",
-    base_url="http://your-endpoint",
-    provider="openai",
-    model="o3-mini"
-)
-
-# Chat
-response = client.chat("Hello, how are you?")
-
-# Chat with web search (if your API endpoint supports it)
-response = client.chat("What's the latest news about AI?", web_search=True)
-
-# Generate shell command
-command = client.generate_shell_command("list all files")
-
-# Generate code
-code = client.generate_code("create a python function that calculates fibonacci numbers")
-```
-
-For more library examples and advanced usage, see the [Library Usage Guide](https://nazdridoy.github.io/ngpt/usage/library_usage.html).
-
-#### Advanced Library Usage
-
-```python
-# Stream responses
-for chunk in client.chat("Write a poem about Python", stream=True):
-    print(chunk, end="", flush=True)
-
-# Customize system prompt
-response = client.chat(
-    "Explain quantum computing",
-    system_prompt="You are a quantum physics professor. Explain complex concepts simply."
-)
-
-# OS-aware shell commands
-# Automatically generates appropriate commands for the current OS
-command = client.generate_shell_command("find large files")
-import subprocess
-result = subprocess.run(command, shell=True, capture_output=True, text=True)
-print(result.stdout)
-
-# Clean code generation
-# Returns only code without markdown or explanations
-code = client.generate_code("function that converts Celsius to Fahrenheit")
-print(code)
-
-# Compare responses from different providers
-openai_config = load_config(config_index=0)  # OpenAI
-groq_config = load_config(config_index=1)    # Groq
-
-openai_client = NGPTClient(**openai_config)
-groq_client = NGPTClient(**groq_config)
-
-openai_response = openai_client.chat("Explain quantum computing")
-groq_response = groq_client.chat("Explain quantum computing")
-```
-
-For advanced usage patterns and integrations, check out the [Advanced Examples](https://nazdridoy.github.io/ngpt/examples/advanced.html).
-
-### As a CLI Framework
-
-nGPT can also be used as a framework to build your own AI-powered command-line tools. You can leverage nGPT's pre-built CLI components to quickly develop sophisticated CLI applications.
-
-```python
-from ngpt import NGPTClient
-from ngpt.utils.config import load_config
-from ngpt.cli.interactive import interactive_chat_session
-from ngpt.cli.renderers import prettify_markdown
-from ngpt.cli.args import setup_argument_parser
-import sys
-
-# Create a custom CLI tool with colorized help
-parser = setup_argument_parser()
-parser.description = "Specialized Code Assistant"
-parser.add_argument("prompt", nargs="?", help="Code description")
-parser.add_argument("--language", "-l", default="python", help="Programming language")
-parser.add_argument("--interactive", "-i", action="store_true", help="Start interactive mode")
-args = parser.parse_args()
-
-# Initialize client
-client = NGPTClient(**load_config())
-
-# Use interactive session for conversation
-if args.interactive:
-    system_prompt = f"You are an expert {args.language} developer. Provide clear, detailed answers."
-    interactive_chat_session(client=client, preprompt=system_prompt, prettify=True)
-elif args.prompt:
-    # Generate and prettify code
-    code = client.generate_code(args.prompt, language=args.language)
-    print(prettify_markdown(f"```{args.language}\n{code}\n```"))
-else:
-    parser.print_help()
-    sys.exit(1)
-```
+For more examples and detailed usage, visit the [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage/).
 
-This allows you to build specialized AI tools like:
-- Code generators for specific languages or frameworks
-- Domain-specific assistants (SQL, legal, finance, etc.)
-- Documentation generators
-- Translation tools
-- And much more
-
-For detailed information about building CLI tools with nGPT, see the [CLI Framework Guide](https://nazdridoy.github.io/ngpt/usage/cli_framework.html) and explore the [CLI Component Examples](https://nazdridoy.github.io/ngpt/examples/cli_components.html).
-
-## Configuration
-
-### API Key Setup
-
-#### OpenAI API Key
-1. Create an account at [OpenAI](https://platform.openai.com/)
-2. Navigate to API keys: https://platform.openai.com/api-keys
-3. Click "Create new secret key" and copy your API key
-4. Configure nGPT with your key:
-   ```bash
-   ngpt --config
-   # Enter provider: OpenAI
-   # Enter API key: your-openai-api-key
-   # Enter base URL: https://api.openai.com/v1/
-   # Enter model: gpt-3.5-turbo (or other model)
-   ```
-
-#### Google Gemini API Key
-1. Create or use an existing Google account
-2. Go to [Google AI Studio](https://aistudio.google.com/)
-3. Navigate to API keys in the left sidebar (or visit https://aistudio.google.com/app/apikey)
-4. Create an API key and copy it
-5. Configure nGPT with your key:
-   ```bash
-   ngpt --config
-   # Enter provider: Gemini
-   # Enter API key: your-gemini-api-key
-   # Enter base URL: https://generativelanguage.googleapis.com/v1beta/openai
-   # Enter model: gemini-2.0-flash
-   ```
+## Usage
 
 ### Command Line Options
 
@@ -449,8 +253,8 @@ For detailed information about building CLI tools with nGPT, see the [CLI Framew
 usage: ngpt [-h] [-v] [--language LANGUAGE] [--config [CONFIG]] [--config-index CONFIG_INDEX] [--provider PROVIDER]
             [--remove] [--show-config] [--all] [--list-models] [--list-renderers] [--cli-config [COMMAND ...]]
             [--api-key API_KEY] [--base-url BASE_URL] [--model MODEL] [--web-search] [--temperature TEMPERATURE]
-            [--top_p TOP_P] [--max_tokens MAX_TOKENS] [--log [FILE]] [--preprompt PREPROMPT] [--no-stream] [--prettify]
-            [--stream-prettify] [--renderer {auto,rich,glow}] [--rec-chunk] [--diff [FILE]] [--chunk-size CHUNK_SIZE]
+            [--top_p TOP_P] [--max_tokens MAX_TOKENS] [--log [FILE]] [--preprompt PREPROMPT] [--no-stream | --prettify |
+            --stream-prettify] [--renderer {auto,rich,glow}] [--rec-chunk] [--diff [FILE]] [--chunk-size CHUNK_SIZE]
             [--analyses-chunk-size ANALYSES_CHUNK_SIZE] [--max-msg-lines MAX_MSG_LINES]
             [--max-recursion-depth MAX_RECURSION_DEPTH] [-i | -s | -c | -t | -p | -r | -g]
             [prompt]
@@ -469,12 +273,10 @@ options::
 
 Configuration Options::
 
---config [CONFIG]                   Path to a custom config file or, if no value provided, enter interactive
-                                    configuration mode to create a new config
+--config [CONFIG]                   Path to a custom config file or, if no value provided, enter interactive configuration mode to create a new config
 --config-index CONFIG_INDEX         Index of the configuration to use or edit (default: 0)
 --provider PROVIDER                 Provider name to identify the configuration to use
---remove                            Remove the configuration at the specified index (requires --config and
-                                    --config-index or --provider)
+--remove                            Remove the configuration at the specified index (requires --config and --config-index or --provider)
 --show-config                       Show the current configuration(s) and exit
 --all                               Show details for all configurations (requires --show-config)
 --list-models                       List all available models for the current configuration and exit
@@ -486,30 +288,28 @@ Global Options::
 --api-key API_KEY                   API key for the service
 --base-url BASE_URL                 Base URL for the API
 --model MODEL                       Model to use
---web-search                        Enable web search capability (Note: Your API endpoint must support this
-                                    feature)
+--web-search                        Enable web search capability using DuckDuckGo to enhance prompts with relevant information
 --temperature TEMPERATURE           Set temperature (controls randomness, default: 0.7)
 --top_p TOP_P                       Set top_p (controls diversity, default: 1.0)
 --max_tokens MAX_TOKENS             Set max response length in tokens
---log [FILE]                        Set filepath to log conversation to, or create a temporary log file if no path
-                                    provided
+--log [FILE]                        Set filepath to log conversation to, or create a temporary log file if no path provided
 --preprompt PREPROMPT               Set custom system prompt to control AI behavior
---no-stream                         Return the whole response without streaming
---prettify                          Render markdown responses and code with syntax highlighting and formatting
---stream-prettify                   Enable streaming with markdown rendering (automatically uses Rich renderer)
---renderer {auto,rich,glow}         Select which markdown renderer to use with --prettify (auto, rich, or glow)
+--renderer {auto,rich,glow}         Select which markdown renderer to use with --prettify or --stream-prettify (auto, rich, or glow)
+
+Output Display Options (mutually exclusive)::
+
+--no-stream                         Return the whole response without streaming or formatting
+--prettify                          Render complete response with markdown and code formatting (non-streaming)
+--stream-prettify                   Stream response with real-time markdown rendering (default)
 
 Git Commit Message Options::
 
 --rec-chunk                         Process large diffs in chunks with recursive analysis if needed
---diff [FILE]                       Use diff from specified file instead of staged changes. If used without a path,
-                                    uses the path from CLI config.
+--diff [FILE]                       Use diff from specified file instead of staged changes. If used without a path, uses the path from CLI config.
 --chunk-size CHUNK_SIZE             Number of lines per chunk when chunking is enabled (default: 200)
---analyses-chunk-size ANALYSES_CHUNK_SIZE
-                                    Number of lines per chunk when recursively chunking analyses (default: 200)
+--analyses-chunk-size ANALYSES_CHUNK_SIZE Number of lines per chunk when recursively chunking analyses (default: 200)
 --max-msg-lines MAX_MSG_LINES       Maximum number of lines in commit message before condensing (default: 20)
---max-recursion-depth MAX_RECURSION_DEPTH
-                                    Maximum recursion depth for commit message condensing (default: 3)
+--max-recursion-depth MAX_RECURSION_DEPTH Maximum recursion depth for commit message condensing (default: 3)
 
 Modes (mutually exclusive)::
 
@@ -517,15 +317,62 @@ Modes (mutually exclusive)::
 -s, --shell                         Generate and execute shell commands
 -c, --code                          Generate code
 -t, --text                          Enter multi-line text input (submit with Ctrl+D)
--p, --pipe                          Read from stdin and use content with prompt. Use {} in prompt as placeholder
-                                    for stdin content
+-p, --pipe                          Read from stdin and use content with prompt. Use {} in prompt as placeholder for stdin content
 -r, --rewrite                       Rewrite text from stdin to be more natural while preserving tone and meaning
 -g, --gitcommsg                     Generate AI-powered git commit messages from staged changes or diff file
 ```
 
 > **Note**: For better visualization of conventional commit messages on GitHub, you can use the [GitHub Commit Labels](https://greasyfork.org/en/scripts/526153-github-commit-labels) userscript, which adds colorful labels to your commits.
 
-For a complete reference of all available options, see the [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage.html).
+For a complete reference of all available options, detailed CLI examples and usage information, see the [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage/).
+
+
+## Documentation
+
+Comprehensive documentation, including usage guides and examples, is available at:
+
+**[https://nazdridoy.github.io/ngpt/](https://nazdridoy.github.io/ngpt/)**
+
+Key documentation sections:
+- [Installation Guide](https://nazdridoy.github.io/ngpt/installation/)
+- [CLI Usage Guide](https://nazdridoy.github.io/ngpt/usage/cli_usage/)
+- [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration/)
+- [Examples & Tutorials](https://nazdridoy.github.io/ngpt/examples/basic/)
+- [Git Commit Message Guide](https://nazdridoy.github.io/ngpt/usage/gitcommsg/)
+
+
+## Configuration
+
+### API Key Setup
+
+#### OpenAI API Key
+1. Create an account at [OpenAI](https://platform.openai.com/)
+2. Navigate to API keys: https://platform.openai.com/api-keys
+3. Click "Create new secret key" and copy your API key
+4. Configure nGPT with your key:
+   ```bash
+   ngpt --config
+   # Enter provider: OpenAI
+   # Enter API key: your-openai-api-key
+   # Enter base URL: https://api.openai.com/v1/
+   # Enter model: gpt-3.5-turbo (or other model)
+   ```
+
+#### Google Gemini API Key
+1. Create or use an existing Google account
+2. Go to [Google AI Studio](https://aistudio.google.com/)
+3. Navigate to API keys in the left sidebar (or visit https://aistudio.google.com/app/apikey)
+4. Create an API key and copy it
+5. Configure nGPT with your key:
+   ```bash
+   ngpt --config
+   # Enter provider: Gemini
+   # Enter API key: your-gemini-api-key
+   # Enter base URL: https://generativelanguage.googleapis.com/v1beta/openai
+   # Enter model: gemini-2.0-flash
+   ```
+
+For more detailed information, refer to the [API Key Setup documentation](https://nazdridoy.github.io/ngpt/configuration/#api-key-setup).
 
 ### CLI Configuration
 
@@ -598,7 +445,7 @@ CLI Configuration Help:
 
 ```
 
-For more details, see the [CLI Configuration Guide](https://nazdridoy.github.io/ngpt/usage/cli_config.html).
+For more details, see the [CLI Configuration Guide](https://nazdridoy.github.io/ngpt/usage/cli_config/).
 
 ### Interactive Configuration
 
@@ -630,7 +477,7 @@ In interactive mode:
 - For security, your API key is not displayed when editing configurations
 - When removing a configuration, you'll be asked to confirm before deletion
 
-For more details on configuring nGPT, see the [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration.html).
+For more details on configuring nGPT, see the [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration/).
 
 ### Configuration File
 
@@ -666,7 +513,7 @@ The configuration file uses a JSON list format, allowing you to store multiple c
 ]
 ```
 
-For details on the configuration file format and structure, see the [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration.html).
+For details on the configuration file format and structure, see the [Configuration Guide](https://nazdridoy.github.io/ngpt/configuration/).
 
 ### Configuration Priority
 

{ngpt-3.4.4.dist-info → ngpt-3.5.0.dist-info}/RECORD

@@ -9,7 +9,7 @@ ngpt/cli/main.py,sha256=oKX7ryTIrsvQRJHVnH2a763pGyNZthq81wkrRILwHLw,28932
 ngpt/cli/renderers.py,sha256=m71BeUXKynpKKGXFzwRSW1XngvyKiZ_xEsdujUbU0MA,16597
 ngpt/cli/ui.py,sha256=HoHDFpLiwMBP5wtMb8YYo244FMiqiPFRoBNcNGp6N0A,7310
 ngpt/cli/modes/__init__.py,sha256=KP7VR6Xw9k1p5Jcu0F38RDxSFvFIzH3j1ThDLNwznUI,363
-ngpt/cli/modes/chat.py,sha256=UlnZWqvxL_CSyVIDZKGVXNlG-KufDfo05E8-8xTcM_Y,6713
+ngpt/cli/modes/chat.py,sha256=pv8Ec1l3XsHnW1o3iukxmKg3pK4UJeODl8mC73MH0qg,6927
 ngpt/cli/modes/code.py,sha256=ounayP8exu8Bb8QFL4TcyF2d3QI_QlsG5tf6v1FktxE,11470
 ngpt/cli/modes/gitcommsg.py,sha256=rsfMoeOupmNp-5p5fsMSPAf18BbzXWq-4PF2HjEz6SY,46991
 ngpt/cli/modes/interactive.py,sha256=COJABbSgfyVC98lvFD1K2yqKPqyH20vbVrYYh8oiPUk,16395
@@ -20,9 +20,9 @@ ngpt/utils/__init__.py,sha256=qu_66I1Vtav2f1LDiPn5J3DUsbK7o1CSScMcTkYqxoM,1179
 ngpt/utils/cli_config.py,sha256=Ug8cECBTIuzOwkBWidLTfs-OAdOsCMJ2bNa70pOADfw,11195
 ngpt/utils/config.py,sha256=wsArA4osnh8fKqOvtsPqqBxAz3DpdjtaWUFaRtnUdyc,10452
 ngpt/utils/log.py,sha256=f1jg2iFo35PAmsarH8FVL_62plq4VXH0Mu2QiP6RJGw,15934
-ngpt/utils/web_search.py,sha256=yvCUDNhwcIcKZ_hWESFQQ-vB-LKsDDCDT17YFzFcGR4,12598
-ngpt-3.4.4.dist-info/METADATA,sha256=JD6g13Horisv-gmE-Dnr0vbeAUY7zq8J9JwJ1DZYsMg,29100
-ngpt-3.4.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ngpt-3.4.4.dist-info/entry_points.txt,sha256=SqAAvLhMrsEpkIr4YFRdUeyuXQ9o0IBCeYgE6AVojoI,44
-ngpt-3.4.4.dist-info/licenses/LICENSE,sha256=mQkpWoADxbHqE0HRefYLJdm7OpdrXBr3vNv5bZ8w72M,1065
-ngpt-3.4.4.dist-info/RECORD,,
+ngpt/utils/web_search.py,sha256=_e78fk-6WDVqHDIaAMP9CDoX4FyOWM9XhdSZcr5KasI,29163
+ngpt-3.5.0.dist-info/METADATA,sha256=BEcmK1bv4EZ_Q6W7MDnDPo304e-_L0zk1bDgH4mmp8Q,23926
+ngpt-3.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ngpt-3.5.0.dist-info/entry_points.txt,sha256=SqAAvLhMrsEpkIr4YFRdUeyuXQ9o0IBCeYgE6AVojoI,44
+ngpt-3.5.0.dist-info/licenses/LICENSE,sha256=mQkpWoADxbHqE0HRefYLJdm7OpdrXBr3vNv5bZ8w72M,1065
+ngpt-3.5.0.dist-info/RECORD,,