ngpt 3.1.1__py3-none-any.whl → 3.3.0__py3-none-any.whl

ngpt/cli/args.py

@@ -68,7 +68,7 @@ def setup_argument_parser():
     global_group.add_argument('--model', 
                               help='Model to use')
     global_group.add_argument('--web-search', action='store_true', 
-                      help='Enable web search capability (Note: Your API endpoint must support this feature)')
+                      help='Enable web search capability using DuckDuckGo to enhance prompts with relevant information')
     global_group.add_argument('--temperature', type=float, default=0.7,
                       help='Set temperature (controls randomness, default: 0.7)')
     global_group.add_argument('--top_p', type=float, default=1.0,

ngpt/cli/interactive.py

@@ -6,6 +6,7 @@ import sys
 import time
 from .formatters import COLORS
 from .renderers import prettify_markdown, prettify_streaming_markdown
+from ..utils import enhance_prompt_with_web_search
 
 # Optional imports for enhanced UI
 try:
@@ -64,9 +65,20 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
     if logger:
         print(f"{COLORS['green']}Logging conversation to: {logger.get_log_path()}{COLORS['reset']}")
     
+    # Display a note about web search if enabled
+    if web_search:
+        print(f"{COLORS['green']}Web search capability is enabled.{COLORS['reset']}")
+    
+    # Display a note about markdown rendering only once at the beginning
+    if prettify and not no_stream and not stream_prettify:
+        print(f"{COLORS['yellow']}Note: Using standard markdown rendering (--prettify). For streaming markdown rendering, use --stream-prettify instead.{COLORS['reset']}")
+    
     # Custom separator - use the same length for consistency
     def print_separator():
-        print(f"\n{separator}\n")
+        # Make sure there's exactly one newline before and after
+        # Use sys.stdout.write for direct control, avoiding any extra newlines
+        sys.stdout.write(f"\n{separator}\n")
+        sys.stdout.flush()
     
     # Initialize conversation history
     system_prompt = preprompt if preprompt else "You are a helpful assistant."
@@ -181,20 +193,49 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
             # Log user message if logging is enabled
             if logger:
                 logger.log("user", user_input)
+                
+            # Enhance prompt with web search if enabled
+            enhanced_prompt = user_input
+            if web_search:
+                try:
+                    print(f"{COLORS['cyan']}Searching the web...{COLORS['reset']}")
+                    enhanced_prompt = enhance_prompt_with_web_search(user_input, logger=logger)
+                    print(f"{COLORS['green']}Enhanced input with web search results.{COLORS['reset']}")
+                    
+                    # Update the user message in conversation with enhanced prompt
+                    for i in range(len(conversation) - 1, -1, -1):
+                        if conversation[i]["role"] == "user" and conversation[i]["content"] == user_input:
+                            conversation[i]["content"] = enhanced_prompt
+                            break
+                    
+                    # Log the enhanced prompt if logging is enabled
+                    if logger:
+                        # Use "web_search" role instead of "system" for clearer logs
+                        logger.log("web_search", enhanced_prompt.replace(user_input, "").strip())
+                except Exception as e:
+                    print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
+                    # Continue with the original prompt if web search fails
             
-            # Print assistant indicator with formatting
-            if not no_stream and not stream_prettify:
-                print(f"\n{ngpt_header()}: {COLORS['reset']}", end="", flush=True)
-            elif not stream_prettify:
-                print(f"\n{ngpt_header()}: {COLORS['reset']}", flush=True)
+            # Print assistant indicator with formatting - but only if we're not going to show a rich formatted box
+            # With Rich prettify, no header should be printed as the Rich panel already includes it
+            should_print_header = True
+
+            # Determine if we should print a header based on formatting options
+            if prettify and renderer != 'glow':
+                # Don't print header for Rich prettify
+                should_print_header = False
+
+            # Print the header if needed
+            if should_print_header:
+                if not no_stream and not stream_prettify:
+                    print(f"\n{ngpt_header()}: {COLORS['reset']}", end="", flush=True)
+                elif not stream_prettify:
+                    print(f"\n{ngpt_header()}: {COLORS['reset']}", flush=True)
             
-            # If prettify is enabled with regular streaming
+            # Determine streaming behavior
             if prettify and not no_stream and not stream_prettify:
-                print(f"\n{COLORS['yellow']}Note: Streaming disabled to enable markdown rendering.{COLORS['reset']}")
-                print(f"\n{ngpt_header()}: {COLORS['reset']}", flush=True)
                 should_stream = False
             else:
-                # Regular behavior with stream-prettify taking precedence
                 should_stream = not no_stream
             
             # Setup for stream-prettify
@@ -205,9 +246,15 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
             first_content_received = False
             
             if stream_prettify and should_stream:
-                # Get the correct header for interactive mode
-                header = ngpt_header()
-                live_display, stream_callback, setup_spinner = prettify_streaming_markdown(renderer, is_interactive=True, header_text=header)
+                # Don't pass the header_text for rich renderer as it already creates its own header,
+                # but pass it for other renderers like glow
+                if renderer == 'rich' or renderer == 'auto':
+                    live_display, stream_callback, setup_spinner = prettify_streaming_markdown(renderer, is_interactive=True)
+                else:
+                    # Get the correct header for interactive mode for non-rich renderers
+                    header = ngpt_header()
+                    live_display, stream_callback, setup_spinner = prettify_streaming_markdown(renderer, is_interactive=True, header_text=header)
+                
                 if not live_display:
                     # Fallback to normal prettify if live display setup failed
                     prettify = True
@@ -229,8 +276,9 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
                             if stop_spinner_func:
                                 stop_spinner_func()
                                 
-                            # Clear the spinner line completely
-                            sys.stdout.write("\r" + " " * 100 + "\r")
+                            # Clear the spinner line completely without leaving extra newlines
+                            # Use direct terminal control to ensure consistency
+                            sys.stdout.write("\r" + " " * shutil.get_terminal_size().columns + "\r")
                             sys.stdout.flush()
                             
                             # Now start the live display
@@ -250,10 +298,9 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
             
             # Get AI response with conversation history
             response = client.chat(
-                prompt=user_input,
+                prompt=enhanced_prompt,
                 messages=conversation,
                 stream=should_stream,
-                web_search=web_search,
                 temperature=temperature,
                 top_p=top_p,
                 max_tokens=max_tokens,
@@ -264,7 +311,9 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
             # Ensure spinner is stopped if no content was received
             if stop_spinner_event and not first_content_received:
                 stop_spinner_event.set()
-                sys.stdout.write("\r" + " " * 100 + "\r")
+                # Clear the spinner line completely without leaving extra newlines
+                # Use direct terminal control to ensure consistency
+                sys.stdout.write("\r" + " " * shutil.get_terminal_size().columns + "\r")
                 sys.stdout.flush()
             
             # Stop live display if using stream-prettify
@@ -281,6 +330,7 @@ def interactive_chat_session(client, web_search=False, no_stream=False, temperat
                 # Print response if not streamed (either due to no_stream or prettify)
                 if no_stream or prettify:
                     if prettify:
+                        # For pretty formatting with rich, don't print any header text as the rich renderer already includes it
                         prettify_markdown(response, renderer)
                     else:
                         print(response)

ngpt/cli/modes/chat.py

@@ -1,6 +1,7 @@
 from ..formatters import COLORS
 from ..renderers import prettify_markdown, prettify_streaming_markdown
 from ..ui import spinner
+from ...utils import enhance_prompt_with_web_search
 import sys
 import threading
 
@@ -53,6 +54,21 @@ def chat_mode(client, args, logger=None):
     # Log the user message if logging is enabled
     if logger:
         logger.log("user", prompt)
+    
+    # Enhance prompt with web search if enabled
+    if args.web_search:
+        try:
+            original_prompt = prompt
+            prompt = enhance_prompt_with_web_search(prompt, logger=logger)
+            print("Enhanced input with web search results.")
+            
+            # Log the enhanced prompt if logging is enabled
+            if logger:
+                # Use "web_search" role instead of "system" for clearer logs
+                logger.log("web_search", prompt.replace(original_prompt, "").strip())
+        except Exception as e:
+            print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
+            # Continue with the original prompt if web search fails
         
     # Create messages array with preprompt if available
     messages = None
@@ -87,7 +103,7 @@ def chat_mode(client, args, logger=None):
     
     # If regular prettify is enabled with streaming, inform the user
     if args.prettify and not args.no_stream:
-        print(f"{COLORS['yellow']}Note: Streaming disabled to enable markdown rendering.{COLORS['reset']}")
+        print(f"{COLORS['yellow']}Note: Using standard markdown rendering (--prettify). For streaming markdown rendering, use --stream-prettify instead.{COLORS['reset']}")
     
     # Show a static message if live_display is not available
     if args.stream_prettify and not live_display:
@@ -123,7 +139,7 @@ def chat_mode(client, args, logger=None):
     if args.stream_prettify and live_display:
         stream_callback = spinner_handling_callback
     
-    response = client.chat(prompt, stream=should_stream, web_search=args.web_search,
+    response = client.chat(prompt, stream=should_stream,
                        temperature=args.temperature, top_p=args.top_p,
                        max_tokens=args.max_tokens, messages=messages,
                        markdown_format=args.prettify or args.stream_prettify,

ngpt/cli/modes/code.py

@@ -1,6 +1,7 @@
 from ..formatters import COLORS
 from ..renderers import prettify_markdown, prettify_streaming_markdown, has_markdown_renderer, show_available_renderers
 from ..ui import spinner
+from ...utils import enhance_prompt_with_web_search
 import sys
 import threading
 
@@ -25,6 +26,21 @@ def code_mode(client, args, logger=None):
     # Log the user prompt if logging is enabled
     if logger:
         logger.log("user", prompt)
+    
+    # Enhance prompt with web search if enabled
+    if args.web_search:
+        try:
+            original_prompt = prompt
+            prompt = enhance_prompt_with_web_search(prompt, logger=logger)
+            print("Enhanced input with web search results.")
+            
+            # Log the enhanced prompt if logging is enabled
+            if logger:
+                # Use "web_search" role instead of "system" for clearer logs
+                logger.log("web_search", prompt.replace(original_prompt, "").strip())
+        except Exception as e:
+            print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
+            # Continue with the original prompt if web search fails
 
     # Setup for streaming and prettify logic
     stream_callback = None
@@ -62,7 +78,7 @@ def code_mode(client, args, logger=None):
         if has_markdown_renderer(args.renderer):
             should_stream = False
             use_regular_prettify = True
-            print(f"{COLORS['yellow']}Note: Streaming disabled to enable regular markdown rendering (--prettify).{COLORS['reset']}")
+            print(f"{COLORS['yellow']}Note: Using standard markdown rendering (--prettify). For streaming markdown rendering, use --stream-prettify instead.{COLORS['reset']}")
         else:
             # Renderer not available for prettify
             print(f"{COLORS['yellow']}Warning: Renderer '{args.renderer}' not available for --prettify.{COLORS['reset']}")
@@ -110,8 +126,7 @@ def code_mode(client, args, logger=None):
         
     generated_code = client.generate_code(
         prompt=prompt, 
-        language=args.language, 
-        web_search=args.web_search,
+        language=args.language,
         temperature=args.temperature, 
         top_p=args.top_p,
         max_tokens=args.max_tokens,

ngpt/cli/modes/rewrite.py

@@ -5,6 +5,7 @@ import time
 from ..formatters import COLORS
 from ..renderers import prettify_markdown, prettify_streaming_markdown
 from ..ui import get_multiline_input, spinner
+from ...utils import enhance_prompt_with_web_search
 
 # System prompt for rewriting text
 REWRITE_SYSTEM_PROMPT = """You are an expert text editor and rewriter. Your task is to rewrite the user's text to improve readability and flow while carefully preserving the original meaning, tone, and style.
@@ -125,6 +126,21 @@ def rewrite_mode(client, args, logger=None):
         print(f"{COLORS['yellow']}Error: Empty input. Please provide text to rewrite.{COLORS['reset']}")
         return
     
+    # Enhance input with web search if enabled
+    if args.web_search:
+        try:
+            original_text = input_text
+            input_text = enhance_prompt_with_web_search(input_text, logger=logger)
+            print("Enhanced input with web search results.")
+            
+            # Log the enhanced input if logging is enabled
+            if logger:
+                # Use "web_search" role instead of "system" for clearer logs
+                logger.log("web_search", input_text.replace(original_text, "").strip())
+        except Exception as e:
+            print(f"{COLORS['yellow']}Warning: Failed to enhance input with web search: {str(e)}{COLORS['reset']}")
+            # Continue with the original input if web search fails
+    
     # Set up messages array with system prompt and user content
     messages = [
         {"role": "system", "content": REWRITE_SYSTEM_PROMPT},
@@ -156,7 +172,7 @@ def rewrite_mode(client, args, logger=None):
     
     # If regular prettify is enabled with streaming, inform the user
     if args.prettify and not args.no_stream:
-        print(f"{COLORS['yellow']}Note: Streaming disabled to enable markdown rendering.{COLORS['reset']}")
+        print(f"{COLORS['yellow']}Note: Using standard markdown rendering (--prettify). For streaming markdown rendering, use --stream-prettify instead.{COLORS['reset']}")
     
     # Show a static message if live_display is not available
     if args.stream_prettify and not live_display:
@@ -195,7 +211,6 @@ def rewrite_mode(client, args, logger=None):
     response = client.chat(
         prompt=None,  # Not used when messages are provided
         stream=should_stream, 
-        web_search=args.web_search,
         temperature=args.temperature, 
         top_p=args.top_p,
         max_tokens=args.max_tokens, 

ngpt/cli/modes/shell.py

@@ -1,5 +1,6 @@
 from ..formatters import COLORS
 from ..ui import spinner
+from ...utils import enhance_prompt_with_web_search
 import subprocess
 import sys
 import threading
@@ -26,6 +27,21 @@ def shell_mode(client, args, logger=None):
     if logger:
         logger.log("user", prompt)
     
+    # Enhance prompt with web search if enabled
+    if args.web_search:
+        try:
+            original_prompt = prompt
+            prompt = enhance_prompt_with_web_search(prompt, logger=logger)
+            print("Enhanced input with web search results.")
+            
+            # Log the enhanced prompt if logging is enabled
+            if logger:
+                # Use "web_search" role instead of "system" for clearer logs
+                logger.log("web_search", prompt.replace(original_prompt, "").strip())
+        except Exception as e:
+            print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
+            # Continue with the original prompt if web search fails
+    
     # Start spinner while waiting for command generation
     stop_spinner = threading.Event()
     spinner_thread = threading.Thread(
@@ -37,9 +53,9 @@ def shell_mode(client, args, logger=None):
     spinner_thread.start()
     
     try:
-        command = client.generate_shell_command(prompt, web_search=args.web_search, 
-                                            temperature=args.temperature, top_p=args.top_p,
-                                            max_tokens=args.max_tokens)
+        command = client.generate_shell_command(prompt,
+                                           temperature=args.temperature, top_p=args.top_p,
+                                           max_tokens=args.max_tokens)
     finally:
         # Stop the spinner
         stop_spinner.set()

ngpt/cli/modes/text.py

@@ -1,6 +1,7 @@
 from ..formatters import COLORS
 from ..renderers import prettify_markdown, prettify_streaming_markdown
 from ..ui import get_multiline_input, spinner
+from ...utils import enhance_prompt_with_web_search
 import threading
 import sys
 
@@ -25,6 +26,21 @@ def text_mode(client, args, logger=None):
     if logger:
         logger.log("user", prompt)
     
+    # Enhance prompt with web search if enabled
+    if args.web_search:
+        try:
+            original_prompt = prompt
+            prompt = enhance_prompt_with_web_search(prompt, logger=logger)
+            print("Enhanced input with web search results.")
+            
+            # Log the enhanced prompt if logging is enabled
+            if logger:
+                # Use "web_search" role instead of "system" for clearer logs
+                logger.log("web_search", prompt.replace(original_prompt, "").strip())
+        except Exception as e:
+            print(f"{COLORS['yellow']}Warning: Failed to enhance prompt with web search: {str(e)}{COLORS['reset']}")
+            # Continue with the original prompt if web search fails
+    
     # Create messages array with preprompt if available
     messages = None
     if args.preprompt:
@@ -58,7 +74,7 @@ def text_mode(client, args, logger=None):
     
     # If regular prettify is enabled with streaming, inform the user
     if args.prettify and not args.no_stream:
-        print(f"{COLORS['yellow']}Note: Streaming disabled to enable markdown rendering.{COLORS['reset']}")
+        print(f"{COLORS['yellow']}Note: Using standard markdown rendering (--prettify). For streaming markdown rendering, use --stream-prettify instead.{COLORS['reset']}")
     
     # Show a static message if live_display is not available
     if args.stream_prettify and not live_display:
@@ -94,7 +110,7 @@ def text_mode(client, args, logger=None):
     if args.stream_prettify and live_display:
         stream_callback = spinner_handling_callback
     
-    response = client.chat(prompt, stream=should_stream, web_search=args.web_search,
+    response = client.chat(prompt, stream=should_stream,
                        temperature=args.temperature, top_p=args.top_p,
                        max_tokens=args.max_tokens, messages=messages,
                        markdown_format=args.prettify or args.stream_prettify,

ngpt/cli/renderers.py

@@ -168,8 +168,31 @@ def prettify_markdown(text, renderer='auto'):
         # Use rich
         try:
             console = Console()
+            
+            # Create a panel around the markdown for consistency with stream_prettify
+            from rich.panel import Panel
+            import rich.box
+            from rich.text import Text
+            
+            # Get terminal dimensions
+            term_width = shutil.get_terminal_size().columns
+            
+            # Create panel with similar styling to stream_prettify
+            clean_header = "🤖 nGPT"
+            panel_title = Text(clean_header, style="cyan bold")
+            
             md = Markdown(text)
-            console.print(md)
+            panel = Panel(
+                md,
+                title=panel_title,
+                title_align="left",
+                border_style="cyan",
+                padding=(1, 1),
+                width=console.width - 4,  # Make panel slightly narrower than console
+                box=rich.box.ROUNDED
+            )
+            
+            console.print(panel)
             return True
         except Exception as e:
             print(f"{COLORS['yellow']}Error using rich for markdown: {str(e)}{COLORS['reset']}")
@@ -237,7 +260,20 @@ def prettify_streaming_markdown(renderer='rich', is_interactive=False, header_te
                 box=rich.box.ROUNDED
             )
         else:
-            md_obj = Markdown("")
+            # Always use a panel - even in non-interactive mode
+            clean_header = "🤖 nGPT"
+            panel_title = Text(clean_header, style="cyan bold")
+            
+            padding = (1, 1)  # Less horizontal padding (left, right)
+            md_obj = Panel(
+                Markdown(""),
+                title=panel_title,
+                title_align="left",
+                border_style="cyan",
+                padding=padding,
+                width=console.width - 4,  # Make panel slightly narrower than console
+                box=rich.box.ROUNDED
+            )
         
         # Get terminal dimensions for better display
         term_width = shutil.get_terminal_size().columns
@@ -275,9 +311,8 @@ def prettify_streaming_markdown(renderer='rich', is_interactive=False, header_te
             # Start live display on first content
             if first_update:
                 first_update = False
-                # Clear the spinner line completely before starting the display
-                sys.stdout.write("\r" + " " * 100 + "\r")
-                sys.stdout.flush()
+                # Let the spinner's clean_exit handle the cleanup
+                # No additional cleanup needed here
                 live.start()
             
             # Update content in live display
@@ -304,17 +339,17 @@ def prettify_streaming_markdown(renderer='rich', is_interactive=False, header_te
                 if not is_complete:
                     # Calculate approximate lines needed
                     content_lines = content.count('\n') + 1
-                    available_height = display_height - 1  # Account for minimal overhead
+                    available_height = display_height - 4  # Account for panel borders and padding
                     
                     if content_lines > available_height:
                         # If content is too big, show only the last part that fits
                         lines = content.split('\n')
                         truncated_content = '\n'.join(lines[-available_height:])
-                        md_obj = Markdown(truncated_content)
+                        md_obj.renderable = Markdown(truncated_content)
                     else:
-                        md_obj = Markdown(content)
+                        md_obj.renderable = Markdown(content)
                 else:
-                    md_obj = Markdown(content)
+                    md_obj.renderable = Markdown(content)
                     
                 live.update(md_obj)
                 
@@ -344,7 +379,7 @@ def prettify_streaming_markdown(renderer='rich', is_interactive=False, header_te
             spinner_thread = threading.Thread(
                 target=spinner,
                 args=(message,),
-                kwargs={"stop_event": stop_event, "color": color}
+                kwargs={"stop_event": stop_event, "color": color, "clean_exit": True}
             )
             spinner_thread.daemon = True
             spinner_thread.start()

ngpt/cli/ui.py

@@ -155,7 +155,7 @@ def get_multiline_input():
             print("\nInput cancelled by user. Exiting gracefully.")
             return None
 
-def spinner(message, duration=5, spinner_chars="⣾⣽⣻⢿⡿⣟⣯⣷", color=None, stop_event=None):
+def spinner(message, duration=5, spinner_chars="⣾⣽⣻⢿⡿⣟⣯⣷", color=None, stop_event=None, clean_exit=False):
     """Display a spinner animation with a message.
     
     Args:
@@ -165,6 +165,7 @@ def spinner(message, duration=5, spinner_chars="⣾⣽⣻⢿⡿⣟⣯⣷", color
         color: Optional color from COLORS dict to use for the message
         stop_event: Optional threading.Event to signal when to stop the spinner
                    If provided, duration is ignored and spinner runs until event is set
+        clean_exit: When True, cleans up more aggressively to prevent blank lines
     """
     char_duration = 0.2
 
@@ -173,19 +174,30 @@ def spinner(message, duration=5, spinner_chars="⣾⣽⣻⢿⡿⣟⣯⣷", color
     if color:
         colored_message = f"{color}{message}{COLORS['reset']}"
 
+    # Save cursor position - will be needed for clean exit
+    if clean_exit:
+        # Start by printing a \r to ensure we begin at the start of the line
+        sys.stdout.write("\r")
+        sys.stdout.flush()
+
     if stop_event:
         i = 0
         while not stop_event.is_set():
             char = spinner_chars[i % len(spinner_chars)]
-            print(f"\r{colored_message} {char}", end="", flush=True)
+            # Always use sys.stdout.write for consistent behavior
+            sys.stdout.write(f"\r{colored_message} {char}")
+            sys.stdout.flush()
             i += 1
             time.sleep(char_duration)
     else:
         total_chars = int(duration / char_duration)
         for i in range(total_chars):
             char = spinner_chars[i % len(spinner_chars)]
-            print(f"\r{colored_message} {char}", end="", flush=True)
+            sys.stdout.write(f"\r{colored_message} {char}")
+            sys.stdout.flush()
             time.sleep(char_duration)
 
-    # Clear the line when done
-    print("\r" + " " * (len(message) + 10) + "\r", end="", flush=True) 
+    # Clear the line when done - use terminal width to clear the entire line
+    terminal_width = shutil.get_terminal_size().columns
+    sys.stdout.write("\r" + " " * terminal_width + "\r")
+    sys.stdout.flush() 

ngpt/client.py

@@ -32,7 +32,6 @@ class NGPTClient:
         max_tokens: Optional[int] = None,
         top_p: float = 1.0,
         messages: Optional[List[Dict[str, str]]] = None,
-        web_search: bool = False,
         markdown_format: bool = False,
         stream_callback: Optional[callable] = None,
         **kwargs
@@ -47,7 +46,6 @@ class NGPTClient:
             max_tokens: Maximum number of tokens to generate
             top_p: Controls diversity via nucleus sampling
             messages: Optional list of message objects to override default behavior
-            web_search: Whether to enable web search capability
             markdown_format: If True, allow markdown-formatted responses, otherwise plain text
             stream_callback: Optional callback function for streaming mode updates
             **kwargs: Additional arguments to pass to the API
@@ -75,10 +73,6 @@ class NGPTClient:
             "top_p": top_p,
         }
         
-        # Conditionally add web_search
-        if web_search:
-            payload["web_search"] = True
-        
         # Add max_tokens if provided
         if max_tokens is not None:
             payload["max_tokens"] = max_tokens
@@ -180,7 +174,6 @@ class NGPTClient:
     def generate_shell_command(
         self, 
         prompt: str, 
-        web_search: bool = False,
         temperature: float = 0.4,
         top_p: float = 0.95,
         max_tokens: Optional[int] = None
@@ -190,7 +183,6 @@ class NGPTClient:
         
         Args:
             prompt: Description of the command to generate
-            web_search: Whether to enable web search capability
             temperature: Controls randomness in the response
             top_p: Controls diversity via nucleus sampling
             max_tokens: Maximum number of tokens to generate
@@ -241,7 +233,6 @@ Command:"""
                 prompt=prompt,
                 stream=False,
                 messages=messages,
-                web_search=web_search,
                 temperature=temperature,
                 top_p=top_p,
                 max_tokens=max_tokens
@@ -254,7 +245,6 @@ Command:"""
         self, 
         prompt: str, 
         language: str = "python", 
-        web_search: bool = False,
         temperature: float = 0.4,
         top_p: float = 0.95,
         max_tokens: Optional[int] = None,
@@ -268,7 +258,6 @@ Command:"""
         Args:
             prompt: Description of the code to generate
             language: Programming language to generate code in
-            web_search: Whether to enable web search capability
             temperature: Controls randomness in the response
             top_p: Controls diversity via nucleus sampling
             max_tokens: Maximum number of tokens to generate
@@ -315,7 +304,6 @@ Code:"""
                 prompt=prompt,
                 stream=stream,
                 messages=messages,
-                web_search=web_search,
                 temperature=temperature,
                 top_p=top_p,
                 max_tokens=max_tokens,

ngpt/utils/__init__.py

@@ -22,6 +22,11 @@ from .cli_config import (
     get_cli_config_dir,
     get_cli_config_path
 )
+from .web_search import (
+    enhance_prompt_with_web_search,
+    get_web_search_results,
+    format_web_search_results_for_prompt
+)
 
 __all__ = [
     "create_logger", "Logger",
@@ -29,5 +34,6 @@ __all__ = [
     "add_config_entry", "remove_config_entry", "DEFAULT_CONFIG", "DEFAULT_CONFIG_ENTRY",
     "load_cli_config", "set_cli_config_option", "get_cli_config_option", 
     "unset_cli_config_option", "apply_cli_config", "list_cli_config_options",
-    "CLI_CONFIG_OPTIONS", "get_cli_config_dir", "get_cli_config_path"
+    "CLI_CONFIG_OPTIONS", "get_cli_config_dir", "get_cli_config_path",
+    "enhance_prompt_with_web_search", "get_web_search_results", "format_web_search_results_for_prompt"
 ]

ngpt/utils/web_search.py

@@ -0,0 +1,270 @@
+"""
+Web search utilities for nGPT using duckduckgo-search and trafilatura.
+
+This module provides functionality to search the web and extract
+information from search results to enhance AI prompts.
+"""
+
+import re
+from typing import List, Dict, Any, Optional
+from duckduckgo_search import DDGS
+from urllib.parse import urlparse
+import requests
+import sys
+
+# Get actual logger from global context instead of using standard logging
+from . import log
+
+# Use a global variable to store the logger provided during runtime
+_logger = None
+
+def set_logger(logger):
+    """Set the logger to use for this module."""
+    global _logger
+    _logger = logger
+
+def get_logger():
+    """Get the current logger or use a default."""
+    if _logger is not None:
+        return _logger
+    else:
+        # Default logging to stderr if no logger provided
+        class DefaultLogger:
+            def info(self, msg): print(f"INFO: {msg}", file=sys.stderr)
+            def error(self, msg): print(f"ERROR: {msg}", file=sys.stderr)
+            def warning(self, msg): print(f"WARNING: {msg}", file=sys.stderr)
+            def debug(self, msg): pass
+        return DefaultLogger()
+
+def perform_web_search(query: str, max_results: int = 3) -> List[Dict[str, Any]]:
+    """
+    Search the web using DuckDuckGo and return relevant results.
+    
+    Args:
+        query: The search query
+        max_results: Maximum number of results to return
+        
+    Returns:
+        List of dictionaries containing search results (title, url, snippet)
+    """
+    logger = get_logger()
+    try:
+        ddgs = DDGS()
+        results = list(ddgs.text(query, max_results=max_results))
+        return results
+    except Exception as e:
+        logger.error(f"Error performing web search: {str(e)}")
+        logger.info("Web search encountered an issue, but will continue with available results")
+        return []
+
+def extract_article_content(url: str, max_chars: int = 2000) -> Optional[str]:
+    """
+    Extract and clean content from a webpage URL.
+    
+    Args:
+        url: The URL to extract content from
+        max_chars: Maximum number of characters to extract
+        
+    Returns:
+        Cleaned article text or None if extraction failed
+    """
+    logger = get_logger()
+    try:
+        # Skip non-http URLs or suspicious domains
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme.startswith('http'):
+            return None
+        
+        # Browser-like user agent
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+        }
+        
+        logger.info(f"Fetching content from {url}")
+        
+        try:
+            # Try using trafilatura
+            import trafilatura
+            
+            # Download with correct parameters
+            # trafilatura handles user-agent internally
+            downloaded = trafilatura.fetch_url(url)
+            
+            if downloaded:
+                # Extract main content
+                content = trafilatura.extract(downloaded, include_comments=False, 
+                                             include_tables=False, 
+                                             no_fallback=False)
+                
+                if content:
+                    # Clean up content if needed
+                    content = content.strip()
+                    
+                    # Truncate if needed
+                    if len(content) > max_chars:
+                        content = content[:max_chars] + "..."
+                        
+                    return content
+            
+            # If trafilatura failed, try direct requests
+            logger.info(f"Trafilatura extraction failed for {url}, trying fallback method")
+            response = requests.get(url, headers=headers, timeout=10)
+            
+            if response.status_code == 200:
+                # Very basic HTML cleaning
+                html_content = response.text
+                # Remove HTML tags
+                text = re.sub(r'<[^>]+>', ' ', html_content)
+                # Remove excess whitespace
+                text = re.sub(r'\s+', ' ', text).strip()
+                
+                if text:
+                    if len(text) > max_chars:
+                        text = text[:max_chars] + "..."
+                    return text
+                
+            else:
+                logger.error(f"Request to {url} returned status code {response.status_code}")
+                
+        except ImportError:
+            logger.error("Trafilatura not installed. Install with 'pip install trafilatura'")
+            # Try direct requests only
+            try:
+                response = requests.get(url, headers=headers, timeout=10)
+                if response.status_code == 200:
+                    # Very basic HTML cleaning
+                    html_content = response.text
+                    text = re.sub(r'<[^>]+>', ' ', html_content)
+                    text = re.sub(r'\s+', ' ', text).strip()
+                    
+                    if text:
+                        if len(text) > max_chars:
+                            text = text[:max_chars] + "..."
+                        return text
+            except Exception as req_error:
+                logger.error(f"Direct request fallback failed: {str(req_error)}")
+                
+        except Exception as e:
+            logger.error(f"Error extracting content with trafilatura: {str(e)}")
+            # Try the requests fallback
+            try:
+                response = requests.get(url, headers=headers, timeout=10)
+                if response.status_code == 200:
+                    html_content = response.text
+                    text = re.sub(r'<[^>]+>', ' ', html_content)
+                    text = re.sub(r'\s+', ' ', text).strip()
+                    
+                    if text:
+                        if len(text) > max_chars:
+                            text = text[:max_chars] + "..."
+                        return text
+            except Exception as req_error:
+                logger.error(f"Direct request fallback failed: {str(req_error)}")
+            
+        return None
+    except Exception as e:
+        logger.error(f"Error extracting content from {url}: {str(e)}")
+        return None
+
+def get_web_search_results(query: str, max_results: int = 3, max_chars_per_result: int = 2000) -> Dict[str, Any]:
+    """
+    Get formatted web search results ready to be included in AI prompts.
+    
+    Args:
+        query: The search query
+        max_results: Maximum number of results to include
+        max_chars_per_result: Maximum characters to include per result
+        
+    Returns:
+        Dictionary containing search results and metadata
+    """
+    logger = get_logger()
+    search_results = perform_web_search(query, max_results)
+    enhanced_results = []
+    success_count = 0
+    failure_count = 0
+    
+    for result in search_results:
+        content = extract_article_content(result['href'], max_chars_per_result)
+        
+        enhanced_results.append({
+            'title': result.get('title', ''),
+            'url': result.get('href', ''),
+            'snippet': result.get('body', ''),
+            'content': content if content else result.get('body', '')
+        })
+        
+        if content:
+            success_count += 1
+        else:
+            failure_count += 1
+    
+    # Log a user-friendly summary
+    if search_results:
+        if failure_count > 0:
+            logger.info(f"Retrieved content from {success_count} out of {len(search_results)} sources")
+        else:
+            logger.info(f"Successfully retrieved content from all {success_count} sources")
+    else:
+        logger.error("No search results were found")
+            
+    return {
+        'query': query,
+        'timestamp': 'current_time',  # Could replace with actual timestamp
+        'results': enhanced_results
+    }
+
+def format_web_search_results_for_prompt(search_results: Dict[str, Any]) -> str:
+    """
+    Format web search results into a string to include in AI prompts.
+    
+    Args:
+        search_results: Dictionary of search results from get_web_search_results()
+        
+    Returns:
+        Formatted string to include in prompts
+    """
+    query = search_results['query']
+    results = search_results['results']
+    
+    formatted_text = f"[Web Search Results for: {query}]\n\n"
+    
+    for i, result in enumerate(results, 1):
+        formatted_text += f"RESULT {i}: {result['title']}\n"
+        formatted_text += f"URL: {result['url']}\n"
+        formatted_text += f"CONTENT:\n{result['content']}\n\n"
+    
+    formatted_text += f"[End of Web Search Results]\n\n"
+    formatted_text += "Consider the above information when answering the following question:\n\n"
+    
+    return formatted_text
+
+def enhance_prompt_with_web_search(prompt: str, max_results: int = 3, logger=None) -> str:
+    """
+    Enhance a prompt with web search results.
+    
+    Args:
+        prompt: The original user prompt
+        max_results: Maximum number of search results to include
+        logger: Optional logger to use
+        
+    Returns:
+        Enhanced prompt with web search results prepended
+    """
+    # Set the logger for this module
+    if logger is not None:
+        set_logger(logger)
+        
+    logger = get_logger()
+    search_results = get_web_search_results(prompt, max_results)
+    formatted_results = format_web_search_results_for_prompt(search_results)
+    
+    # Combine results with original prompt
+    enhanced_prompt = formatted_results + prompt
+    
+    logger.info("Enhanced input with web search results")
+    return enhanced_prompt 

{ngpt-3.1.1.dist-info → ngpt-3.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngpt
-Version: 3.1.1
+Version: 3.3.0
 Summary: Swiss army knife for LLMs: powerful CLI, interactive chatbot, and flexible Python library. Works with OpenAI, Ollama, Groq, Claude, Gemini, and any OpenAI-compatible API.
 Project-URL: Homepage, https://github.com/nazdridoy/ngpt
 Project-URL: Repository, https://github.com/nazdridoy/ngpt
@@ -28,10 +28,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Requires-Python: >=3.8
+Requires-Dist: duckduckgo-search>=3.0.0
 Requires-Dist: prompt-toolkit>=3.0.0
 Requires-Dist: pyperclip>=1.8.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=10.0.0
+Requires-Dist: trafilatura>=1.6.0
 Description-Content-Type: text/markdown
 
 # nGPT
@@ -63,7 +65,7 @@ Description-Content-Type: text/markdown
 - 🔄 **API Flexibility**: Works with OpenAI, Ollama, Groq, Claude, Gemini, and any compatible endpoint
 - 💬 **Interactive Chat**: Continuous conversation with memory in modern UI
 - 📊 **Streaming Responses**: Real-time output for better user experience
-- 🔍 **Web Search**: Integrated with compatible API endpoints
+- 🔍 **Web Search**: Enhance any model with contextual information from the web
 - 📥 **Stdin Processing**: Process piped content by using `{}` placeholder in prompts
 - 🎨 **Markdown Rendering**: Beautiful formatting of markdown and code with syntax highlighting
 - ⚡ **Real-time Markdown**: Stream responses with live updating syntax highlighting and formatting
@@ -271,7 +273,7 @@ ngpt --list-models --provider Gemini
 # With custom options
 ngpt --api-key your-key --base-url http://your-endpoint --model your-model "Hello"
 
-# Enable web search (if your API endpoint supports it)
+# Enable web search capability to enhance prompts with web information
 ngpt --web-search "What's the latest news about AI?"
 
 # Generate and execute shell commands (using -s or --shell flag)

ngpt-3.3.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+ngpt/__init__.py,sha256=kpKhViLakwMdHZkuLht2vWcjt0uD_5gR33gvMhfXr6w,664
+ngpt/__main__.py,sha256=j3eFYPOtCCFBOGh7NK5IWEnADnTMMSEB9GLyIDoW724,66
+ngpt/client.py,sha256=1kn-kVQ2ZYhOlQ5OPM9c_btVKajfk1qb52QbMyGdYtU,14645
+ngpt/cli/__init__.py,sha256=hebbDSMGiOd43YNnQP67uzr67Ue6rZPwm2czynr5iZY,43
+ngpt/cli/args.py,sha256=7ixh5anahTeHWsrEKwhaLA_dR3sJ-ZzO8NyGOeDmd3Q,11392
+ngpt/cli/config_manager.py,sha256=NQQcWnjUppAAd0s0p9YAf8EyKS1ex5-0EB4DvKdB4dk,3662
+ngpt/cli/formatters.py,sha256=HBYGlx_7eoAKyzfy0Vq5L0yn8yVKjngqYBukMmXCcz0,9401
+ngpt/cli/interactive.py,sha256=6SrnANvE5T9Luu-CSGPfW7lub4-yog9aqruAfs1XEIg,16392
+ngpt/cli/main.py,sha256=9um40RplKHSW5UHcUUO2cwMNqkGUhfQwikI1CHHFbnk,28926
+ngpt/cli/renderers.py,sha256=m71BeUXKynpKKGXFzwRSW1XngvyKiZ_xEsdujUbU0MA,16597
+ngpt/cli/ui.py,sha256=HoHDFpLiwMBP5wtMb8YYo244FMiqiPFRoBNcNGp6N0A,7310
+ngpt/cli/modes/__init__.py,sha256=R3aO662RIzWEOvr3moTrEI8Tpg0zDDyMGGh1-OxiRgM,285
+ngpt/cli/modes/chat.py,sha256=UlnZWqvxL_CSyVIDZKGVXNlG-KufDfo05E8-8xTcM_Y,6713
+ngpt/cli/modes/code.py,sha256=4TpIV-EkyXb8d3XJKrsLs9Wxq5P9jC82z94PI2Zck_g,6730
+ngpt/cli/modes/gitcommsg.py,sha256=rsfMoeOupmNp-5p5fsMSPAf18BbzXWq-4PF2HjEz6SY,46991
+ngpt/cli/modes/rewrite.py,sha256=ftD-6M9iQ7g4rLdlKyyLTRiJWYtbz64LIG4PIByxmOk,11472
+ngpt/cli/modes/shell.py,sha256=fxE9LEEo4arSn5-q_6zxdnUH7RlqifWmk-_kcA76OhM,4070
+ngpt/cli/modes/text.py,sha256=gdn4opioZ6G3nvfrTkp-dpoD-Of_ZvjVVRggVd6edkg,5528
+ngpt/utils/__init__.py,sha256=qu_66I1Vtav2f1LDiPn5J3DUsbK7o1CSScMcTkYqxoM,1179
+ngpt/utils/cli_config.py,sha256=IlHnOEEGpLoGZInynM778wgpxLVcJ_STKWxg2Ypvir4,11196
+ngpt/utils/config.py,sha256=wsArA4osnh8fKqOvtsPqqBxAz3DpdjtaWUFaRtnUdyc,10452
+ngpt/utils/log.py,sha256=f1jg2iFo35PAmsarH8FVL_62plq4VXH0Mu2QiP6RJGw,15934
+ngpt/utils/web_search.py,sha256=DXjf5zJW4jXbmpir6Oqv7abfrFxjqJqvnuAA9gJIFJk,9919
+ngpt-3.3.0.dist-info/METADATA,sha256=1gGlQ6gzQwnnB8wfdmzcwlf3DRV9JNup7HgfXsSRo2Y,29100
+ngpt-3.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ngpt-3.3.0.dist-info/entry_points.txt,sha256=SqAAvLhMrsEpkIr4YFRdUeyuXQ9o0IBCeYgE6AVojoI,44
+ngpt-3.3.0.dist-info/licenses/LICENSE,sha256=mQkpWoADxbHqE0HRefYLJdm7OpdrXBr3vNv5bZ8w72M,1065
+ngpt-3.3.0.dist-info/RECORD,,

ngpt-3.1.1.dist-info/RECORD

@@ -1,27 +0,0 @@
-ngpt/__init__.py,sha256=kpKhViLakwMdHZkuLht2vWcjt0uD_5gR33gvMhfXr6w,664
-ngpt/__main__.py,sha256=j3eFYPOtCCFBOGh7NK5IWEnADnTMMSEB9GLyIDoW724,66
-ngpt/client.py,sha256=rLgDPmJe8_yi13-XUiHJ45z54rJVrupxWmeb-fQZGF4,15129
-ngpt/cli/__init__.py,sha256=hebbDSMGiOd43YNnQP67uzr67Ue6rZPwm2czynr5iZY,43
-ngpt/cli/args.py,sha256=XQvofZs_WkbgUto3Dbx7Yw-AmPAQHh8kdHUe3uWy8w4,11382
-ngpt/cli/config_manager.py,sha256=NQQcWnjUppAAd0s0p9YAf8EyKS1ex5-0EB4DvKdB4dk,3662
-ngpt/cli/formatters.py,sha256=HBYGlx_7eoAKyzfy0Vq5L0yn8yVKjngqYBukMmXCcz0,9401
-ngpt/cli/interactive.py,sha256=Zep4yiGRFPVrJV1zp7xB5lXg7hVhXhbRlaVA4DgRBbQ,13241
-ngpt/cli/main.py,sha256=9um40RplKHSW5UHcUUO2cwMNqkGUhfQwikI1CHHFbnk,28926
-ngpt/cli/renderers.py,sha256=ovV0IexZyrjCk8LK5ZGVvf5K7wCLco4jRhDv_tp6Kvw,15184
-ngpt/cli/ui.py,sha256=m8qtd4cCSHBGHPUlHVdBEfun1G1Se4vLKTSgnS7QOKE,6775
-ngpt/cli/modes/__init__.py,sha256=R3aO662RIzWEOvr3moTrEI8Tpg0zDDyMGGh1-OxiRgM,285
-ngpt/cli/modes/chat.py,sha256=oibPaN4LAqrzIbWpw--ql0TwU0u_jlLCeNuxxyO7wPg,5887
-ngpt/cli/modes/code.py,sha256=AEsQKdL8mh4ImcI5R7TXzwEq_JBTkmS-L0AA_-dpHm0,5934
-ngpt/cli/modes/gitcommsg.py,sha256=rsfMoeOupmNp-5p5fsMSPAf18BbzXWq-4PF2HjEz6SY,46991
-ngpt/cli/modes/rewrite.py,sha256=CrzUHxUoZc48D3ooVIvDbCjIIuERO1v-ddG3ajhETQs,10646
-ngpt/cli/modes/shell.py,sha256=QkprnOxMMTg2v5DIwcofDnnr3JPNfuk-YgSQaae5Xps,3311
-ngpt/cli/modes/text.py,sha256=vCwZnVxIbsTCLE8YVJIPwZsGgk7VRs8v-eeaVwCOZqI,4702
-ngpt/utils/__init__.py,sha256=E46suk2-QgYBI0Qrs6WXOajOUOebF3ETAFY7ah8DTWs,942
-ngpt/utils/cli_config.py,sha256=IlHnOEEGpLoGZInynM778wgpxLVcJ_STKWxg2Ypvir4,11196
-ngpt/utils/config.py,sha256=wsArA4osnh8fKqOvtsPqqBxAz3DpdjtaWUFaRtnUdyc,10452
-ngpt/utils/log.py,sha256=f1jg2iFo35PAmsarH8FVL_62plq4VXH0Mu2QiP6RJGw,15934
-ngpt-3.1.1.dist-info/METADATA,sha256=YPavkwb6Jg4oblYwYtBX3XrDmvcQWVhQZO2d9r5yByI,28992
-ngpt-3.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ngpt-3.1.1.dist-info/entry_points.txt,sha256=SqAAvLhMrsEpkIr4YFRdUeyuXQ9o0IBCeYgE6AVojoI,44
-ngpt-3.1.1.dist-info/licenses/LICENSE,sha256=mQkpWoADxbHqE0HRefYLJdm7OpdrXBr3vNv5bZ8w72M,1065
-ngpt-3.1.1.dist-info/RECORD,,