webscout 8.1__py3-none-any.whl → 8.2__py3-none-any.whl

webscout/Local/cli.py

@@ -0,0 +1,338 @@
+"""
+Command-line interface for webscout.Local
+"""
+
+import typer
+from rich.console import Console
+from rich.table import Table
+from rich.prompt import Prompt
+from typing import Optional
+
+from .model_manager import ModelManager
+from .llm import LLMInterface
+from .server import start_server
+
+app: typer.Typer = typer.Typer(help="webscout.Local - A llama-cpp-python based LLM serving tool")
+console: Console = Console()
+
+model_manager: ModelManager = ModelManager()
+
+@app.command("serve")
+def run_model(
+    model_string: str = typer.Argument(..., help="Model to run (format: 'name', 'repo_id' or 'repo_id:filename')"),
+    host: Optional[str] = typer.Option(None, help="Host to bind the server to"),
+    port: Optional[int] = typer.Option(None, help="Port to bind the server to"),
+) -> None:
+    """
+    Start a model server (downloads if needed).
+    """
+    # First check if this is a filename that already exists
+    model_path = model_manager.get_model_path(model_string)
+    if model_path:
+        # This is a filename that exists, find the model name
+        for model_info in model_manager.list_models():
+            if model_info.get("filename") == model_string or model_info.get("path") == model_path:
+                model_name = model_info.get("name")
+                break
+        else:
+            # Fallback to using the string as model name
+            model_name = model_string
+    else:
+        # Parse the model string to see if it's a repo_id:filename format
+        repo_id, _ = model_manager.parse_model_string(model_string)
+        model_name = repo_id.split("/")[-1] if "/" in repo_id else repo_id
+
+        # Check if model exists, if not try to download it
+        if not model_manager.get_model_path(model_name):
+            console.print(f"[yellow]Model {model_name} not found locally. Attempting to download...[/yellow]")
+            try:
+                # We don't need to use the parsed values directly as download_model handles this
+                _ = model_manager.parse_model_string(model_string)  # Just to validate the format
+                # Download the model
+                model_name, _ = model_manager.download_model(model_string)
+                console.print(f"[bold green]Model {model_name} downloaded successfully[/bold green]")
+            except Exception as e:
+                console.print(f"[bold red]Error downloading model: {str(e)}[/bold red]")
+                return
+
+    # Try to load the model to verify it works
+    try:
+        llm = LLMInterface(model_name)
+        llm.load_model(verbose=False)
+        console.print(f"[bold green]Model {model_name} loaded successfully[/bold green]")
+    except Exception as e:
+        console.print(f"[bold red]Error loading model: {str(e)}[/bold red]")
+        return
+
+    # Start the server
+    console.print(f"[bold blue]Starting webscout.Local server with model {model_name}...[/bold blue]")
+    start_server(host=host, port=port)
+
+@app.command("pull")
+def pull_model(
+    model_string: str = typer.Argument(..., help="Model to download (format: 'repo_id' or 'repo_id:filename')"),
+) -> None:
+    """
+    Download a model from Hugging Face without running it.
+    """
+    try:
+        model_name, model_path = model_manager.download_model(model_string)
+        console.print(f"[bold green]Model {model_name} downloaded successfully to {model_path}[/bold green]")
+    except Exception as e:
+        console.print(f"[bold red]Error downloading model: {str(e)}[/bold red]")
+
+@app.command("list")
+def list_models() -> None:
+    """
+    List downloaded models.
+    """
+    models = model_manager.list_models()
+
+    if not models:
+        console.print("[yellow]No models found. Use 'webscout.Local pull' to download a model.[/yellow]")
+        return
+
+    table = Table(title="Downloaded Models")
+    table.add_column("Name", style="cyan")
+    table.add_column("Repository", style="green")
+    table.add_column("Filename", style="blue")
+
+    for model in models:
+        table.add_row(
+            model["name"],
+            model.get("repo_id", "Unknown"),
+            model.get("filename", "Unknown"),
+        )
+
+    console.print(table)
+
+@app.command(name="remove", help="Remove a downloaded model")
+def remove_model(
+    model_string: str = typer.Argument(..., help="Name or filename of the model to remove"),
+    force: bool = typer.Option(False, "--force", "-f", help="Force removal without confirmation"),
+) -> None:
+    """
+    Remove a downloaded model.
+    """
+    # First check if this is a model name
+    model_info = model_manager.get_model_info(model_string)
+
+    # If not found by name, check if it's a filename
+    if not model_info:
+        for info in model_manager.list_models():
+            if info.get("filename") == model_string:
+                model_info = info
+                model_string = info["name"]
+                break
+
+    if not model_info:
+        console.print(f"[yellow]Model {model_string} not found.[/yellow]")
+        return
+
+    if not force:
+        confirm = Prompt.ask(
+            f"Are you sure you want to remove model {model_string}?",
+            choices=["y", "n"],
+            default="n",
+        )
+
+        if confirm.lower() != "y":
+            console.print("[yellow]Operation cancelled.[/yellow]")
+            return
+
+    if model_manager.remove_model(model_string):
+        console.print(f"[bold green]Model {model_string} removed successfully[/bold green]")
+    else:
+        console.print(f"[bold red]Error removing model {model_string}[/bold red]")
+
+@app.command("run")
+def chat(
+    model_string: str = typer.Argument(..., help="Name or filename of the model to chat with"),
+) -> None:
+    """
+    Interactive chat with a model.
+    """
+    # First check if this is a filename that already exists
+    model_path = model_manager.get_model_path(model_string)
+    if model_path:
+        # This is a filename that exists, find the model name
+        for model_info in model_manager.list_models():
+            if model_info.get("filename") == model_string or model_info.get("path") == model_path:
+                model_name = model_info.get("name")
+                break
+        else:
+            # Fallback to using the string as model name
+            model_name = model_string
+    else:
+        # Use the string as model name
+        model_name = model_string
+
+        # Check if model exists, if not try to download it
+        if not model_manager.get_model_path(model_name):
+            console.print(f"[yellow]Model {model_name} not found locally. Attempting to download...[/yellow]")
+            try:
+                # Parse the model string to see if it's a repo_id:filename format
+                # We don't need to use the parsed values directly as download_model handles this
+                _ = model_manager.parse_model_string(model_string)  # Just to validate the format
+                # Download the model
+                model_name, _ = model_manager.download_model(model_string)
+                console.print(f"[bold green]Model {model_name} downloaded successfully[/bold green]")
+            except Exception as e:
+                console.print(f"[bold red]Error downloading model: {str(e)}[/bold red]")
+                return
+
+    # Load the model
+    try:
+        llm = LLMInterface(model_name)
+        llm.load_model(verbose=False)
+    except Exception as e:
+        console.print(f"[bold red]Error loading model: {str(e)}[/bold red]")
+        return
+
+    console.print(f"[bold green]Chat with {model_name}. Type '/help' for available commands or '/bye' to exit.[bold green]")
+
+    # Chat history
+    messages = []
+    system_prompt = None
+
+    # Initialize with empty system prompt
+    messages.append({"role": "system", "content": ""})
+
+    # Define help text
+    help_text = """
+    Available commands:
+    /help or /? - Show this help message
+    /bye - Exit the chat
+    /set system <prompt> - Set the system prompt
+    /set context <size> - Set context window size (default: 4096)
+    /clear or /cls - Clear the terminal screen
+    /reset - Reset all settings
+    """
+
+    while True:
+        # Get user input
+        user_input = input("\n> ")
+
+        # Handle commands
+        if user_input.startswith("/"):
+            cmd_parts = user_input.split(maxsplit=2)
+            cmd = cmd_parts[0].lower()
+
+            if cmd == "/bye" or user_input.lower() in ["exit", "quit"]:
+                console.print("[yellow]Goodbye![/yellow]")
+                break
+
+            elif cmd == "/help" or cmd == "/?":
+                console.print(help_text)
+                continue
+
+            elif cmd == "/clear" or cmd == "/cls":
+                # Do not clear history, just clear the terminal screen
+                import os
+                os.system('cls' if os.name == 'nt' else 'clear')
+                console.print(f"[bold green]Chat with {model_name}. Type '/help' for available commands or '/bye' to exit.[/bold green]")
+                console.print("[yellow]Screen cleared. Chat history preserved.[/yellow]")
+                continue
+
+            elif cmd == "/reset":
+                messages = [{"role": "system", "content": ""}]
+                system_prompt = None
+                console.print("[yellow]All settings reset.[/yellow]")
+                continue
+
+            elif cmd == "/set" and len(cmd_parts) >= 2:
+                if len(cmd_parts) < 3:
+                    console.print("[red]Error: Missing value for setting[/red]")
+                    continue
+
+                setting = cmd_parts[1].lower()
+                value = cmd_parts[2]
+
+                if setting == "system":
+                    # Remove quotes if present
+                    if value.startswith('"') and value.endswith('"'):
+                        value = value[1:-1]
+
+                    system_prompt = value
+                    # Update system message
+                    if messages and messages[0].get("role") == "system":
+                        messages[0]["content"] = system_prompt
+                    else:
+                        # Clear messages and add system prompt
+                        messages = [{"role": "system", "content": system_prompt}]
+
+                    # Print confirmation that it's been applied
+                    console.print(f"[yellow]System prompt set to:[/yellow]")
+                    console.print(f"[cyan]\"{system_prompt}\"[/cyan]")
+                    console.print(f"[green]System prompt applied. Next responses will follow this instruction.[/green]")
+
+                    # Force a test message to ensure the system prompt is applied
+                    test_messages = messages.copy()
+                    test_messages.append({"role": "user", "content": "Say 'System prompt active'."})
+
+                    # Test if the system prompt is working
+                    console.print("[dim]Testing system prompt...[/dim]")
+                    response = llm.create_chat_completion(
+                        messages=test_messages,
+                        stream=False,
+                        max_tokens=20
+                    )
+                    console.print("[dim]System prompt test complete.[/dim]")
+                elif setting == "context":
+                    try:
+                        context_size = int(value)
+                        # Reload the model with new context size
+                        console.print(f"[yellow]Reloading model with context size: {context_size}...[/yellow]")
+                        llm.load_model(n_ctx=context_size, verbose=False)
+                        console.print(f"[green]Context size set to: {context_size}[/green]")
+                    except ValueError:
+                        console.print(f"[red]Invalid context size: {value}. Must be an integer.[/red]")
+                else:
+                    console.print(f"[red]Unknown setting: {setting}[/red]")
+                continue
+            else:
+                console.print(f"[red]Unknown command: {cmd}[/red]")
+                continue
+
+        # Add user message to history
+        messages.append({"role": "user", "content": user_input})
+
+        # Generate response
+        console.print("\n")  # Add extra spacing between user input and response
+
+        # Use a buffer to collect the response
+        response_buffer = ""
+
+        def print_token(token):
+            nonlocal response_buffer
+            response_buffer += token
+            console.print(token, end="", highlight=False)
+
+        llm.stream_chat_completion(
+            messages=messages,
+            callback=print_token,
+        )
+
+        # Get the full response to add to history
+        response = llm.create_chat_completion(
+            messages=messages,
+            stream=False,
+        )
+
+        assistant_message = response["choices"][0]["message"]["content"]
+        messages.append({"role": "assistant", "content": assistant_message})
+
+        # Add extra spacing after the response
+        console.print("")
+
+@app.command("version")
+def version() -> None:
+    """
+    Show version information.
+    """
+    from webscout.Local import __version__
+    console.print(f"[bold]webscout.Local[/bold] version [cyan]{__version__}[/cyan]")
+    console.print("A llama-cpp-python based LLM serving tool")
+
+if __name__ == "__main__":
+    app()

webscout/Local/config.py

@@ -0,0 +1,75 @@
+"""
+Configuration management for webscout
+"""
+
+import os
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+
+# Default configuration
+default_config: Dict[str, Any] = {
+    "models_dir": "~/.webscout/models",
+    "api_host": "127.0.0.1",
+    "api_port": 8000,
+    "default_context_length": 4096,
+    "default_gpu_layers": -1,  # -1 means use all available GPU layers
+}
+
+class Config:
+    """
+    Configuration manager for webscout.
+    Handles loading, saving, and accessing configuration values.
+    """
+    config_dir: Path
+    config_file: Path
+    models_dir: Path
+    config: Dict[str, Any]
+
+    def __init__(self) -> None:
+        self.config_dir = Path(os.path.expanduser("~/.webscout"))
+        self.config_file = self.config_dir / "config.json"
+        self.models_dir = Path(os.path.expanduser(default_config["models_dir"]))
+        self._ensure_dirs()
+        self._load_config()
+
+    def _ensure_dirs(self) -> None:
+        """Ensure configuration and models directories exist."""
+        self.config_dir.mkdir(exist_ok=True, parents=True)
+        self.models_dir.mkdir(exist_ok=True, parents=True)
+
+    def _load_config(self) -> None:
+        """Load configuration from file or create default."""
+        if not self.config_file.exists():
+            self._save_config(default_config)
+            self.config = default_config.copy()
+        else:
+            with open(self.config_file, "r") as f:
+                self.config = json.load(f)
+
+    def _save_config(self, config: Dict[str, Any]) -> None:
+        """Save configuration to file."""
+        with open(self.config_file, "w") as f:
+            json.dump(config, f, indent=2)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get configuration value by key."""
+        return self.config.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        """Set configuration value by key."""
+        self.config[key] = value
+        self._save_config(self.config)
+
+    def get_model_path(self, model_name: str) -> Path:
+        """Get the path to a model directory by model name."""
+        return self.models_dir / model_name
+
+    def list_models(self) -> List[str]:
+        """List all downloaded model names."""
+        if not self.models_dir.exists():
+            return []
+        return [d.name for d in self.models_dir.iterdir() if d.is_dir() and ":" not in d.name]
+
+# Global configuration instance
+config: Config = Config()

webscout/Local/llm.py

@@ -0,0 +1,188 @@
+"""
+LLM interface for webscout.Local using llama-cpp-python
+"""
+
+from typing import Dict, Any, List, Optional, Union, Generator, Callable
+
+from llama_cpp import Llama
+from rich.console import Console
+
+from .config import config
+from .model_manager import ModelManager
+
+console = Console()
+
+class LLMInterface:
+    """
+    Interface for LLM models using llama-cpp-python.
+    Provides methods for loading models and generating completions or chat responses.
+    """
+    model_name: str
+    model_manager: ModelManager
+    model_path: Optional[str]
+    llm: Optional[Llama]
+
+    def __init__(self, model_name: str) -> None:
+        """
+        Initialize the LLM interface.
+        Args:
+            model_name (str): Name of the model to load.
+        Raises:
+            ValueError: If the model is not found locally.
+        """
+        self.model_name = model_name
+        self.model_manager = ModelManager()
+        self.model_path = self.model_manager.get_model_path(model_name)
+        if not self.model_path:
+            raise ValueError(f"Model {model_name} not found. Please download it first.")
+        self.llm = None
+
+    def load_model(self, n_gpu_layers: Optional[int] = None, n_ctx: Optional[int] = None, verbose: bool = False) -> None:
+        """
+        Load the model into memory.
+        Args:
+            n_gpu_layers (Optional[int]): Number of layers to offload to GPU (-1 for all).
+            n_ctx (Optional[int]): Context size.
+            verbose (bool): Whether to show verbose output.
+        Raises:
+            ValueError: If model loading fails.
+        """
+        if n_gpu_layers is None:
+            n_gpu_layers = config.get("default_gpu_layers", -1)
+        if n_ctx is None:
+            n_ctx = config.get("default_context_length", 4096)
+        console.print(f"[bold blue]Loading model {self.model_name}...[/bold blue]")
+        try:
+            self.llm = Llama(
+                model_path=self.model_path,
+                n_gpu_layers=n_gpu_layers,
+                n_ctx=n_ctx,
+                verbose=verbose
+            )
+            console.print(f"[bold green]Model {self.model_name} loaded successfully[/bold green]")
+        except Exception as e:
+            raise ValueError(f"Failed to load model from file: {self.model_path}\n{str(e)}")
+
+    def create_completion(
+        self,
+        prompt: str,
+        max_tokens: int = 256,
+        temperature: float = 0.7,
+        top_p: float = 0.95,
+        stream: bool = False,
+        stop: Optional[List[str]] = None,
+    ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]:
+        """
+        Create a completion for the given prompt.
+        Args:
+            prompt (str): The prompt to complete.
+            max_tokens (int): Maximum number of tokens to generate.
+            temperature (float): Sampling temperature.
+            top_p (float): Top-p sampling.
+            stream (bool): Whether to stream the response.
+            stop (Optional[List[str]]): List of strings to stop generation when encountered.
+        Returns:
+            Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]: Completion result or generator for streaming.
+        """
+        if self.llm is None:
+            self.load_model()
+        if stream:
+            return self.llm.create_completion(
+                prompt=prompt,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=True,
+                stop=stop or [],
+            )
+        else:
+            return self.llm.create_completion(
+                prompt=prompt,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=False,
+                stop=stop or [],
+            )
+
+    def create_chat_completion(
+        self,
+        messages: List[Dict[str, str]],
+        max_tokens: int = 256,
+        temperature: float = 0.7,
+        top_p: float = 0.95,
+        stream: bool = False,
+        stop: Optional[List[str]] = None,
+    ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]:
+        """
+        Create a chat completion for the given messages.
+        Args:
+            messages (List[Dict[str, str]]): List of chat messages.
+            max_tokens (int): Maximum number of tokens to generate.
+            temperature (float): Sampling temperature.
+            top_p (float): Top-p sampling.
+            stream (bool): Whether to stream the response.
+            stop (Optional[List[str]]): List of strings to stop generation when encountered.
+        Returns:
+            Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]: Chat completion result or generator for streaming.
+        """
+        if self.llm is None:
+            self.load_model()
+        processed_messages: List[Dict[str, str]] = messages.copy()
+        system_messages = [m for m in processed_messages if m.get("role") == "system"]
+        non_system_messages = [m for m in processed_messages if m.get("role") != "system"]
+        if system_messages:
+            processed_messages = [system_messages[0]] + non_system_messages
+        else:
+            processed_messages = non_system_messages
+        if stream:
+            return self.llm.create_chat_completion(
+                messages=processed_messages,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=True,
+                stop=stop or [],
+            )
+        else:
+            return self.llm.create_chat_completion(
+                messages=processed_messages,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=False,
+                stop=stop or [],
+            )
+
+    def stream_chat_completion(
+        self,
+        messages: List[Dict[str, str]],
+        callback: Callable[[str], None],
+        max_tokens: int = 256,
+        temperature: float = 0.7,
+        top_p: float = 0.95,
+        stop: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Stream a chat completion with a callback for each token.
+        Args:
+            messages (List[Dict[str, str]]): List of chat messages.
+            callback (Callable[[str], None]): Function to call with each token.
+            max_tokens (int): Maximum number of tokens to generate.
+            temperature (float): Sampling temperature.
+            top_p (float): Top-p sampling.
+            stop (Optional[List[str]]): List of strings to stop generation when encountered.
+        """
+        stream = self.create_chat_completion(
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=True,
+            stop=stop,
+        )
+        for chunk in stream:
+            if "choices" in chunk and len(chunk["choices"]) > 0:
+                if "delta" in chunk["choices"][0] and "content" in chunk["choices"][0]["delta"]:
+                    content = chunk["choices"][0]["delta"]["content"]
+                    callback(content)