webscout 8.2.4__py3-none-any.whl → 8.2.5__py3-none-any.whl

webscout/Local/llm.py

@@ -1,287 +0,0 @@
-"""
-LLM interface for webscout.Local using llama-cpp-python
-"""
-
-from typing import Dict, Any, List, Optional, Union, Generator, Callable
-
-from llama_cpp import Llama
-from rich.console import Console
-
-from .config import config
-from .model_manager import ModelManager
-
-console = Console()
-
-class LLMInterface:
-    """
-    Interface for LLM models using llama-cpp-python.
-    Provides methods for loading models and generating completions or chat responses.
-    """
-    model_name: str
-    model_manager: ModelManager
-    model_path: Optional[str]
-    llm: Optional[Llama]
-
-    def __init__(self, model_name: str) -> None:
-        """
-        Initialize the LLM interface.
-        Args:
-            model_name (str): Name of the model to load.
-        Raises:
-            ValueError: If the model is not found locally.
-        """
-        self.model_name = model_name
-        self.model_manager = ModelManager()
-        self.model_path = self.model_manager.get_model_path(model_name)
-        if not self.model_path:
-            raise ValueError(f"Model {model_name} not found. Please download it first.")
-        self.llm = None
-
-    def load_model(
-        self,
-        n_gpu_layers: Optional[int] = None,
-        n_ctx: Optional[int] = None,
-        verbose: bool = False,
-        n_threads: Optional[int] = None,
-        n_batch: Optional[int] = None,
-        use_mlock: bool = False,
-        use_mmap: bool = True,
-        rope_freq_base: Optional[float] = None,
-        rope_freq_scale: Optional[float] = None,
-        low_vram: bool = False,
-    ) -> None:
-        """
-        Load the model into memory.
-        Args:
-            n_gpu_layers (Optional[int]): Number of layers to offload to GPU (-1 for all).
-            n_ctx (Optional[int]): Context size.
-            verbose (bool): Whether to show verbose output.
-            n_threads (Optional[int]): Number of threads to use.
-            n_batch (Optional[int]): Batch size for prompt processing.
-            use_mlock (bool): Whether to use mlock to keep model in memory.
-            use_mmap (bool): Whether to use memory mapping for the model.
-            rope_freq_base (Optional[float]): RoPE base frequency.
-            rope_freq_scale (Optional[float]): RoPE frequency scaling factor.
-            low_vram (bool): Whether to optimize for low VRAM usage.
-        Raises:
-            ValueError: If model loading fails.
-        """
-        # If model is already loaded, check if we need to reload with different parameters
-        if self.llm is not None:
-            if n_ctx is not None and hasattr(self.llm, 'n_ctx') and self.llm.n_ctx != n_ctx:
-                # Need to reload with new context size
-                self.llm = None
-            else:
-                # Model already loaded with compatible parameters
-                return
-
-        if n_gpu_layers is None:
-            n_gpu_layers = config.get("default_gpu_layers", -1)
-        if n_ctx is None:
-            n_ctx = config.get("default_context_length", 4096)
-
-        # Determine number of threads if not specified
-        if n_threads is None:
-            import multiprocessing
-            n_threads = max(1, multiprocessing.cpu_count() // 2)
-
-        console.print(f"[bold blue]Loading model {self.model_name}...[/bold blue]")
-        try:
-            self.llm = Llama(
-                model_path=self.model_path,
-                n_gpu_layers=n_gpu_layers,
-                n_ctx=n_ctx,
-                verbose=verbose,
-                n_threads=n_threads,
-                n_batch=n_batch or 512,
-                use_mlock=use_mlock,
-                use_mmap=use_mmap,
-                rope_freq_base=rope_freq_base,
-                rope_freq_scale=rope_freq_scale,
-                low_vram=low_vram,
-            )
-
-            console.print(f"[bold green]Model {self.model_name} loaded successfully[/bold green]")
-            if verbose:
-                console.print(f"[dim]Using {n_threads} threads, context size: {n_ctx}[/dim]")
-                if n_gpu_layers and n_gpu_layers > 0:
-                    console.print(f"[dim]GPU acceleration: {n_gpu_layers} layers offloaded to GPU[/dim]")
-        except Exception as e:
-            raise ValueError(f"Failed to load model from file: {self.model_path}\n{str(e)}")
-
-    def create_completion(
-        self,
-        prompt: str,
-        max_tokens: int = 256,
-        temperature: float = 0.7,
-        top_p: float = 0.95,
-        stream: bool = False,
-        stop: Optional[List[str]] = None,
-        suffix: Optional[str] = None,
-        images: Optional[List[str]] = None,
-        system: Optional[str] = None,
-        template: Optional[str] = None,
-        context: Optional[List[int]] = None,
-        raw: bool = False,
-        format: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]:
-        """
-        Create a completion for the given prompt.
-        Args:
-            prompt (str): The prompt to complete.
-            max_tokens (int): Maximum number of tokens to generate.
-            temperature (float): Sampling temperature.
-            top_p (float): Top-p sampling.
-            stream (bool): Whether to stream the response.
-            stop (Optional[List[str]]): List of strings to stop generation when encountered.
-        Returns:
-            Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]: Completion result or generator for streaming.
-        """
-        if self.llm is None:
-            self.load_model()
-        if stream:
-            return self.llm.create_completion(
-                prompt=prompt,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
-                stream=True,
-                stop=stop or [],
-            )
-        else:
-            return self.llm.create_completion(
-                prompt=prompt,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
-                stream=False,
-                stop=stop or [],
-            )
-
-    def create_chat_completion(
-        self,
-        messages: List[Dict[str, Any]],
-        max_tokens: int = 256,
-        temperature: float = 0.7,
-        top_p: float = 0.95,
-        stream: bool = False,
-        stop: Optional[List[str]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        format: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]:
-        """
-        Create a chat completion for the given messages.
-        Args:
-            messages (List[Dict[str, str]]): List of chat messages.
-            max_tokens (int): Maximum number of tokens to generate.
-            temperature (float): Sampling temperature.
-            top_p (float): Top-p sampling.
-            stream (bool): Whether to stream the response.
-            stop (Optional[List[str]]): List of strings to stop generation when encountered.
-        Returns:
-            Union[Dict[str, Any], Generator[Dict[str, Any], None, None]]: Chat completion result or generator for streaming.
-        """
-        if self.llm is None:
-            self.load_model()
-        processed_messages: List[Dict[str, str]] = messages.copy()
-        system_messages = [m for m in processed_messages if m.get("role") == "system"]
-        non_system_messages = [m for m in processed_messages if m.get("role") != "system"]
-        if system_messages:
-            processed_messages = [system_messages[0]] + non_system_messages
-        else:
-            processed_messages = non_system_messages
-        if stream:
-            return self.llm.create_chat_completion(
-                messages=processed_messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
-                stream=True,
-                stop=stop or [],
-            )
-        else:
-            return self.llm.create_chat_completion(
-                messages=processed_messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
-                stream=False,
-                stop=stop or [],
-            )
-
-    def stream_chat_completion(
-        self,
-        messages: List[Dict[str, Any]],
-        callback: Callable[[str], None],
-        max_tokens: int = 256,
-        temperature: float = 0.7,
-        top_p: float = 0.95,
-        stop: Optional[List[str]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        format: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> None:
-        """
-        Stream a chat completion with a callback for each token.
-        Args:
-            messages (List[Dict[str, Any]]): List of chat messages.
-            callback (Callable[[str], None]): Function to call with each token.
-            max_tokens (int): Maximum number of tokens to generate.
-            temperature (float): Sampling temperature.
-            top_p (float): Top-p sampling.
-            stop (Optional[List[str]]): List of strings to stop generation when encountered.
-            tools (Optional[List[Dict[str, Any]]]): List of tools for function calling.
-            format (Optional[Union[str, Dict[str, Any]]]): Format for structured output.
-        """
-        stream = self.create_chat_completion(
-            messages=messages,
-            max_tokens=max_tokens,
-            temperature=temperature,
-            top_p=top_p,
-            stream=True,
-            stop=stop,
-        )
-        for chunk in stream:
-            if "choices" in chunk and len(chunk["choices"]) > 0:
-                if "delta" in chunk["choices"][0] and "content" in chunk["choices"][0]["delta"]:
-                    content = chunk["choices"][0]["delta"]["content"]
-                    callback(content)
-
-    def create_embeddings(
-        self,
-        input: Union[str, List[str]],
-        truncate: bool = True,
-    ) -> Dict[str, Any]:
-        """
-        Generate embeddings for the given input.
-        Args:
-            input (Union[str, List[str]]): Text or list of texts to generate embeddings for.
-            truncate (bool): Whether to truncate the input to fit within context length.
-        Returns:
-            Dict[str, Any]: Embeddings response.
-        """
-        if self.llm is None:
-            self.load_model()
-
-        # Convert input to list if it's a string
-        if isinstance(input, str):
-            input_texts = [input]
-        else:
-            input_texts = input
-
-        # Generate embeddings for each input text
-        embeddings = []
-        for text in input_texts:
-            # Use llama-cpp-python's embedding method
-            embedding = self.llm.embed(text)
-            embeddings.append(embedding)
-
-        # Create response
-        response = {
-            "model": self.model_name,
-            "embeddings": embeddings,
-            "total_duration": 0,  # Could be improved with actual timing
-            "load_duration": 0,   # Could be improved with actual timing
-            "prompt_eval_count": len(input_texts)
-        }
-
-        return response

webscout/Local/model_manager.py

@@ -1,253 +0,0 @@
-"""
-Model management for webscout.local
-"""
-
-import os
-import json
-import datetime
-from pathlib import Path
-from typing import Dict, Any, Optional, List, Tuple
-import shutil
-
-from rich.console import Console
-from rich.prompt import Prompt
-from huggingface_hub import hf_hub_download, HfFileSystem
-
-from .config import config
-
-console = Console()
-
-class ModelManager:
-    """
-    Manager for downloading and managing models.
-    Handles model download, listing, removal, and path resolution.
-    """
-    models_dir: Path
-
-    def __init__(self) -> None:
-        self.models_dir = config.models_dir
-
-    def parse_model_string(self, model_string: str) -> Tuple[str, Optional[str]]:
-        """
-        Parse a model string in the format 'repo_id:filename' or just 'repo_id'.
-        Args:
-            model_string (str): The model string to parse.
-        Returns:
-            Tuple[str, Optional[str]]: (repo_id, filename)
-        """
-        if ":" in model_string:
-            repo_id, filename = model_string.split(":", 1)
-            return repo_id, filename
-        else:
-            return model_string, None
-
-    def list_repo_gguf_files(self, repo_id: str) -> List[str]:
-        """
-        List all GGUF files in a repository.
-        Args:
-            repo_id (str): The Hugging Face repository ID.
-        Returns:
-            List[str]: List of filenames.
-        """
-        fs = HfFileSystem()
-        try:
-            files = fs.ls(repo_id, detail=False)
-            gguf_files = [os.path.basename(f) for f in files if f.endswith(".gguf")]
-            return gguf_files
-        except Exception as e:
-            console.print(f"[bold red]Error listing files in repository {repo_id}: {str(e)}[/bold red]")
-            return []
-
-    def select_file_interactive(self, repo_id: str) -> Optional[str]:
-        """
-        Interactively select a file from a repository.
-        Args:
-            repo_id (str): The Hugging Face repository ID.
-        Returns:
-            Optional[str]: Selected filename or None if cancelled.
-        """
-        gguf_files = self.list_repo_gguf_files(repo_id)
-        if not gguf_files:
-            console.print(f"[bold red]No GGUF files found in repository {repo_id}[/bold red]")
-            return None
-        console.print(f"[bold blue]Available GGUF files in {repo_id}:[/bold blue]")
-        for i, filename in enumerate(gguf_files):
-            console.print(f"  [{i+1}] {filename}")
-        choice = Prompt.ask(
-            "Select a file to download (number or filename)",
-            default="1"
-        )
-        try:
-            idx = int(choice) - 1
-            if 0 <= idx < len(gguf_files):
-                return gguf_files[idx]
-        except ValueError:
-            if choice in gguf_files:
-                return choice
-        console.print(f"[bold red]Invalid selection: {choice}[/bold red]")
-        return None
-
-    def download_model(self, model_string: str, filename: Optional[str] = None) -> Tuple[str, Path]:
-        """
-        Download a model from Hugging Face Hub.
-        Args:
-            model_string (str): The model string in format 'repo_id' or 'repo_id:filename'.
-            filename (Optional[str]): Specific filename to download, overrides filename in model_string.
-        Returns:
-            Tuple[str, Path]: (model_name, model_path)
-        """
-        repo_id, file_from_string = self.parse_model_string(model_string)
-        filename = filename or file_from_string
-        model_name = repo_id.split("/")[-1] if "/" in repo_id else repo_id
-        model_dir = config.get_model_path(model_name)
-        model_dir.mkdir(exist_ok=True, parents=True)
-        model_info: Dict[str, Any] = {
-            "repo_id": repo_id,
-            "name": model_name,
-            "downloaded_at": datetime.datetime.now().isoformat(),
-        }
-        with open(model_dir / "info.json", "w") as f:
-            json.dump(model_info, f, indent=2)
-        if not filename:
-            console.print(f"[yellow]No filename provided, searching for GGUF files in {repo_id}...[/yellow]")
-            filename = self.select_file_interactive(repo_id)
-            if not filename:
-                raise ValueError(f"No GGUF file selected from repository {repo_id}")
-            console.print(f"[green]Selected GGUF file: {filename}[/green]")
-        console.print(f"[bold blue]Downloading {filename} from {repo_id}...[/bold blue]")
-        try:
-            model_path = hf_hub_download(
-                repo_id=repo_id,
-                filename=filename,
-                local_dir=model_dir,
-            )
-        except Exception as e:
-            console.print(f"[bold red]Error downloading file: {str(e)}[/bold red]")
-            raise
-        console.print(f"[bold green]Model downloaded to {model_path}[/bold green]")
-        model_info["filename"] = filename
-        model_info["path"] = str(model_path)
-        with open(model_dir / "info.json", "w") as f:
-            json.dump(model_info, f, indent=2)
-        return model_name, Path(model_path)
-
-    def get_model_info(self, model_name: str) -> Optional[Dict[str, Any]]:
-        """
-        Get information about a downloaded model.
-        Args:
-            model_name (str): Name of the model.
-        Returns:
-            Optional[Dict[str, Any]]: Model info dict or None if not found.
-        """
-        model_dir = config.get_model_path(model_name)
-        info_file = model_dir / "info.json"
-        if not info_file.exists():
-            return None
-        with open(info_file, "r") as f:
-            return json.load(f)
-
-    def list_models(self) -> List[Dict[str, Any]]:
-        """
-        List all downloaded models with their information.
-        Returns:
-            List[Dict[str, Any]]: List of model info dicts.
-        """
-        models: List[Dict[str, Any]] = []
-        seen_paths: set = set()
-        if not config.models_dir.exists():
-            return []
-        model_dirs = [d for d in config.models_dir.iterdir() if d.is_dir()]
-        for model_dir in model_dirs:
-            if ":" in model_dir.name:
-                continue
-            info_file = model_dir / "info.json"
-            if info_file.exists():
-                try:
-                    with open(info_file, "r") as f:
-                        info = json.load(f)
-                    if "path" in info and info["path"] in seen_paths:
-                        continue
-                    if "path" in info:
-                        seen_paths.add(info["path"])
-                    models.append(info)
-                except Exception:
-                    pass
-        return models
-
-    def remove_model(self, model_name: str) -> bool:
-        """
-        Remove a downloaded model.
-        Args:
-            model_name (str): Name of the model to remove.
-        Returns:
-            bool: True if removed, False if not found.
-        """
-        model_dir = config.get_model_path(model_name)
-        if not model_dir.exists():
-            return False
-        shutil.rmtree(model_dir)
-        return True
-
-    def get_model_path(self, model_name: str) -> Optional[str]:
-        """
-        Get the path to a model file.
-        Args:
-            model_name (str): Name or filename of the model.
-        Returns:
-            Optional[str]: Path to the model file or None if not found.
-        """
-        info = self.get_model_info(model_name)
-        if not info or "path" not in info:
-            for model_info in self.list_models():
-                if model_info.get("filename") == model_name:
-                    return model_info.get("path")
-            return None
-        return info["path"]
-
-    def copy_model(self, source_model: str, destination_model: str) -> bool:
-        """
-        Copy a model to a new name.
-        Args:
-            source_model (str): Name of the source model.
-            destination_model (str): Name for the destination model.
-        Returns:
-            bool: True if copied successfully, False otherwise.
-        """
-        # Get source model info
-        source_info = self.get_model_info(source_model)
-        if not source_info or "path" not in source_info:
-            console.print(f"[bold red]Source model {source_model} not found[/bold red]")
-            return False
-
-        # Create destination directory
-        dest_dir = config.get_model_path(destination_model)
-        dest_dir.mkdir(exist_ok=True, parents=True)
-
-        # Copy the model file
-        source_path = Path(source_info["path"])
-        dest_path = dest_dir / source_path.name
-
-        try:
-            console.print(f"[bold blue]Copying model from {source_path} to {dest_path}...[/bold blue]")
-            shutil.copy2(source_path, dest_path)
-
-            # Create info file for the destination model
-            dest_info = source_info.copy()
-            dest_info["name"] = destination_model
-            dest_info["path"] = str(dest_path)
-            dest_info["copied_from"] = source_model
-            dest_info["copied_at"] = datetime.datetime.now().isoformat()
-
-            with open(dest_dir / "info.json", "w") as f:
-                json.dump(dest_info, f, indent=2)
-
-            console.print(f"[bold green]Model copied successfully to {dest_path}[/bold green]")
-            return True
-        except Exception as e:
-            console.print(f"[bold red]Error copying model: {str(e)}[/bold red]")
-            # Clean up if there was an error
-            if dest_path.exists():
-                dest_path.unlink()
-            if dest_dir.exists():
-                shutil.rmtree(dest_dir)
-            return False