isa-model 0.0.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

isa_model/core/model_router.py

@@ -1,226 +0,0 @@
-import random
-import time
-from typing import Dict, List, Any, Optional, Callable
-import threading
-
-class ModelRouter:
-    """
-    Routes requests to appropriate model instances based on different strategies:
-    - Weighted round-robin
-    - Least connections
-    - Least response time
-    - Dynamic load balancing
-    """
-    
-    def __init__(self, registry):
-        self.registry = registry
-        self.model_stats = {}  # Track performance metrics for each model
-        self.lock = threading.RLock()
-        
-        # Maps model_type -> list of model_ids of that type
-        self.model_type_mapping = {}
-        
-        # Maps routing_strategy_name -> routing_function
-        self.routing_strategies = {
-            "round_robin": self._route_round_robin,
-            "weighted_random": self._route_weighted_random,
-            "least_connections": self._route_least_connections,
-            "least_response_time": self._route_least_response_time,
-            "dynamic_load": self._route_dynamic_load
-        }
-        
-        # Round-robin counters for each model type
-        self.rr_counters = {}
-        
-    def register_model_type(self, model_type: str, model_ids: List[str], weights: Optional[List[float]] = None):
-        """Register models of a specific type with optional weights"""
-        with self.lock:
-            self.model_type_mapping[model_type] = model_ids
-            
-            # Initialize stats for each model
-            for i, model_id in enumerate(model_ids):
-                weight = weights[i] if weights and i < len(weights) else 1.0
-                
-                if model_id not in self.model_stats:
-                    self.model_stats[model_id] = {
-                        "active_connections": 0,
-                        "total_requests": 0,
-                        "avg_response_time": 0,
-                        "weight": weight,
-                        "last_used": 0
-                    }
-                else:
-                    # Update weight if model already exists
-                    self.model_stats[model_id]["weight"] = weight
-            
-            # Initialize round-robin counter
-            self.rr_counters[model_type] = 0
-    
-    def route_request(self, model_type: str, routing_strategy: str = "round_robin") -> Optional[str]:
-        """
-        Route a request to an appropriate model of the given type
-        
-        Args:
-            model_type: Type of model needed
-            routing_strategy: Strategy to use for routing
-            
-        Returns:
-            model_id: ID of the model to use, or None if no models available
-        """
-        with self.lock:
-            if model_type not in self.model_type_mapping:
-                return None
-                
-            if not self.model_type_mapping[model_type]:
-                return None
-                
-            # Get the routing function
-            routing_func = self.routing_strategies.get(routing_strategy, self._route_round_robin)
-            
-            # Route the request
-            model_id = routing_func(model_type)
-            
-            if model_id:
-                # Update stats
-                self.model_stats[model_id]["active_connections"] += 1
-                self.model_stats[model_id]["total_requests"] += 1
-                self.model_stats[model_id]["last_used"] = time.time()
-                
-            return model_id
-    
-    def release_connection(self, model_id: str, response_time: float = None):
-        """Release a connection and update stats"""
-        with self.lock:
-            if model_id in self.model_stats:
-                stats = self.model_stats[model_id]
-                stats["active_connections"] = max(0, stats["active_connections"] - 1)
-                
-                # Update average response time
-                if response_time is not None:
-                    old_avg = stats["avg_response_time"]
-                    total_req = stats["total_requests"]
-                    
-                    if total_req > 0:
-                        # Weighted average
-                        stats["avg_response_time"] = (old_avg * (total_req - 1) + response_time) / total_req
-    
-    # Routing strategies
-    def _route_round_robin(self, model_type: str) -> Optional[str]:
-        """Simple round-robin routing"""
-        models = self.model_type_mapping.get(model_type, [])
-        if not models:
-            return None
-            
-        # Get and increment counter
-        counter = self.rr_counters[model_type]
-        self.rr_counters[model_type] = (counter + 1) % len(models)
-        
-        return models[counter]
-    
-    def _route_weighted_random(self, model_type: str) -> Optional[str]:
-        """Weighted random selection based on configured weights"""
-        models = self.model_type_mapping.get(model_type, [])
-        if not models:
-            return None
-            
-        # Get weights
-        weights = [self.model_stats[model_id]["weight"] for model_id in models]
-        
-        # Weighted random selection
-        total = sum(weights)
-        r = random.uniform(0, total)
-        upto = 0
-        
-        for i, w in enumerate(weights):
-            upto += w
-            if upto >= r:
-                return models[i]
-                
-        # Fallback
-        return models[-1]
-    
-    def _route_least_connections(self, model_type: str) -> Optional[str]:
-        """Route to the model with the fewest active connections"""
-        models = self.model_type_mapping.get(model_type, [])
-        if not models:
-            return None
-            
-        # Find model with least connections
-        min_connections = float('inf')
-        selected_model = None
-        
-        for model_id in models:
-            connections = self.model_stats[model_id]["active_connections"]
-            if connections < min_connections:
-                min_connections = connections
-                selected_model = model_id
-                
-        return selected_model
-    
-    def _route_least_response_time(self, model_type: str) -> Optional[str]:
-        """Route to the model with the lowest average response time"""
-        models = self.model_type_mapping.get(model_type, [])
-        if not models:
-            return None
-            
-        # Find model with lowest response time
-        min_response_time = float('inf')
-        selected_model = None
-        
-        for model_id in models:
-            response_time = self.model_stats[model_id]["avg_response_time"]
-            # Skip models with no data yet
-            if response_time == 0:
-                continue
-                
-            if response_time < min_response_time:
-                min_response_time = response_time
-                selected_model = model_id
-                
-        # If no model has response time data, fall back to least connections
-        if selected_model is None:
-            return self._route_least_connections(model_type)
-            
-        return selected_model
-    
-    def _route_dynamic_load(self, model_type: str) -> Optional[str]:
-        """
-        Dynamic load balancing based on a combination of:
-        - Connection count
-        - Response time
-        - Recent usage
-        """
-        models = self.model_type_mapping.get(model_type, [])
-        if not models:
-            return None
-            
-        # Calculate a score for each model (lower is better)
-        best_score = float('inf')
-        selected_model = None
-        now = time.time()
-        
-        for model_id in models:
-            stats = self.model_stats[model_id]
-            
-            # Normalize each factor between 0 and 1
-            connections = stats["active_connections"]
-            conn_score = connections / (connections + 1)  # Approaches 1 as connections increase
-            
-            resp_time = stats["avg_response_time"]
-            # Max expected response time (adjust as needed)
-            max_resp_time = 5.0  
-            resp_score = min(1.0, resp_time / max_resp_time)
-            
-            # Time since last use (for distributing load)
-            recency = now - stats["last_used"] if stats["last_used"] > 0 else 60
-            recency_score = 1.0 - min(1.0, recency / 60.0)  # Unused for 60s approaches 0
-            
-            # Combined score (lower is better)
-            # Weights can be adjusted based on importance
-            score = (0.4 * conn_score) + (0.4 * resp_score) + (0.2 * recency_score)
-            
-            if score < best_score:
-                best_score = score
-                selected_model = model_id
-                
-        return selected_model

isa_model/core/resource_manager.py

@@ -1,202 +0,0 @@
-import time
-import threading
-import logging
-from typing import Dict, List, Any, Optional
-import psutil
-try:
-    import torch
-    import nvidia_smi
-    HAS_GPU = torch.cuda.is_available()
-except ImportError:
-    HAS_GPU = False
-
-logger = logging.getLogger(__name__)
-
-class ResourceManager:
-    """
-    Monitors system resources and manages model loading/unloading
-    to prevent resource exhaustion.
-    """
-    
-    def __init__(self, model_registry, monitoring_interval=30):
-        self.registry = model_registry
-        self.monitoring_interval = monitoring_interval  # seconds
-        
-        self.max_memory_percent = 90  # Maximum memory usage percentage
-        self.max_gpu_memory_percent = 90  # Maximum GPU memory usage percentage
-        
-        self._stop_event = threading.Event()
-        self._monitor_thread = None
-        
-        # Track resource usage over time
-        self.resource_history = {
-            "timestamps": [],
-            "cpu_percent": [],
-            "memory_percent": [],
-            "gpu_utilization": [],
-            "gpu_memory_percent": []
-        }
-        
-        # Initialize GPU monitoring if available
-        if HAS_GPU:
-            try:
-                nvidia_smi.nvmlInit()
-                self.gpu_count = torch.cuda.device_count()
-                logger.info(f"Initialized GPU monitoring with {self.gpu_count} devices")
-            except Exception as e:
-                logger.warning(f"Failed to initialize NVIDIA SMI: {str(e)}")
-                self.gpu_count = 0
-        else:
-            self.gpu_count = 0
-            
-        logger.info("Initialized ResourceManager")
-    
-    def start_monitoring(self):
-        """Start the resource monitoring thread"""
-        if self._monitor_thread is not None and self._monitor_thread.is_alive():
-            logger.warning("Resource monitoring already running")
-            return
-            
-        self._stop_event.clear()
-        self._monitor_thread = threading.Thread(
-            target=self._monitor_resources,
-            daemon=True
-        )
-        self._monitor_thread.start()
-        logger.info("Started resource monitoring thread")
-    
-    def stop_monitoring(self):
-        """Stop the resource monitoring thread"""
-        if self._monitor_thread is not None:
-            self._stop_event.set()
-            self._monitor_thread.join(timeout=5)
-            self._monitor_thread = None
-            logger.info("Stopped resource monitoring thread")
-    
-    def _monitor_resources(self):
-        """Monitor system resources in a loop"""
-        while not self._stop_event.is_set():
-            try:
-                # Get current resource usage
-                cpu_percent = psutil.cpu_percent(interval=1)
-                memory = psutil.virtual_memory()
-                memory_percent = memory.percent
-                
-                # GPU monitoring
-                gpu_utilization = 0
-                gpu_memory_percent = 0
-                
-                if self.gpu_count > 0:
-                    gpu_util_sum = 0
-                    gpu_mem_percent_sum = 0
-                    
-                    for i in range(self.gpu_count):
-                        try:
-                            handle = nvidia_smi.nvmlDeviceGetHandleByIndex(i)
-                            util = nvidia_smi.nvmlDeviceGetUtilizationRates(handle)
-                            mem_info = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)
-                            
-                            gpu_util_sum += util.gpu
-                            gpu_mem_percent = (mem_info.used / mem_info.total) * 100
-                            gpu_mem_percent_sum += gpu_mem_percent
-                        except Exception as e:
-                            logger.error(f"Error getting GPU {i} stats: {str(e)}")
-                    
-                    if self.gpu_count > 0:
-                        gpu_utilization = gpu_util_sum / self.gpu_count
-                        gpu_memory_percent = gpu_mem_percent_sum / self.gpu_count
-                
-                # Record history (keep last 60 samples)
-                now = time.time()
-                self.resource_history["timestamps"].append(now)
-                self.resource_history["cpu_percent"].append(cpu_percent)
-                self.resource_history["memory_percent"].append(memory_percent)
-                self.resource_history["gpu_utilization"].append(gpu_utilization)
-                self.resource_history["gpu_memory_percent"].append(gpu_memory_percent)
-                
-                # Trim history to last 60 samples
-                max_history = 60
-                if len(self.resource_history["timestamps"]) > max_history:
-                    for key in self.resource_history:
-                        self.resource_history[key] = self.resource_history[key][-max_history:]
-                
-                # Check if we need to unload models
-                self._check_resource_constraints(memory_percent, gpu_memory_percent)
-                
-                # Log current usage
-                logger.debug(
-                    f"Resource usage - CPU: {cpu_percent:.1f}%, Memory: {memory_percent:.1f}%, "
-                    f"GPU: {gpu_utilization:.1f}%, GPU Memory: {gpu_memory_percent:.1f}%"
-                )
-                
-                # Wait for next check
-                self._stop_event.wait(self.monitoring_interval)
-                
-            except Exception as e:
-                logger.error(f"Error in resource monitoring: {str(e)}")
-                # Wait a bit before retrying
-                self._stop_event.wait(5)
-    
-    def _check_resource_constraints(self, memory_percent, gpu_memory_percent):
-        """Check if we need to unload models due to resource constraints"""
-        # Check memory usage
-        if memory_percent > self.max_memory_percent:
-            logger.warning(
-                f"Memory usage ({memory_percent:.1f}%) exceeds threshold ({self.max_memory_percent}%). "
-                "Unloading least used model."
-            )
-            # This would trigger model unloading
-            # self.registry._evict_least_used_model()
-        
-        # Check GPU memory usage
-        if HAS_GPU and gpu_memory_percent > self.max_gpu_memory_percent:
-            logger.warning(
-                f"GPU memory usage ({gpu_memory_percent:.1f}%) exceeds threshold ({self.max_gpu_memory_percent}%). "
-                "Unloading least used model."
-            )
-            # This would trigger model unloading
-            # self.registry._evict_least_used_model()
-    
-    def get_resource_usage(self) -> Dict[str, Any]:
-        """Get current resource usage stats"""
-        try:
-            cpu_percent = psutil.cpu_percent(interval=0.1)
-            memory = psutil.virtual_memory()
-            memory_percent = memory.percent
-            
-            result = {
-                "cpu_percent": cpu_percent,
-                "memory_total_gb": memory.total / (1024**3),
-                "memory_available_gb": memory.available / (1024**3),
-                "memory_percent": memory_percent,
-                "gpus": []
-            }
-            
-            # GPU stats
-            if HAS_GPU:
-                for i in range(self.gpu_count):
-                    try:
-                        handle = nvidia_smi.nvmlDeviceGetHandleByIndex(i)
-                        name = nvidia_smi.nvmlDeviceGetName(handle)
-                        util = nvidia_smi.nvmlDeviceGetUtilizationRates(handle)
-                        mem_info = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)
-                        temp = nvidia_smi.nvmlDeviceGetTemperature(
-                            handle, nvidia_smi.NVML_TEMPERATURE_GPU
-                        )
-                        
-                        result["gpus"].append({
-                            "index": i,
-                            "name": name,
-                            "utilization_percent": util.gpu,
-                            "memory_total_gb": mem_info.total / (1024**3),
-                            "memory_used_gb": mem_info.used / (1024**3),
-                            "memory_percent": (mem_info.used / mem_info.total) * 100,
-                            "temperature_c": temp
-                        })
-                    except Exception as e:
-                        logger.error(f"Error getting GPU {i} stats: {str(e)}")
-            
-            return result
-        except Exception as e:
-            logger.error(f"Error getting resource usage: {str(e)}")
-            return {"error": str(e)}

isa_model/deployment/gpu_fp16_ds8/models/deepseek_r1/1/model.py

@@ -1,120 +0,0 @@
-import json
-import numpy as np
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import os 
-import triton_python_backend_utils as pb_utils
-
-class TritonPythonModel:
-    def initialize(self, args):
-        """初始化模型"""
-        self.model_config = json.loads(args['model_config'])
-        
-        # --- START: CORRECTED PATH LOGIC ---
-
-        # model_repository 是父目录, e.g., /models/deepseek_r1
-        model_repository = args['model_repository']
-        # model_version 是版本号, e.g., '1'
-        model_version = args['model_version']
-        
-        # 将它们拼接成指向模型文件的确切路径
-        model_path = os.path.join(model_repository, model_version)
-
-        print(f"Loading model from specific version path: {model_path}")
-
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            model_path,  # 从正确的版本目录加载
-            trust_remote_code=True
-        )
-        
-        self.model = AutoModelForCausalLM.from_pretrained(
-            model_path,  # 从正确的版本目录加载
-            torch_dtype=torch.bfloat16, 
-            device_map="gpu",           
-            trust_remote_code=True
-        )
-        
-        # --- END: CORRECTED PATH LOGIC ---
-        
-        # ... (您代码的其余部分保持不变) ...
-        output_config = pb_utils.get_output_config_by_name(
-            self.model_config, "OUTPUT_TEXT"
-        )
-        self.output_dtype = pb_utils.triton_string_to_numpy(
-            output_config['data_type']
-        )
-        
-        self.generation_config = {
-            'max_new_tokens': 512,
-            'temperature': 0.7,
-            'do_sample': True,
-            'top_p': 0.9,
-            'repetition_penalty': 1.1,
-            'pad_token_id': self.tokenizer.eos_token_id
-        }
-        
-        print("Model loaded successfully!")
-    
-    def execute(self, requests):
-        """执行推理"""
-        responses = []
-        
-        for request in requests:
-            # 获取输入文本
-            input_text = pb_utils.get_input_tensor_by_name(
-                request, "INPUT_TEXT"
-            ).as_numpy()
-            
-            # 解码输入文本
-            input_texts = [text.decode('utf-8') for text in input_text.flatten()]
-            
-            # 批量推理
-            output_texts = []
-            for text in input_texts:
-                try:
-                    # 编码输入
-                    inputs = self.tokenizer.encode(
-                        text, 
-                        return_tensors="pt"
-                    ).to(self.model.device)
-                    
-                    # 生成响应
-                    with torch.no_grad():
-                        outputs = self.model.generate(
-                            inputs,
-                            **self.generation_config
-                        )
-                    
-                    # 解码输出
-                    response = self.tokenizer.decode(
-                        outputs[0][inputs.shape[-1]:], 
-                        skip_special_tokens=True
-                    )
-                    
-                    output_texts.append(response)
-                    
-                except Exception as e:
-                    print(f"Error processing text: {e}")
-                    output_texts.append(f"Error: {str(e)}")
-            
-            # 准备输出
-            output_texts_np = np.array(
-                [[text.encode('utf-8')] for text in output_texts], 
-                dtype=object
-            )
-            
-            output_tensor = pb_utils.Tensor(
-                "OUTPUT_TEXT", 
-                output_texts_np.astype(self.output_dtype)
-            )
-            
-            response = pb_utils.InferenceResponse(
-                output_tensors=[output_tensor]
-            )
-            responses.append(response)
-        
-        return responses
-    
-    def finalize(self):
-        """清理资源"""
-        print("Cleaning up...")

isa_model/deployment/gpu_fp16_ds8/scripts/download_model.py

@@ -1,18 +0,0 @@
-from huggingface_hub import snapshot_download
-import os
-
-model_name = 'deepseek-ai/DeepSeek-R1-0528-Qwen3-8B'
-# 定义Triton模型仓库中该模型的版本路径
-local_model_path = os.path.join("models", "deepseek_r1", "1")
-
-print(f"开始下载模型 '{model_name}' 到 '{local_model_path}'...")
-
-# 使用 snapshot_download 下载整个模型仓库
-# 它会下载所有文件，包括.safetensors权重文件
-snapshot_download(
-    repo_id=model_name,
-    local_dir=local_model_path,
-    local_dir_use_symlinks=False,  
-)
-
-print("模型所有文件下载完成!")

isa_model/training/engine/llama_factory/__init__.py

@@ -1,39 +0,0 @@
-"""
-LlamaFactory engine for fine-tuning and reinforcement learning.
-
-This package provides interfaces for using LlamaFactory to:
-- Fine-tune models with various datasets
-- Perform reinforcement learning from human feedback (RLHF)
-- Support instruction tuning and preference optimization
-"""
-
-from .config import (
-    LlamaFactoryConfig,
-    SFTConfig,
-    RLConfig,
-    DPOConfig,
-    TrainingStrategy,
-    DatasetFormat,
-    create_default_config
-)
-from .trainer import LlamaFactoryTrainer
-from .rl import LlamaFactoryRL
-from .factory import LlamaFactory
-from .data_adapter import DataAdapter, AlpacaAdapter, ShareGPTAdapter, DataAdapterFactory
-
-__all__ = [
-    "LlamaFactoryTrainer",
-    "LlamaFactoryRL",
-    "LlamaFactoryConfig",
-    "SFTConfig",
-    "RLConfig",
-    "DPOConfig",
-    "TrainingStrategy",
-    "DatasetFormat",
-    "create_default_config",
-    "LlamaFactory",
-    "DataAdapter",
-    "AlpacaAdapter",
-    "ShareGPTAdapter",
-    "DataAdapterFactory"
-]