isa-model 0.3.91__py3-none-any.whl → 0.4.3__py3-none-any.whl

isa_model/training/annotation/tests/test_annotation_flow.py

@@ -1,109 +0,0 @@
-# test_annotation_flow.py
-import os 
-os.environ["ENV"] = "local"
-
-import asyncio
-from datetime import datetime
-from bson import ObjectId
-from app.services.llm_model.annotation.views.annotation_controller import AnnotationController
-from app.services.llm_model.annotation.processors.annotation_processor import AnnotationProcessor
-from app.services.llm_model.annotation.annotation_schema import (
-    AnnotationFeedback, 
-    RatingScale, 
-    AnnotationType,
-    AnnotationAspects,
-    BetterResponse
-)
-from app.config.config_manager import config_manager 
-
-async def setup_test_data():
-    """Setup initial test data in MongoDB"""
-    db = await config_manager.get_db('mongodb')
-    
-    # Create a test annotation
-    test_annotation = {
-        "_id": ObjectId(),
-        "project_name": "test_project",
-        "items": [{
-            "item_id": "test_item_1",
-            "input": {
-                "messages": [{
-                    "role": "user",
-                    "content": "What is the capital of France?"
-                }]
-            },
-            "output": {
-                "content": "The capital of France is Paris."
-            },
-            "status": "pending"
-        }],
-        "created_at": datetime.utcnow().isoformat()
-    }
-    
-    await db['annotations'].insert_one(test_annotation)
-    return test_annotation
-
-async def test_annotation_flow():
-    """Test the complete annotation flow"""
-    try:
-        # Initialize controllers
-        annotation_controller = AnnotationController()
-        annotation_processor = AnnotationProcessor()
-        
-        # Setup test data
-        test_data = await setup_test_data()
-        annotation_id = str(test_data["_id"])
-        item_id = test_data["items"][0]["item_id"]
-        
-        print("1. Created test annotation")
-        
-        # Create test feedback
-        feedback = AnnotationFeedback(
-            rating=RatingScale.EXCELLENT,
-            category=AnnotationType.ACCURACY,
-            aspects=AnnotationAspects(
-                factually_correct=True,
-                relevant=True,
-                harmful=False,
-                biased=False,
-                complete=True,
-                efficient=True
-            ),
-            better_response=BetterResponse(
-                content="Paris is the capital city of France, known for its iconic Eiffel Tower.",
-                reason="Added more context and detail"
-            ),
-            comment="Good response, but could be more detailed"
-        )
-        
-        # Submit annotation
-        result = await annotation_controller.submit_annotation(
-            annotation_id=annotation_id,
-            item_id=item_id,
-            feedback=feedback,
-            annotator_id="test_annotator"
-        )
-        
-        print("2. Submitted annotation:", result)
-        
-        # Process annotation queue
-        await annotation_processor.process_queue()
-        print("3. Processed annotation queue")
-        
-        # Verify dataset creation
-        db = await config_manager.get_db('mongodb')
-        datasets = await db['training_datasets'].find().to_list(length=10)
-        
-        print("\nCreated Datasets:")
-        for dataset in datasets:
-            print(f"- {dataset['name']} ({dataset['type']})")
-            print(f"  Status: {dataset['status']}")
-            print(f"  Examples: {dataset['stats']['total_examples']}")
-            
-    except Exception as e:
-        print(f"Error during test: {e}")
-
-if __name__ == "__main__":
-    # Run the test
-    print("Starting annotation flow test...")
-    asyncio.run(test_annotation_flow())

isa_model/training/annotation/tests/test_minio copy.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from minio import Minio
-import json
-import logging
-from io import BytesIO
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-def test_minio():
-    try:
-        # Get MinIO host from environment variable or use default
-        minio_host = os.getenv("MINIO_HOST", "localhost:9000")
-        logger.info(f"Using MinIO host: {minio_host}")
-        
-        # 1. Create MinIO client
-        logger.info("Creating MinIO client...")
-        client = Minio(
-            minio_host,
-            access_key="minioadmin",
-            secret_key="minioadmin",
-            secure=False
-        )
-        
-        # 2. Test bucket operations
-        bucket_name = "knowledge-files"  # Changed to match the actual bucket name
-        logger.info(f"Testing bucket operations with {bucket_name}...")
-        
-        # Create bucket if it doesn't exist
-        if not client.bucket_exists(bucket_name):
-            client.make_bucket(bucket_name)
-            logger.info(f"Created new bucket: {bucket_name}")
-        else:
-            logger.info(f"Using existing bucket: {bucket_name}")
-        
-        # Set bucket policy - allow all operations
-        policy = {
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Effect": "Allow",
-                    "Principal": {"AWS": ["*"]},
-                    "Action": [
-                        "s3:GetBucketLocation",
-                        "s3:ListBucket",
-                        "s3:ListBucketMultipartUploads"
-                    ],
-                    "Resource": [f"arn:aws:s3:::{bucket_name}"]
-                },
-                {
-                    "Effect": "Allow",
-                    "Principal": {"AWS": ["*"]},
-                    "Action": [
-                        "s3:AbortMultipartUpload",
-                        "s3:DeleteObject",
-                        "s3:GetObject",
-                        "s3:ListMultipartUploadParts",
-                        "s3:PutObject"
-                    ],
-                    "Resource": [f"arn:aws:s3:::{bucket_name}/*"]
-                }
-            ]
-        }
-        
-        try:
-            client.set_bucket_policy(bucket_name, json.dumps(policy))
-            logger.info("Set bucket policy successfully")
-        except Exception as e:
-            logger.warning(f"Failed to set bucket policy: {e}")
-        
-        # 3. Test file upload
-        source_file = "init-scripts/files/haley_system.txt"
-        object_name = "haley_system.txt"
-        
-        if os.path.exists(source_file):
-            # Get file size
-            file_size = os.path.getsize(source_file)
-            logger.info(f"Found source file: {source_file} (size: {file_size} bytes)")
-            
-            # Upload file
-            with open(source_file, 'rb') as file_data:
-                client.put_object(
-                    bucket_name,
-                    object_name,
-                    file_data,
-                    file_size,
-                    content_type="text/plain"
-                )
-            logger.info(f"Uploaded file: {object_name}")
-            
-            # 4. Test file download
-            data = client.get_object(bucket_name, object_name)
-            content = data.read().decode('utf-8')
-            logger.info(f"Successfully downloaded file. First 100 chars: {content[:100]}...")
-            
-            # 5. Verify file exists
-            stat = client.stat_object(bucket_name, object_name)
-            logger.info(f"File stats: {stat}")
-            
-            logger.info("Test completed successfully!")
-        else:
-            logger.error(f"Source file not found: {source_file}")
-            raise FileNotFoundError(f"Source file not found: {source_file}")
-        
-    except Exception as e:
-        logger.error(f"Test failed: {str(e)}")
-        raise
-
-if __name__ == "__main__":
-    test_minio() 

isa_model/training/annotation/tests/test_minio_upload.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-import os
-from minio import Minio
-import io
-
-# MinIO client setup
-client = Minio(
-    "localhost:9000",
-    access_key="minioadmin",
-    secret_key="minioadmin",
-    secure=False
-)
-
-# Test bucket name
-bucket_name = "test-bucket"
-
-# Create bucket if not exists
-if not client.bucket_exists(bucket_name):
-    client.make_bucket(bucket_name)
-    print(f"Created bucket: {bucket_name}")
-else:
-    print(f"Bucket already exists: {bucket_name}")
-
-# Upload test file
-source_file = "init-scripts/files/haley_system.txt"
-if os.path.exists(source_file):
-    with open(source_file, 'rb') as file_data:
-        content = file_data.read()
-        # Upload file
-        client.put_object(
-            bucket_name,
-            "haley_system.txt",
-            io.BytesIO(content),
-            len(content),
-            content_type="text/plain"
-        )
-        print(f"Successfully uploaded {source_file}")
-        
-        # Verify upload
-        stat = client.stat_object(bucket_name, "haley_system.txt")
-        print(f"File stats: {stat}")
-else:
-    print(f"Source file not found: {source_file}") 

isa_model/training/annotation/views/annotation_controller.py

@@ -1,158 +0,0 @@
-# app/services/llm_model/tracing/annotation/annotation_controller.py
-from typing import Dict, Any, List, Optional
-from datetime import datetime
-from bson import ObjectId
-from app.config.config_manager import config_manager
-from app.services.training.llm_model.annotation.annotation_schema import AnnotationFeedback, RatingScale
-from app.services.training.llm_model.annotation.storage.dataset_manager import DatasetManager
-
-
-class AnnotationController:
-    def __init__(self):
-        self.logger = config_manager.get_logger(__name__)
-        
-    async def get_pending_annotations(
-        self, 
-        project_name: str,
-        category: Optional[str] = None,
-        min_rating: Optional[int] = None,
-        page: int = 1,
-        limit: int = 10
-    ) -> Dict[str, Any]:
-        """Get filtered list of pending annotations"""
-        db = await config_manager.get_db('mongodb')
-        collection = db['annotations']
-        
-        # Build query with filters
-        query = {"status": "pending", "project_name": project_name}
-        if category:
-            query["annotation_type"] = category
-        if min_rating:
-            query["items.feedback.rating"] = {"$gte": min_rating}
-            
-        annotations = await collection.find(query)\
-            .sort("created_at", -1)\
-            .skip((page - 1) * limit)\
-            .limit(limit)\
-            .to_list(length=limit)
-            
-        return {
-            "annotations": annotations,
-            "pagination": {
-                "page": page,
-                "limit": limit,
-                "total": await collection.count_documents(query)
-            }
-        }
-
-    async def submit_annotation(
-        self,
-        annotation_id: str,
-        item_id: str,
-        feedback: AnnotationFeedback,
-        annotator_id: str
-    ) -> Dict[str, Any]:
-        """Submit and process annotation feedback"""
-        db = await config_manager.get_db('mongodb')
-        collection = db['annotations']
-        
-        # Determine if annotation should be selected for training
-        is_selected = self._evaluate_for_training(feedback)
-        feedback_dict = feedback.dict()
-        feedback_dict["is_selected_for_training"] = is_selected
-        
-        # Update annotation
-        result = await collection.update_one(
-            {
-                "_id": ObjectId(annotation_id),
-                "items.item_id": item_id
-            },
-            {
-                "$set": {
-                    "items.$.feedback": feedback_dict,
-                    "items.$.status": "completed",
-                    "items.$.annotated_at": datetime.utcnow().isoformat(),
-                    "items.$.annotator_id": annotator_id,
-                    "items.$.training_status": "pending" if is_selected else "none"
-                }
-            }
-        )
-        
-        # Process for training if selected
-        if is_selected:
-            await self._queue_for_training(annotation_id, item_id, feedback)
-        
-        return {
-            "status": "success",
-            "selected_for_training": is_selected,
-            "message": "Annotation submitted successfully"
-        }
-
-    def _evaluate_for_training(self, feedback: AnnotationFeedback) -> bool:
-        """Evaluate if annotation should be used for training"""
-        # Select for SFT if rating is excellent and aspects are positive
-        if feedback.rating == RatingScale.EXCELLENT:
-            aspects = feedback.aspects
-            if all([
-                aspects.factually_correct,
-                aspects.relevant,
-                not aspects.harmful,
-                not aspects.biased
-            ]):
-                return True
-        
-        # Select for RLHF if better response is provided
-        if feedback.better_response:
-            return True
-            
-        return False
-
-    async def _queue_for_training(
-        self,
-        annotation_id: str,
-        item_id: str,
-        feedback: AnnotationFeedback
-    ):
-        """Queue selected annotations for training data generation"""
-        db = await config_manager.get_db('mongodb')
-        training_queue = db['training_queue']
-        
-        await training_queue.insert_one({
-            "annotation_id": annotation_id,
-            "item_id": item_id,
-            "type": "sft" if feedback.rating == RatingScale.EXCELLENT else "rlhf",
-            "feedback": feedback.dict(),
-            "status": "pending",
-            "created_at": datetime.utcnow().isoformat()
-        })
-
-class DatasetPreparationProcessor:
-    def __init__(self):
-        self.logger = config_manager.get_logger(__name__)
-        self.dataset_manager = DatasetManager()
-        self.batch_size = 1000  # Configure as needed
-    
-    async def process_annotation_queue(self) -> None:
-        """Process pending annotations and prepare datasets"""
-        db = await config_manager.get_db('mongodb')
-        annotation_queue = db['dataset_preparation_queue']
-        
-        # Process items for SFT dataset
-        sft_items = await self._get_pending_annotations("sft")
-        if len(sft_items) >= self.batch_size:
-            await self._create_sft_dataset(sft_items)
-        
-        # Process items for RLHF dataset
-        rlhf_items = await self._get_pending_annotations("rlhf")
-        if len(rlhf_items) >= self.batch_size:
-            await self._create_rlhf_dataset(rlhf_items)
-
-    async def _get_pending_annotations(self, dataset_type: str) -> List[Dict[str, Any]]:
-        """Get pending annotations for dataset preparation"""
-        db = await config_manager.get_db('mongodb')
-        queue = db['dataset_preparation_queue']
-        
-        return await queue.find({
-            "status": "pending",
-            "dataset_type": dataset_type
-        }).to_list(length=self.batch_size)

isa_model/training/cloud/__init__.py

@@ -1,22 +0,0 @@
-"""
-Cloud Training Module for ISA Model Framework
-
-This module provides cloud training capabilities including:
-- RunPod integration for on-demand GPU training
-- Remote storage management (S3, GCS, etc.)
-- Training job orchestration and monitoring
-- Automatic resource scaling and management
-"""
-
-from .runpod_trainer import RunPodTrainer
-from .storage_manager import CloudStorageManager
-from .job_orchestrator import TrainingJobOrchestrator
-
-# Import config classes - these are defined in each module that needs them
-# from ..core.config import RunPodConfig, StorageConfig, JobConfig
-
-__all__ = [
-    "RunPodTrainer",
-    "CloudStorageManager", 
-    "TrainingJobOrchestrator"
-]