isa-model 0.0.1__py3-none-any.whl

isa_model/training/llm_model/annotation/storage/dataset_schema.py

@@ -0,0 +1,44 @@
+# app/services/llm_model/annotation/dataset/dataset_schema.py
+from enum import Enum
+from pydantic import BaseModel, Field
+from typing import Dict, List, Optional
+from datetime import datetime
+from bson import ObjectId
+
+class DatasetType(str, Enum):
+    SFT = "sft"
+    RLHF = "rlhf"
+
+class DatasetStatus(str, Enum):
+    PENDING = "pending"
+    PROCESSING = "processing"
+    READY = "ready"
+    ERROR = "error"
+
+class DatasetFiles(BaseModel):
+    train: str
+    eval: Optional[str]
+    test: Optional[str]
+
+class DatasetStats(BaseModel):
+    total_examples: int
+    avg_length: Optional[float]
+    num_conversations: Optional[int]
+    additional_metrics: Optional[Dict] = {}
+
+class Dataset(BaseModel):
+    id: Optional[ObjectId] = Field(None, alias="_id")
+    name: str
+    type: DatasetType
+    version: str
+    storage_path: str
+    files: DatasetFiles
+    stats: DatasetStats
+    source_annotations: List[str]
+    created_at: datetime
+    status: DatasetStatus
+    metadata: Optional[Dict] = {}
+
+    class Config:
+        arbitrary_types_allowed = True
+        populate_by_name = True

isa_model/training/llm_model/annotation/tests/test_annotation_flow.py

@@ -0,0 +1,109 @@
+# test_annotation_flow.py
+import os 
+os.environ["ENV"] = "local"
+
+import asyncio
+from datetime import datetime
+from bson import ObjectId
+from app.services.llm_model.annotation.views.annotation_controller import AnnotationController
+from app.services.llm_model.annotation.processors.annotation_processor import AnnotationProcessor
+from app.services.llm_model.annotation.annotation_schema import (
+    AnnotationFeedback, 
+    RatingScale, 
+    AnnotationType,
+    AnnotationAspects,
+    BetterResponse
+)
+from app.config.config_manager import config_manager 
+
+async def setup_test_data():
+    """Setup initial test data in MongoDB"""
+    db = await config_manager.get_db('mongodb')
+    
+    # Create a test annotation
+    test_annotation = {
+        "_id": ObjectId(),
+        "project_name": "test_project",
+        "items": [{
+            "item_id": "test_item_1",
+            "input": {
+                "messages": [{
+                    "role": "user",
+                    "content": "What is the capital of France?"
+                }]
+            },
+            "output": {
+                "content": "The capital of France is Paris."
+            },
+            "status": "pending"
+        }],
+        "created_at": datetime.utcnow().isoformat()
+    }
+    
+    await db['annotations'].insert_one(test_annotation)
+    return test_annotation
+
+async def test_annotation_flow():
+    """Test the complete annotation flow"""
+    try:
+        # Initialize controllers
+        annotation_controller = AnnotationController()
+        annotation_processor = AnnotationProcessor()
+        
+        # Setup test data
+        test_data = await setup_test_data()
+        annotation_id = str(test_data["_id"])
+        item_id = test_data["items"][0]["item_id"]
+        
+        print("1. Created test annotation")
+        
+        # Create test feedback
+        feedback = AnnotationFeedback(
+            rating=RatingScale.EXCELLENT,
+            category=AnnotationType.ACCURACY,
+            aspects=AnnotationAspects(
+                factually_correct=True,
+                relevant=True,
+                harmful=False,
+                biased=False,
+                complete=True,
+                efficient=True
+            ),
+            better_response=BetterResponse(
+                content="Paris is the capital city of France, known for its iconic Eiffel Tower.",
+                reason="Added more context and detail"
+            ),
+            comment="Good response, but could be more detailed"
+        )
+        
+        # Submit annotation
+        result = await annotation_controller.submit_annotation(
+            annotation_id=annotation_id,
+            item_id=item_id,
+            feedback=feedback,
+            annotator_id="test_annotator"
+        )
+        
+        print("2. Submitted annotation:", result)
+        
+        # Process annotation queue
+        await annotation_processor.process_queue()
+        print("3. Processed annotation queue")
+        
+        # Verify dataset creation
+        db = await config_manager.get_db('mongodb')
+        datasets = await db['training_datasets'].find().to_list(length=10)
+        
+        print("\nCreated Datasets:")
+        for dataset in datasets:
+            print(f"- {dataset['name']} ({dataset['type']})")
+            print(f"  Status: {dataset['status']}")
+            print(f"  Examples: {dataset['stats']['total_examples']}")
+            
+    except Exception as e:
+        print(f"Error during test: {e}")
+
+if __name__ == "__main__":
+    # Run the test
+    print("Starting annotation flow test...")
+    asyncio.run(test_annotation_flow())

isa_model/training/llm_model/annotation/tests/test_minio copy.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+
+import os
+from minio import Minio
+import json
+import logging
+from io import BytesIO
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def test_minio():
+    try:
+        # Get MinIO host from environment variable or use default
+        minio_host = os.getenv("MINIO_HOST", "localhost:9000")
+        logger.info(f"Using MinIO host: {minio_host}")
+        
+        # 1. Create MinIO client
+        logger.info("Creating MinIO client...")
+        client = Minio(
+            minio_host,
+            access_key="minioadmin",
+            secret_key="minioadmin",
+            secure=False
+        )
+        
+        # 2. Test bucket operations
+        bucket_name = "knowledge-files"  # Changed to match the actual bucket name
+        logger.info(f"Testing bucket operations with {bucket_name}...")
+        
+        # Create bucket if it doesn't exist
+        if not client.bucket_exists(bucket_name):
+            client.make_bucket(bucket_name)
+            logger.info(f"Created new bucket: {bucket_name}")
+        else:
+            logger.info(f"Using existing bucket: {bucket_name}")
+        
+        # Set bucket policy - allow all operations
+        policy = {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {"AWS": ["*"]},
+                    "Action": [
+                        "s3:GetBucketLocation",
+                        "s3:ListBucket",
+                        "s3:ListBucketMultipartUploads"
+                    ],
+                    "Resource": [f"arn:aws:s3:::{bucket_name}"]
+                },
+                {
+                    "Effect": "Allow",
+                    "Principal": {"AWS": ["*"]},
+                    "Action": [
+                        "s3:AbortMultipartUpload",
+                        "s3:DeleteObject",
+                        "s3:GetObject",
+                        "s3:ListMultipartUploadParts",
+                        "s3:PutObject"
+                    ],
+                    "Resource": [f"arn:aws:s3:::{bucket_name}/*"]
+                }
+            ]
+        }
+        
+        try:
+            client.set_bucket_policy(bucket_name, json.dumps(policy))
+            logger.info("Set bucket policy successfully")
+        except Exception as e:
+            logger.warning(f"Failed to set bucket policy: {e}")
+        
+        # 3. Test file upload
+        source_file = "init-scripts/files/haley_system.txt"
+        object_name = "haley_system.txt"
+        
+        if os.path.exists(source_file):
+            # Get file size
+            file_size = os.path.getsize(source_file)
+            logger.info(f"Found source file: {source_file} (size: {file_size} bytes)")
+            
+            # Upload file
+            with open(source_file, 'rb') as file_data:
+                client.put_object(
+                    bucket_name,
+                    object_name,
+                    file_data,
+                    file_size,
+                    content_type="text/plain"
+                )
+            logger.info(f"Uploaded file: {object_name}")
+            
+            # 4. Test file download
+            data = client.get_object(bucket_name, object_name)
+            content = data.read().decode('utf-8')
+            logger.info(f"Successfully downloaded file. First 100 chars: {content[:100]}...")
+            
+            # 5. Verify file exists
+            stat = client.stat_object(bucket_name, object_name)
+            logger.info(f"File stats: {stat}")
+            
+            logger.info("Test completed successfully!")
+        else:
+            logger.error(f"Source file not found: {source_file}")
+            raise FileNotFoundError(f"Source file not found: {source_file}")
+        
+    except Exception as e:
+        logger.error(f"Test failed: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    test_minio() 

isa_model/training/llm_model/annotation/tests/test_minio_upload.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import os
+from minio import Minio
+import io
+
+# MinIO client setup
+client = Minio(
+    "localhost:9000",
+    access_key="minioadmin",
+    secret_key="minioadmin",
+    secure=False
+)
+
+# Test bucket name
+bucket_name = "test-bucket"
+
+# Create bucket if not exists
+if not client.bucket_exists(bucket_name):
+    client.make_bucket(bucket_name)
+    print(f"Created bucket: {bucket_name}")
+else:
+    print(f"Bucket already exists: {bucket_name}")
+
+# Upload test file
+source_file = "init-scripts/files/haley_system.txt"
+if os.path.exists(source_file):
+    with open(source_file, 'rb') as file_data:
+        content = file_data.read()
+        # Upload file
+        client.put_object(
+            bucket_name,
+            "haley_system.txt",
+            io.BytesIO(content),
+            len(content),
+            content_type="text/plain"
+        )
+        print(f"Successfully uploaded {source_file}")
+        
+        # Verify upload
+        stat = client.stat_object(bucket_name, "haley_system.txt")
+        print(f"File stats: {stat}")
+else:
+    print(f"Source file not found: {source_file}") 

isa_model/training/llm_model/annotation/views/annotation_controller.py

@@ -0,0 +1,158 @@
+# app/services/llm_model/tracing/annotation/annotation_controller.py
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+from bson import ObjectId
+from app.config.config_manager import config_manager
+from app.services.training.llm_model.annotation.annotation_schema import AnnotationFeedback, RatingScale
+from app.services.training.llm_model.annotation.storage.dataset_manager import DatasetManager
+
+
+class AnnotationController:
+    def __init__(self):
+        self.logger = config_manager.get_logger(__name__)
+        
+    async def get_pending_annotations(
+        self, 
+        project_name: str,
+        category: Optional[str] = None,
+        min_rating: Optional[int] = None,
+        page: int = 1,
+        limit: int = 10
+    ) -> Dict[str, Any]:
+        """Get filtered list of pending annotations"""
+        db = await config_manager.get_db('mongodb')
+        collection = db['annotations']
+        
+        # Build query with filters
+        query = {"status": "pending", "project_name": project_name}
+        if category:
+            query["annotation_type"] = category
+        if min_rating:
+            query["items.feedback.rating"] = {"$gte": min_rating}
+            
+        annotations = await collection.find(query)\
+            .sort("created_at", -1)\
+            .skip((page - 1) * limit)\
+            .limit(limit)\
+            .to_list(length=limit)
+            
+        return {
+            "annotations": annotations,
+            "pagination": {
+                "page": page,
+                "limit": limit,
+                "total": await collection.count_documents(query)
+            }
+        }
+
+    async def submit_annotation(
+        self,
+        annotation_id: str,
+        item_id: str,
+        feedback: AnnotationFeedback,
+        annotator_id: str
+    ) -> Dict[str, Any]:
+        """Submit and process annotation feedback"""
+        db = await config_manager.get_db('mongodb')
+        collection = db['annotations']
+        
+        # Determine if annotation should be selected for training
+        is_selected = self._evaluate_for_training(feedback)
+        feedback_dict = feedback.dict()
+        feedback_dict["is_selected_for_training"] = is_selected
+        
+        # Update annotation
+        result = await collection.update_one(
+            {
+                "_id": ObjectId(annotation_id),
+                "items.item_id": item_id
+            },
+            {
+                "$set": {
+                    "items.$.feedback": feedback_dict,
+                    "items.$.status": "completed",
+                    "items.$.annotated_at": datetime.utcnow().isoformat(),
+                    "items.$.annotator_id": annotator_id,
+                    "items.$.training_status": "pending" if is_selected else "none"
+                }
+            }
+        )
+        
+        # Process for training if selected
+        if is_selected:
+            await self._queue_for_training(annotation_id, item_id, feedback)
+        
+        return {
+            "status": "success",
+            "selected_for_training": is_selected,
+            "message": "Annotation submitted successfully"
+        }
+
+    def _evaluate_for_training(self, feedback: AnnotationFeedback) -> bool:
+        """Evaluate if annotation should be used for training"""
+        # Select for SFT if rating is excellent and aspects are positive
+        if feedback.rating == RatingScale.EXCELLENT:
+            aspects = feedback.aspects
+            if all([
+                aspects.factually_correct,
+                aspects.relevant,
+                not aspects.harmful,
+                not aspects.biased
+            ]):
+                return True
+        
+        # Select for RLHF if better response is provided
+        if feedback.better_response:
+            return True
+            
+        return False
+
+    async def _queue_for_training(
+        self,
+        annotation_id: str,
+        item_id: str,
+        feedback: AnnotationFeedback
+    ):
+        """Queue selected annotations for training data generation"""
+        db = await config_manager.get_db('mongodb')
+        training_queue = db['training_queue']
+        
+        await training_queue.insert_one({
+            "annotation_id": annotation_id,
+            "item_id": item_id,
+            "type": "sft" if feedback.rating == RatingScale.EXCELLENT else "rlhf",
+            "feedback": feedback.dict(),
+            "status": "pending",
+            "created_at": datetime.utcnow().isoformat()
+        })
+
+class DatasetPreparationProcessor:
+    def __init__(self):
+        self.logger = config_manager.get_logger(__name__)
+        self.dataset_manager = DatasetManager()
+        self.batch_size = 1000  # Configure as needed
+    
+    async def process_annotation_queue(self) -> None:
+        """Process pending annotations and prepare datasets"""
+        db = await config_manager.get_db('mongodb')
+        annotation_queue = db['dataset_preparation_queue']
+        
+        # Process items for SFT dataset
+        sft_items = await self._get_pending_annotations("sft")
+        if len(sft_items) >= self.batch_size:
+            await self._create_sft_dataset(sft_items)
+        
+        # Process items for RLHF dataset
+        rlhf_items = await self._get_pending_annotations("rlhf")
+        if len(rlhf_items) >= self.batch_size:
+            await self._create_rlhf_dataset(rlhf_items)
+
+    async def _get_pending_annotations(self, dataset_type: str) -> List[Dict[str, Any]]:
+        """Get pending annotations for dataset preparation"""
+        db = await config_manager.get_db('mongodb')
+        queue = db['dataset_preparation_queue']
+        
+        return await queue.find({
+            "status": "pending",
+            "dataset_type": dataset_type
+        }).to_list(length=self.batch_size)