abstractcore 2.6.2__py3-none-any.whl → 2.6.5__py3-none-any.whl

abstractcore/apps/judge.py

@@ -178,7 +178,7 @@ def format_assessment_plain(assessment: dict) -> str:
     lines.append(f"Overall Score: {assessment.get('overall_score', 0)}/5")
     lines.append("")
 
-    # Individual scores
+    # Predefined criterion scores
     score_fields = [
         ('clarity_score', 'Clarity'),
         ('simplicity_score', 'Simplicity'),
@@ -191,13 +191,28 @@ def format_assessment_plain(assessment: dict) -> str:
         ('coherence_score', 'Coherence')
     ]
 
-    lines.append("Individual Scores:")
-    lines.append("-" * 20)
-    for field, label in score_fields:
-        score = assessment.get(field)
-        if score is not None:
-            lines.append(f"{label:15}: {score}/5")
-    lines.append("")
+    # Check if any predefined scores exist
+    has_predefined_scores = any(assessment.get(field) is not None for field, _ in score_fields)
+
+    if has_predefined_scores:
+        lines.append("📋 Predefined Criterion Scores:")
+        lines.append("-" * 32)
+        for field, label in score_fields:
+            score = assessment.get(field)
+            if score is not None:
+                lines.append(f"{label:15}: {score}/5")
+        lines.append("")
+
+    # Custom criterion scores
+    custom_scores = assessment.get('custom_scores', {})
+    if custom_scores:
+        lines.append("🎯 Custom Criterion Scores:")
+        lines.append("-" * 28)
+        for criterion, score in custom_scores.items():
+            # Format criterion name nicely
+            criterion_display = criterion.replace('_', ' ').title()
+            lines.append(f"{criterion_display:30}: {score}/5")
+        lines.append("")
 
     # Strengths
     strengths = assessment.get('strengths', [])

abstractcore/core/session.py

@@ -939,13 +939,18 @@ class BasicSession:
         summary_tokens = self._estimate_tokens_for_summary(summary_text)
         return original_tokens / summary_tokens if summary_tokens > 0 else 1.0
 
-    def generate_assessment(self, criteria: Optional[Dict[str, bool]] = None) -> Dict[str, Any]:
+    def generate_assessment(
+        self,
+        criteria: Optional[Dict[str, bool]] = None,
+        custom_criteria: Optional[Dict[str, str]] = None
+    ) -> Dict[str, Any]:
         """
         Generate a quality assessment of the entire conversation and store it in session.assessment.
-        
+
         Args:
-            criteria: Optional criteria for assessment
-            
+            criteria: Optional predefined criteria toggles (e.g., {"clarity": True, "coherence": False})
+            custom_criteria: Optional custom domain-specific criteria with descriptions (e.g., {"logical_coherence": "Are results logically consistent?"})
+
         Returns:
             Dict containing the generated assessment
         """
@@ -989,13 +994,27 @@ class BasicSession:
         assessment_result = judge.evaluate(
             content=conversation_text,
             context="conversation quality assessment",
-            criteria=judge_criteria
+            criteria=judge_criteria,
+            custom_criteria=custom_criteria
         )
         
         # Store assessment in session
         self.assessment = {
             "created_at": start_time.isoformat(),
             "criteria": criteria,
+            "custom_criteria": custom_criteria,
+            "scores": {
+                "clarity": assessment_result.get('clarity_score'),
+                "simplicity": assessment_result.get('simplicity_score'),
+                "actionability": assessment_result.get('actionability_score'),
+                "soundness": assessment_result.get('soundness_score'),
+                "innovation": assessment_result.get('innovation_score'),
+                "effectiveness": assessment_result.get('effectiveness_score'),
+                "relevance": assessment_result.get('relevance_score'),
+                "completeness": assessment_result.get('completeness_score'),
+                "coherence": assessment_result.get('coherence_score'),
+            },
+            "custom_scores": assessment_result.get('custom_scores', {}),
             "overall_score": assessment_result.get('overall_score', 0),
             "judge_summary": assessment_result.get('judge_summary', ''),
             "strengths": assessment_result.get('strengths', []),

abstractcore/processing/basic_judge.py

@@ -9,11 +9,11 @@ Features:
 - Clear, simple and actionable feedback
 """
 
-from typing import Optional, List, Dict, Any, Union
+from typing import Optional, List, Dict, Any, Union, Type
 import json
 import logging
 from pathlib import Path
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, create_model
 
 from ..core.interface import AbstractCoreInterface
 from ..core.factory import create_llm
@@ -44,7 +44,7 @@ class Assessment(BaseModel):
     judge_summary: str = Field(..., description="Judge's experiential note summarizing the assessment task and key findings")
     source_reference: str = Field(..., description="Reference to what was assessed (file, content type, context)")
 
-    # Individual criterion scores
+    # Individual criterion scores (predefined criteria)
     clarity_score: Optional[int] = Field(None, description="Clarity score (1-5)")
     simplicity_score: Optional[int] = Field(None, description="Simplicity score (1-5)")
     actionability_score: Optional[int] = Field(None, description="Actionability score (1-5)")
@@ -161,6 +161,30 @@ class BasicJudge:
 
         self.retry_strategy = FeedbackRetry(max_attempts=3)
 
+    def _create_dynamic_assessment_model(self, custom_criteria: Optional[Dict[str, str]]) -> Type[BaseModel]:
+        """Create a dynamic Assessment model with custom score fields"""
+        if not custom_criteria:
+            return Assessment
+
+        # Build fields dict for dynamic model creation
+        fields_dict = {}
+
+        # Add custom score fields dynamically as REQUIRED (not Optional)
+        # This forces the LLM to provide scores for all custom criteria
+        for criterion_name in custom_criteria.keys():
+            field_name = f"{criterion_name}_score"
+            # Make it required (int, not Optional[int]) with Field(...)
+            fields_dict[field_name] = (int, Field(..., description=f"{criterion_name} score (1-5)", ge=1, le=5))
+
+        # Create dynamic model that inherits from Assessment using create_model
+        DynamicAssessment = create_model(
+            'DynamicAssessment',
+            __base__=Assessment,
+            **fields_dict
+        )
+
+        return DynamicAssessment
+
     def evaluate(
         self,
         content: str,
@@ -168,7 +192,8 @@ class BasicJudge:
         criteria: Optional[JudgmentCriteria] = None,
         focus: Optional[str] = None,
         reference: Optional[str] = None,
-        include_criteria: bool = False
+        include_criteria: bool = False,
+        custom_criteria: Optional[Dict[str, str]] = None
     ) -> dict:
         """
         Evaluate content against specified criteria
@@ -180,6 +205,7 @@ class BasicJudge:
             focus: Specific areas to focus evaluation on (e.g., "technical accuracy, performance")
             reference: Optional reference/expected output for comparison
             include_criteria: Include detailed explanation of evaluation criteria in assessment
+            custom_criteria: Custom domain-specific criteria with descriptions (e.g., {"logical_coherence": "Are results logically consistent?"})
 
         Returns:
             dict: Structured assessment result
@@ -196,13 +222,16 @@ class BasicJudge:
         logger.info("Starting evaluation", context=context)
 
         # Build the evaluation prompt
-        prompt = self._build_evaluation_prompt(content, context, criteria, focus, reference, include_criteria)
+        prompt = self._build_evaluation_prompt(content, context, criteria, focus, reference, include_criteria, custom_criteria)
+
+        # Create dynamic assessment model with custom score fields
+        AssessmentModel = self._create_dynamic_assessment_model(custom_criteria)
 
         # Generate structured assessment
         try:
             result = self.llm.generate(
                 prompt,
-                response_model=Assessment,
+                response_model=AssessmentModel,
                 retry_strategy=self.retry_strategy
             )
             
@@ -216,6 +245,19 @@ class BasicJudge:
             # Convert to dict and add metadata
             assessment_dict = result.dict() if hasattr(result, 'dict') else result
 
+            # Extract custom scores from individual fields and add to custom_scores dict
+            if custom_criteria:
+                custom_scores = {}
+                for criterion_name in custom_criteria.keys():
+                    field_name = f"{criterion_name}_score"
+                    if field_name in assessment_dict:
+                        score_value = assessment_dict.pop(field_name)  # Remove individual field
+                        if score_value is not None:
+                            custom_scores[criterion_name] = score_value
+                assessment_dict['custom_scores'] = custom_scores
+            else:
+                assessment_dict['custom_scores'] = {}
+
             # Log results
             overall_score = assessment_dict.get('overall_score', 0)
             logger.info("Evaluation completed", overall_score=overall_score, max_score=5)
@@ -247,7 +289,8 @@ class BasicJudge:
         reference: Optional[str] = None,
         include_criteria: bool = False,
         max_file_size: int = 1000000,  # 1MB default limit per file
-        exclude_global: bool = False  # Include global assessment by default
+        exclude_global: bool = False,  # Include global assessment by default
+        custom_criteria: Optional[Dict[str, str]] = None
     ) -> Union[dict, List[dict]]:
         """
         Evaluate content from one or multiple files sequentially to avoid context overflow
@@ -261,6 +304,7 @@ class BasicJudge:
             include_criteria: Include detailed explanation of evaluation criteria in assessment
             max_file_size: Maximum file size in bytes (default 1MB to avoid context overflow)
             exclude_global: If True, skip global assessment for multiple files (default False)
+            custom_criteria: Custom domain-specific criteria with descriptions (e.g., {"logical_coherence": "Are results logically consistent?"})
 
         Returns:
             dict: Single assessment if one file provided
@@ -360,7 +404,8 @@ class BasicJudge:
                 criteria=criteria,
                 focus=focus,
                 reference=reference,
-                include_criteria=include_criteria
+                include_criteria=include_criteria,
+                custom_criteria=custom_criteria
             )
 
             # Update source reference to include file name
@@ -382,7 +427,7 @@ class BasicJudge:
             # Generate global assessment and return structured result
             logger.info("Generating global assessment from individual file evaluations", file_count=len(assessments))
             global_assessment = self._generate_global_assessment(
-                assessments, context, criteria, focus, include_criteria
+                assessments, context, criteria, focus, include_criteria, custom_criteria
             )
 
             return {
@@ -396,7 +441,8 @@ class BasicJudge:
         context: str,
         criteria: Optional[JudgmentCriteria],
         focus: Optional[str],
-        include_criteria: bool
+        include_criteria: bool,
+        custom_criteria: Optional[Dict[str, str]] = None
     ) -> dict:
         """
         Generate a global assessment from multiple individual file assessments
@@ -475,7 +521,8 @@ Provide a comprehensive global assessment of overall quality and recommendations
                 context=f"global assessment summary for {total_files} files ({context})",
                 criteria=criteria,
                 focus=focus,
-                include_criteria=include_criteria
+                include_criteria=include_criteria,
+                custom_criteria=custom_criteria
             )
 
             # Update the source reference to indicate this is a global assessment
@@ -506,6 +553,19 @@ Provide a comprehensive global assessment of overall quality and recommendations
                 "evaluation_criteria_details": None
             }
 
+    def _build_custom_scores_format(self, custom_criteria: Optional[Dict[str, str]]) -> str:
+        """Build custom score fields for the prompt (individual fields, not dict)"""
+        if not custom_criteria:
+            return ""
+
+        # Build individual score fields for each custom criterion
+        score_fields = []
+        for criterion_name in custom_criteria.keys():
+            field_name = f"{criterion_name}_score"
+            score_fields.append(f'    "{field_name}": <1-5 integer>,')
+
+        return "\n" + "\n".join(score_fields)
+
     def _build_evaluation_prompt(
         self,
         content: str,
@@ -513,7 +573,8 @@ Provide a comprehensive global assessment of overall quality and recommendations
         criteria: JudgmentCriteria,
         focus: Optional[str],
         reference: Optional[str],
-        include_criteria: bool = False
+        include_criteria: bool = False,
+        custom_criteria: Optional[Dict[str, str]] = None
     ) -> str:
         """Build the evaluation prompt with chain-of-thought reasoning"""
 
@@ -565,6 +626,12 @@ Provide a comprehensive global assessment of overall quality and recommendations
                 active_criteria.append(focus_item)
                 criteria_descriptions.append(f"- **{focus_item.title()}**: PRIMARY FOCUS AREA - This is a key evaluation target")
 
+        # Add custom criteria with their specific descriptions
+        if custom_criteria:
+            for name, description in custom_criteria.items():
+                active_criteria.append(name)
+                criteria_descriptions.append(f"- **{name.replace('_', ' ').title()}**: {description}")
+
         criteria_text = "\n".join(criteria_descriptions)
 
         # Build reference section if provided
@@ -613,12 +680,26 @@ SCORING RUBRIC (1-5 scale):
 - **Score 2**: Poor - Falls short of expectations with significant issues
 - **Score 1**: Very Poor - Fails to meet basic standards in this dimension
 
+SCORING PRINCIPLES - CRITICAL:
+- **Be rigorous and avoid grade inflation**: Most adequate responses should be scored 2-3, not 3-4
+- **Context matters**: For routine tasks (e.g., basic arithmetic), criteria like "innovation" should be scored 1-2 unless truly creative
+- **If a criterion doesn't meaningfully apply to the task**, score it 1-2, not 3 (e.g., innovation for standard formula application = 1)
+- **Reserve 4-5 for genuinely excellent work**: Don't give high scores by default
+- **Apply task-appropriate expectations**:
+  * Routine calculations: innovation 1-2, soundness 4-5 (if correct)
+  * Creative explanations: innovation 3-4 if novel approach shown
+  * Complex problem-solving: innovation 4-5 if breakthrough thinking demonstrated
+- **Be appropriately critical**: Question whether the response truly meets each criterion
+
 EVALUATION PROCESS:
 1. **STEP 1**: Carefully analyze the content for each active criterion
-2. **STEP 2**: Identify specific strengths and weaknesses
-3. **STEP 3**: Provide actionable recommendations for improvement
-4. **STEP 4**: Assign scores based on the rubric (be fair but appropriately critical)
-5. **STEP 5**: Calculate overall score - PRIMARY FOCUS AREAS should heavily influence the final score
+2. **STEP 2**: Assess if each criterion meaningfully applies to this task (if not, score 1-2)
+3. **STEP 3**: Identify specific strengths and weaknesses
+4. **STEP 4**: Provide actionable recommendations for improvement
+5. **STEP 5**: Assign scores based on the rubric (be rigorous and appropriately critical)
+   - For standard criteria: populate the corresponding _score fields (e.g., clarity_score, soundness_score)
+   - For custom criteria: populate the custom_scores object with scores for EACH custom criterion listed in EVALUATION CRITERIA
+6. **STEP 6**: Calculate overall score - PRIMARY FOCUS AREAS should heavily influence the final score
 
 CRITICAL ASSESSMENT PRINCIPLES:
 - Be objective and evidence-based in your evaluation
@@ -628,6 +709,11 @@ CRITICAL ASSESSMENT PRINCIPLES:
 - Ensure recommendations are specific and implementable
 - PRIMARY FOCUS AREAS are the most important evaluation targets - weaknesses in these areas should significantly impact the overall score
 
+IMPORTANT - SCORING REQUIREMENTS:
+- You MUST provide individual scores (1-5) for EVERY criterion in the custom_scores object if custom criteria are present
+- Do NOT leave custom_scores as an empty object {{}} - populate it with scores for each custom criterion
+- Each custom criterion listed in EVALUATION CRITERIA must have a corresponding score in custom_scores
+
 RESPONSE FORMAT:
 Provide your assessment as a structured JSON response with the following format:
 
@@ -643,7 +729,7 @@ Provide your assessment as a structured JSON response with the following format:
     "effectiveness_score": <1-5 integer or null if not evaluated>,
     "relevance_score": <1-5 integer or null if not evaluated>,
     "completeness_score": <1-5 integer or null if not evaluated>,
-    "coherence_score": <1-5 integer or null if not evaluated>,
+    "coherence_score": <1-5 integer or null if not evaluated>,{self._build_custom_scores_format(custom_criteria)}
     "strengths": ["list of specific strengths identified"],
     "weaknesses": ["list of specific areas for improvement"],
     "actionable_feedback": ["list of specific actionable recommendations"],

abstractcore/providers/__init__.py

@@ -7,6 +7,8 @@ from .ollama_provider import OllamaProvider
 from .lmstudio_provider import LMStudioProvider
 from .huggingface_provider import HuggingFaceProvider
 from .mlx_provider import MLXProvider
+from .vllm_provider import VLLMProvider
+from .openai_compatible_provider import OpenAICompatibleProvider
 
 # Provider registry for centralized provider discovery and management
 from .registry import (
@@ -41,6 +43,8 @@ __all__ = [
     'LMStudioProvider',
     'HuggingFaceProvider',
     'MLXProvider',
+    'VLLMProvider',
+    'OpenAICompatibleProvider',
 
     # Provider registry
     'ProviderRegistry',