moai-adk 0.25.4__py3-none-any.whl → 0.41.0__py3-none-any.whl

moai_adk/core/spec/quality_validator.py

@@ -1,721 +0,0 @@
-"""Quality Validator for Auto-Generated SPECs."""
-
-import logging
-import re
-import time
-from typing import Any, Dict, List
-
-from moai_adk.core.spec.confidence_scoring import ConfidenceScoringSystem
-
-# Configure logging
-logger = logging.getLogger(__name__)
-
-
-class QualityValidator:
-    """
-    Quality Validator for auto-generated SPEC documents.
-
-    This validator ensures that auto-generated SPECs meet quality standards,
-    follow EARS format, and are complete and accurate.
-    """
-
-    def __init__(self, config: Dict[str, Any] = None):
-        """Initialize the quality validator."""
-        self.config = config or {}
-        self.confidence_scorer = ConfidenceScoringSystem()
-
-        # Quality thresholds from config
-        self.min_ears_compliance = self.config.get("min_ears_compliance", 0.85)
-        self.min_confidence_score = self.config.get("min_confidence_score", 0.7)
-        self.min_content_length = self.config.get("min_content_length", 500)
-        self.max_review_suggestions = self.config.get("max_review_suggestions", 10)
-
-        # Quality metrics weights
-        self.quality_weights = self.config.get(
-            "quality_weights",
-            {
-                "ears_compliance": 0.3,
-                "content_completeness": 0.25,
-                "technical_accuracy": 0.2,
-                "clarity_score": 0.15,
-                "traceability": 0.1,
-            },
-        )
-
-        # Validation rules
-        self.validation_rules = {
-            "required_sections": [
-                "Overview",
-                "Environment",
-                "Assumptions",
-                "Requirements",
-                "Specifications",
-                "Traceability",
-            ],
-            "required_plan_sections": [
-                "Implementation Phases",
-                "Technical Approach",
-                "Success Criteria",
-                "Next Steps",
-            ],
-            "required_acceptance_sections": [
-                "Acceptance Criteria",
-                "Validation Process",
-                "Completion Criteria",
-            ],
-            "technical_keywords": [
-                "API",
-                "Database",
-                "Authentication",
-                "Security",
-                "Performance",
-                "Scalability",
-                "Testing",
-            ],
-        }
-
-    def validate_spec_quality(
-        self, spec_content: Dict[str, str], code_analysis: Dict[str, Any] = None
-    ) -> Dict[str, Any]:
-        """
-        Validate the quality of a generated SPEC.
-
-        Args:
-            spec_content: Dictionary with spec_md, plan_md, and acceptance_md
-            code_analysis: Original code analysis for reference
-
-        Returns:
-            Quality validation results with scores and recommendations
-        """
-        start_time = time.time()
-
-        logger.info("Starting SPEC quality validation")
-
-        # Initialize validation results
-        validation_result = {
-            "validation_time": 0.0,
-            "overall_score": 0.0,
-            "quality_grade": "F",
-            "passed_checks": [],
-            "failed_checks": [],
-            "recommendations": [],
-            "metrics": {},
-            "details": {},
-        }
-
-        try:
-            # Validate EARS format compliance
-            ears_result = self._validate_ears_format(spec_content["spec_md"])
-            validation_result["details"]["ears_compliance"] = ears_result
-
-            # Validate content completeness
-            completeness_result = self._validate_content_completeness(spec_content)
-            validation_result["details"]["content_completeness"] = completeness_result
-
-            # Validate technical accuracy
-            technical_result = self._validate_technical_accuracy(
-                spec_content, code_analysis
-            )
-            validation_result["details"]["technical_accuracy"] = technical_result
-
-            # Validate clarity and readability
-            clarity_result = self._validate_clarity(spec_content)
-            validation_result["details"]["clarity_score"] = clarity_result
-
-            # Validate traceability
-            traceability_result = self._validate_traceability(spec_content)
-            validation_result["details"]["traceability"] = traceability_result
-
-            # Calculate overall quality score
-            overall_score = self._calculate_overall_score(validation_result["details"])
-            validation_result["overall_score"] = overall_score
-
-            # Determine quality grade
-            validation_result["quality_grade"] = self._determine_quality_grade(
-                overall_score
-            )
-
-            # Check if SPEC meets minimum quality standards
-            meets_standards = self._check_quality_standards(validation_result)
-            validation_result["meets_minimum_standards"] = meets_standards
-
-            # Generate recommendations
-            recommendations = self._generate_recommendations(validation_result)
-            validation_result["recommendations"] = recommendations
-
-            # Compile check results
-            validation_result["passed_checks"] = self._compile_passed_checks(
-                validation_result["details"]
-            )
-            validation_result["failed_checks"] = self._compile_failed_checks(
-                validation_result["details"]
-            )
-
-        except Exception as e:
-            logger.error(f"Error during SPEC validation: {str(e)}")
-            validation_result["error"] = str(e)
-
-        # Set validation time
-        validation_result["validation_time"] = time.time() - start_time
-
-        logger.info(
-            f"SPEC quality validation completed in {validation_result['validation_time']:.2f}s"
-        )
-
-        return validation_result
-
-    def _validate_ears_format(self, spec_md: str) -> Dict[str, Any]:
-        """Validate EARS format compliance."""
-        logger.info("Validating EARS format compliance")
-
-        required_sections = self.validation_rules["required_sections"]
-        section_scores = {}
-        missing_sections = []
-
-        # Check each required section
-        for section in required_sections:
-            if section in spec_md:
-                section_scores[section] = 1.0
-            else:
-                section_scores[section] = 0.0
-                missing_sections.append(section)
-
-        # Calculate overall compliance
-        overall_compliance = sum(section_scores.values()) / len(required_sections)
-
-        # Check for META information
-
-        # Check for proper structure
-        has_proper_headings = self._check_heading_structure(spec_md)
-
-        return {
-            "overall_compliance": round(overall_compliance, 2),
-            "section_scores": section_scores,
-            "missing_sections": missing_sections,
-            "has_meta_info": has_meta,
-            "has_tags": has_tags,
-            "has_proper_structure": has_proper_headings,
-            "total_sections": len(required_sections),
-            "present_sections": sum(
-                1 for score in section_scores.values() if score > 0
-            ),
-        }
-
-    def _validate_content_completeness(
-        self, spec_content: Dict[str, str]
-    ) -> Dict[str, Any]:
-        """Validate content completeness across all SPEC files."""
-        logger.info("Validating content completeness")
-
-        results = {
-            "spec_completeness": 0.0,
-            "plan_completeness": 0.0,
-            "acceptance_completeness": 0.0,
-            "overall_completeness": 0.0,
-        }
-
-        # Validate spec.md completeness
-        spec_md = spec_content.get("spec_md", "")
-        spec_completeness = self._assess_section_completeness(spec_md)
-        results["spec_completeness"] = spec_completeness
-
-        # Validate plan.md completeness
-        plan_md = spec_content.get("plan_md", "")
-        plan_completeness = self._assess_plan_completeness(plan_md)
-        results["plan_completeness"] = plan_completeness
-
-        # Validate acceptance.md completeness
-        acceptance_md = spec_content.get("acceptance_md", "")
-        acceptance_completeness = self._assess_acceptance_completeness(acceptance_md)
-        results["acceptance_completeness"] = acceptance_completeness
-
-        # Calculate overall completeness
-        overall_completeness = (
-            spec_completeness + plan_completeness + acceptance_completeness
-        ) / 3
-        results["overall_completeness"] = round(overall_completeness, 2)
-
-        return results
-
-    def _validate_technical_accuracy(
-        self, spec_content: Dict[str, str], code_analysis: Dict[str, Any] = None
-    ) -> Dict[str, Any]:
-        """Validate technical accuracy of the SPEC."""
-        logger.info("Validating technical accuracy")
-
-        spec_md = spec_content.get("spec_md", "")
-
-        # Check for technical keywords presence
-        technical_keywords = self.validation_rules["technical_keywords"]
-        keyword_presence = self._check_technical_keywords(spec_md, technical_keywords)
-
-        # Check consistency with code analysis
-        consistency_score = 1.0
-        if code_analysis:
-            consistency_score = self._check_code_consistency(spec_md, code_analysis)
-
-        # Check for technical specifications
-        has_technical_specs = self._check_technical_specifications(spec_md)
-
-        # Check for realistic estimates
-        has_realistic_estimates = self._check_realistic_estimates(spec_md)
-
-        return {
-            "keyword_presence": keyword_presence,
-            "code_consistency": consistency_score,
-            "has_technical_specs": has_technical_specs,
-            "has_realistic_estimates": has_realistic_estimates,
-            "overall_accuracy": round(
-                (
-                    keyword_presence
-                    + consistency_score
-                    + (1.0 if has_technical_specs else 0.0)
-                    + (1.0 if has_realistic_estimates else 0.0)
-                )
-                / 4,
-                2,
-            ),
-        }
-
-    def _validate_clarity(self, spec_content: Dict[str, str]) -> Dict[str, Any]:
-        """Validate clarity and readability of the SPEC."""
-        logger.info("Validating clarity and readability")
-
-        spec_md = spec_content.get("spec_md", "")
-
-        # Check for proper language use
-        language_quality = self._check_language_quality(spec_md)
-
-        # Check for clarity requirements
-        clarity_requirements = self._check_clarity_requirements(spec_md)
-
-        # Check for ambiguity
-        ambiguity_score = self._check_ambiguity(spec_md)
-
-        # Check for consistency
-        consistency_score = self._check_consistency(spec_md)
-
-        return {
-            "language_quality": language_quality,
-            "clarity_requirements": clarity_requirements,
-            "ambiguity_score": ambiguity_score,
-            "consistency_score": consistency_score,
-            "overall_clarity": round(
-                (
-                    language_quality
-                    + clarity_requirements
-                    + (1.0 - ambiguity_score)
-                    + consistency_score
-                )
-                / 4,
-                2,
-            ),
-        }
-
-    def _validate_traceability(self, spec_content: Dict[str, str]) -> Dict[str, Any]:
-        """Validate traceability in the SPEC."""
-        logger.info("Validating traceability")
-
-        spec_md = spec_content.get("spec_md", "")
-
-        # Check for basic traceability elements
-        has_traceability = "@SPEC:" in spec_md or "Requirements:" in spec_md
-
-        return {
-            "has_traceability_elements": has_traceability,
-            "overall_traceability": 1.0 if has_traceability else 0.5,
-        }
-
-    def _calculate_overall_score(self, details: Dict[str, Any]) -> float:
-        """Calculate overall quality score."""
-        weights = self.quality_weights
-
-        overall_score = (
-            details["ears_compliance"]["overall_compliance"]
-            * weights["ears_compliance"]
-            + details["content_completeness"]["overall_completeness"]
-            * weights["content_completeness"]
-            + details["technical_accuracy"]["overall_accuracy"]
-            * weights["technical_accuracy"]
-            + details["clarity_score"]["overall_clarity"] * weights["clarity_score"]
-            + details["traceability"]["overall_traceability"] * weights["traceability"]
-        )
-
-        return round(overall_score, 2)
-
-    def _determine_quality_grade(self, score: float) -> str:
-        """Determine quality grade based on score."""
-        if score >= 0.9:
-            return "A"
-        elif score >= 0.8:
-            return "B"
-        elif score >= 0.7:
-            return "C"
-        elif score >= 0.6:
-            return "D"
-        else:
-            return "F"
-
-    def _check_quality_standards(self, validation_result: Dict[str, Any]) -> bool:
-        """Check if SPEC meets minimum quality standards."""
-        overall_score = validation_result["overall_score"]
-        ears_compliance = validation_result["details"]["ears_compliance"][
-            "overall_compliance"
-        ]
-
-        meets_overall = overall_score >= self.min_confidence_score
-        meets_ears = ears_compliance >= self.min_ears_compliance
-
-        return meets_overall and meets_ears
-
-    def _generate_recommendations(self, validation_result: Dict[str, Any]) -> List[str]:
-        """Generate improvement recommendations."""
-        recommendations = []
-        details = validation_result["details"]
-
-        # EARS compliance recommendations
-        if details["ears_compliance"]["overall_compliance"] < 1.0:
-            missing = details["ears_compliance"]["missing_sections"]
-            recommendations.append(
-                f"Add missing EARS sections: {', '.join(missing[:3])}"
-            )
-
-        # Content completeness recommendations
-        if details["content_completeness"]["overall_completeness"] < 0.8:
-            recommendations.append(
-                "Expand content sections with more detailed requirements"
-            )
-
-        # Technical accuracy recommendations
-        if details["technical_accuracy"]["overall_accuracy"] < 0.8:
-            recommendations.append("Add more technical specifications and details")
-
-        # Clarity recommendations
-        if details["clarity_score"]["overall_clarity"] < 0.7:
-            recommendations.append("Improve language clarity and reduce ambiguity")
-
-        # Traceability recommendations
-        if details["traceability"]["overall_traceability"] < 0.8:
-            recommendations.append("Add proper traceability tags and relationships")
-
-        # Limit recommendations
-        return recommendations[: self.max_review_suggestions]
-
-    def _compile_passed_checks(self, details: Dict[str, Any]) -> List[str]:
-        """Compile list of passed quality checks."""
-        passed = []
-
-        if details["ears_compliance"]["overall_compliance"] >= self.min_ears_compliance:
-            passed.append("EARS format compliance")
-
-        if details["content_completeness"]["overall_completeness"] >= 0.8:
-            passed.append("Content completeness")
-
-        if details["technical_accuracy"]["overall_accuracy"] >= 0.7:
-            passed.append("Technical accuracy")
-
-        if details["clarity_score"]["overall_clarity"] >= 0.7:
-            passed.append("Clarity and readability")
-
-        if details["traceability"]["overall_traceability"] >= 0.7:
-            passed.append("Traceability")
-
-        return passed
-
-    def _compile_failed_checks(self, details: Dict[str, Any]) -> List[str]:
-        """Compile list of failed quality checks."""
-        failed = []
-
-        if details["ears_compliance"]["overall_compliance"] < self.min_ears_compliance:
-            failed.append("EARS format compliance")
-
-        if details["content_completeness"]["overall_completeness"] < 0.8:
-            failed.append("Content completeness")
-
-        if details["technical_accuracy"]["overall_accuracy"] < 0.7:
-            failed.append("Technical accuracy")
-
-        if details["clarity_score"]["overall_clarity"] < 0.7:
-            failed.append("Clarity and readability")
-
-        if details["traceability"]["overall_traceability"] < 0.7:
-            failed.append("Traceability")
-
-        return failed
-
-    # Helper methods
-    def _check_heading_structure(self, spec_md: str) -> bool:
-        """Check if proper heading structure exists."""
-        # Look for proper markdown heading structure
-        heading_pattern = r"^#+\s+.*$"
-        headings = re.findall(heading_pattern, spec_md, re.MULTILINE)
-        return len(headings) >= 5  # At least 5 headings
-
-    def _assess_section_completeness(self, content: str) -> float:
-        """Assess completeness of a section."""
-        if len(content) < self.min_content_length:
-            return 0.0
-
-        # Check for key indicators of completeness
-        completeness_indicators = [
-            r"##\s+.*",  # Subheadings
-            r"\*\*.*\*\*",  # Bold text
-            r"-\s+.*",  # Lists
-            r"`[^`]+`",  # Code snippets
-            r"\d+\.",  # Numbered lists
-        ]
-
-        score = 0.0
-        for indicator in completeness_indicators:
-            matches = re.findall(indicator, content)
-            if matches:
-                score += 0.2
-
-        return min(score, 1.0)
-
-    def _assess_plan_completeness(self, plan_content: str) -> float:
-        """Assess completeness of implementation plan."""
-        if not plan_content:
-            return 0.0
-
-        # Check for plan-specific elements
-        plan_indicators = [
-            r"Phase",  # Phases
-            r"Priority",  # Priorities
-            r"Task",  # Tasks
-            r"\[\s*\]",  # Checkboxes
-            r"Phase 1",  # Phase indicators
-        ]
-
-        score = 0.0
-        for indicator in plan_indicators:
-            if re.search(indicator, plan_content):
-                score += 0.2
-
-        return min(score, 1.0)
-
-    def _assess_acceptance_criteria_completeness(
-        self, acceptance_content: str
-    ) -> float:
-        """Assess completeness of acceptance criteria."""
-        if not acceptance_content:
-            return 0.0
-
-        # Check for acceptance-specific elements
-        acceptance_indicators = [
-            r"Acceptance",  # Acceptance
-            r"Criteria",  # Criteria
-            r"Pass",  # Pass
-            r"Fail",  # Fail
-            r"Test",  # Test
-        ]
-
-        score = 0.0
-        for indicator in acceptance_indicators:
-            if re.search(indicator, acceptance_content):
-                score += 0.2
-
-        return min(score, 1.0)
-
-    def _check_technical_keywords(self, spec_md: str, keywords: List[str]) -> float:
-        """Check for presence of technical keywords."""
-        found_keywords = []
-        for keyword in keywords:
-            if keyword in spec_md:
-                found_keywords.append(keyword)
-
-        return len(found_keywords) / len(keywords)
-
-    def _check_code_consistency(
-        self, spec_md: str, code_analysis: Dict[str, Any]
-    ) -> float:
-        """Check consistency between SPEC and code analysis."""
-        # Extract key elements from code analysis
-        classes = code_analysis.get("structure_info", {}).get("classes", [])
-        functions = code_analysis.get("structure_info", {}).get("functions", [])
-
-        # Check if these elements are mentioned in the SPEC
-        class_mentions = sum(1 for cls in classes if cls in spec_md)
-        function_mentions = sum(1 for func in functions if func in spec_md)
-
-        total_elements = len(classes) + len(functions)
-        if total_elements == 0:
-            return 1.0
-
-        consistency = (class_mentions + function_mentions) / total_elements
-        return min(consistency, 1.0)
-
-    def _check_technical_specifications(self, spec_md: str) -> bool:
-        """Check for presence of technical specifications."""
-        technical_patterns = [
-            r"API\s+Endpoint",
-            r"Database\s+Schema",
-            r"Authentication",
-            r"Performance\s+Requirement",
-            r"Security\s+Requirement",
-            r"Scalability\s+Plan",
-            r"Testing\s+Strategy",
-        ]
-
-        return any(re.search(pattern, spec_md) for pattern in technical_patterns)
-
-    def _check_realistic_estimates(self, spec_md: str) -> bool:
-        """Check for realistic time/effort estimates."""
-        # Look for realistic time estimates
-        time_patterns = [r"1-2\s*days", r"3-5\s*days", r"1-2\s*weeks", r"\d+\s*hours"]
-
-        return any(re.search(pattern, spec_md) for pattern in time_patterns)
-
-    def _check_language_quality(self, spec_md: str) -> float:
-        """Check language quality."""
-        # Simple language quality checks
-        sentences = spec_md.split(".")
-        avg_sentence_length = sum(len(s.split()) for s in sentences) / len(sentences)
-
-        # Optimal sentence length is around 15-20 words
-        if 15 <= avg_sentence_length <= 25:
-            return 1.0
-        elif 10 <= avg_sentence_length <= 30:
-            return 0.8
-        else:
-            return 0.5
-
-    def _check_clarity_requirements(self, spec_md: str) -> float:
-        """Check clarity requirements."""
-        clarity_indicators = [
-            r"Clear",
-            r"Specific",
-            r"Measurable",
-            r"Achievable",
-            r"Relevant",
-        ]
-
-        found_indicators = sum(
-            1 for indicator in clarity_indicators if re.search(indicator, spec_md)
-        )
-
-        return min(found_indicators / len(clarity_indicators), 1.0)
-
-    def _check_ambiguity(self, spec_md: str) -> float:
-        """Check for ambiguous language."""
-        ambiguity_indicators = [
-            r"degree",
-            r"extent",
-            r"approximately",
-            r"about",
-            r"around",
-        ]
-
-        ambiguous_count = sum(
-            1 for indicator in ambiguity_indicators if re.search(indicator, spec_md)
-        )
-
-        # Normalize by content length
-        content_length = len(spec_md.split())
-        ambiguity_ratio = ambiguous_count / max(content_length / 100, 1)
-
-        return min(ambiguity_ratio, 1.0)
-
-    def _check_consistency(self, spec_md: str) -> float:
-        """Check for consistency in terminology."""
-        # Check for consistent terminology
-        # This is a simplified check - in practice, you'd use more sophisticated NLP
-        sentences = spec_md.split(".")
-        if len(sentences) < 2:
-            return 1.0
-
-        # Simple consistency check: look for repeated terms
-        words = spec_md.lower().split()
-        unique_words = set(words)
-        consistency_ratio = len(unique_words) / len(words)
-
-        return consistency_ratio
-
-    def _check_traceability_tags(self, spec_md: str) -> bool:
-        """Check for traceability tags."""
-        return any(tag in spec_md for tag in traceability_tags)
-
-    def _check_tag_formatting(self, spec_md: str) -> float:
-        """Check proper tag formatting."""
-        # Look for properly formatted tags
-        tag_pattern = r"@[A-Z]+:[A-Za-z0-9\-]+"
-        matches = re.findall(tag_pattern, spec_md)
-
-        # Count total tags to check ratio
-        total_tags = len(re.findall(r"@[A-Z]+:", spec_md))
-
-        if total_tags == 0:
-            return 1.0
-
-        return len(matches) / total_tags
-
-    def _check_traceability_relationships(self, spec_md: str) -> float:
-        """Check traceability relationships."""
-        # Look for traceability indicators
-        traceability_indicators = [
-            r"←|→",  # Arrows for relationships
-            r"Relationship",
-            r"Connect",
-            r"Trace",
-        ]
-
-        found_indicators = sum(
-            1 for indicator in traceability_indicators if re.search(indicator, spec_md)
-        )
-
-        return min(found_indicators / len(traceability_indicators), 1.0)
-
-    def _assess_acceptance_completeness(self, acceptance_content: str) -> float:
-        """Assess completeness of acceptance criteria section."""
-        if not acceptance_content:
-            return 0.0
-
-        # Check for acceptance-specific elements
-        acceptance_indicators = [
-            r"Acceptance",  # Acceptance
-            r"Criteria",  # Criteria
-            r"Pass",  # Pass
-            r"Fail",  # Fail
-            r"Test",  # Test
-        ]
-
-        score = 0.0
-        for indicator in acceptance_indicators:
-            if re.search(indicator, acceptance_content):
-                score += 0.2
-
-        return min(score, 1.0)
-
-    def generate_quality_report(self, validation_result: Dict[str, Any]) -> str:
-        """Generate a quality report string."""
-        report = f"""
-# Quality Validation Report
-
-## Overall Quality Score: {validation_result['overall_score']:.2f}/1.0
-## Quality Grade: {validation_result['quality_grade']}
-## Validation Time: {validation_result['validation_time']:.2f}s
-
-## Summary
-- Passed Checks: {len(validation_result['passed_checks'])}
-- Failed Checks: {len(validation_result['failed_checks'])}
-- Recommendations: {len(validation_result['recommendations'])}
-
-## Detailed Metrics
-"""
-
-        # Add detailed metrics
-        for metric_name, metric_value in validation_result["metrics"].items():
-            report += f"- {metric_name}: {metric_value:.2f}\n"
-
-        # Add recommendations
-        if validation_result["recommendations"]:
-            report += "\n## Recommendations\n"
-            for i, rec in enumerate(validation_result["recommendations"], 1):
-                report += f"{i}. {rec}\n"
-
-        # Add quality determination
-        meets_standards = validation_result.get("meets_minimum_standards", False)
-        status = "PASSED" if meets_standards else "FAILED"
-        report += f"\n## Quality Status: {status}\n"
-
-        return report