quantum-framework 0.18.0__py3-none-any.whl

quantum_admin/backend/test_generator.py

@@ -0,0 +1,782 @@
+"""
+Test Generator Service - Automatically generate test cases for Quantum components
+
+Features:
+- Analyze component metadata to generate test cases
+- Generate test files for functions, endpoints, and queries
+- Support pytest test format
+- Generate test stubs with assertions
+- Template-based test generation
+"""
+
+import os
+import re
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Any, Tuple
+from datetime import datetime
+from dataclasses import dataclass, field
+from sqlalchemy.orm import Session
+
+try:
+    from .models import Project, Component, Endpoint, TestRun, TestResult, ComponentTest
+    from .component_discovery import get_discovery_service, ComponentMetadata, DiscoveredFunction, DiscoveredEndpoint, DiscoveredQuery
+except ImportError:
+    from models import Project, Component, Endpoint, TestRun, TestResult, ComponentTest
+    from component_discovery import get_discovery_service, ComponentMetadata, DiscoveredFunction, DiscoveredEndpoint, DiscoveredQuery
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GeneratedTest:
+    """Represents a generated test case"""
+    name: str
+    description: str
+    test_type: str  # unit, integration, api, e2e
+    file_path: str
+    content: str
+    component_name: Optional[str] = None
+    target_function: Optional[str] = None
+    target_endpoint: Optional[str] = None
+    assertions: List[str] = field(default_factory=list)
+
+
+@dataclass
+class TestSuite:
+    """Collection of generated tests"""
+    name: str
+    component_name: str
+    tests: List[GeneratedTest] = field(default_factory=list)
+    setup_code: str = ""
+    teardown_code: str = ""
+
+
+class TestTemplates:
+    """Test code templates for different test types"""
+
+    # File header template
+    FILE_HEADER = '''"""
+Auto-generated tests for {component_name}
+Generated by Quantum Admin Test Generator
+Date: {date}
+"""
+
+import pytest
+from pathlib import Path
+import sys
+
+# Add project root to path
+PROJECT_ROOT = Path(__file__).parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+'''
+
+    # Function test template
+    FUNCTION_TEST = '''
+def test_{function_name}_basic():
+    """Test {function_name} with basic inputs"""
+    # Arrange
+    {setup_code}
+
+    # Act
+    result = {function_call}
+
+    # Assert
+    assert result is not None
+    {assertions}
+
+
+'''
+
+    # Function with params test template
+    FUNCTION_PARAMS_TEST = '''
+@pytest.mark.parametrize("input_value,expected", [
+    {param_cases}
+])
+def test_{function_name}_params(input_value, expected):
+    """Test {function_name} with various inputs"""
+    # Arrange
+    {setup_code}
+
+    # Act
+    result = {function_call}
+
+    # Assert
+    assert result == expected
+
+
+'''
+
+    # Endpoint test template
+    ENDPOINT_TEST = '''
+def test_{endpoint_name}_{method}_basic(client):
+    """Test {method} {path}"""
+    # Arrange
+    {setup_code}
+
+    # Act
+    response = client.{method_lower}("{path}")
+
+    # Assert
+    assert response.status_code == 200
+    {assertions}
+
+
+'''
+
+    # Endpoint with body template
+    ENDPOINT_POST_TEST = '''
+def test_{endpoint_name}_{method}_with_data(client):
+    """Test {method} {path} with request body"""
+    # Arrange
+    data = {{
+        {request_body}
+    }}
+
+    # Act
+    response = client.{method_lower}("{path}", json=data)
+
+    # Assert
+    assert response.status_code in [200, 201]
+    {assertions}
+
+
+'''
+
+    # Query test template
+    QUERY_TEST = '''
+def test_{query_name}_query(db_session):
+    """Test {query_name} database query"""
+    # Arrange
+    {setup_code}
+
+    # Act
+    result = execute_query("{query_name}", db_session)
+
+    # Assert
+    assert result is not None
+    assert len(result) >= 0
+    {assertions}
+
+
+'''
+
+    # Fixtures template
+    FIXTURES = '''
+@pytest.fixture
+def client():
+    """Create test client"""
+    from quantum.runtime.web_server import create_app
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as client:
+        yield client
+
+
+@pytest.fixture
+def db_session():
+    """Create database session for testing"""
+    from quantum.runtime.database_service import get_db_session
+    session = get_db_session()
+    yield session
+    session.rollback()
+    session.close()
+
+
+'''
+
+    # Conftest template
+    CONFTEST = '''"""
+Pytest configuration and fixtures for {project_name}
+Auto-generated by Quantum Admin Test Generator
+"""
+
+import pytest
+import sys
+from pathlib import Path
+
+# Add project root to path
+PROJECT_ROOT = Path(__file__).parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+@pytest.fixture(scope="session")
+def project_root():
+    """Return project root path"""
+    return PROJECT_ROOT
+
+
+@pytest.fixture
+def sample_data():
+    """Provide sample test data"""
+    return {{
+        "string": "test_value",
+        "number": 42,
+        "boolean": True,
+        "list": [1, 2, 3],
+        "dict": {{"key": "value"}}
+    }}
+
+
+'''
+
+
+class TestGeneratorService:
+    """Service for generating tests for Quantum components"""
+
+    def __init__(self, db: Session):
+        self.db = db
+        self.discovery = get_discovery_service(db)
+        self.templates = TestTemplates()
+
+    def generate_tests_for_component(
+        self,
+        project_id: int,
+        component_id: int,
+        output_dir: Optional[str] = None,
+        test_types: Optional[List[str]] = None,
+        save_to_db: bool = True
+    ) -> TestSuite:
+        """
+        Generate tests for a specific component
+
+        Args:
+            project_id: Project ID
+            component_id: Component ID
+            output_dir: Directory to write test files
+            test_types: List of test types to generate (unit, integration, api)
+            save_to_db: Whether to save test records to database
+
+        Returns:
+            TestSuite with generated tests
+        """
+        # Get component metadata
+        metadata = self.discovery.get_component_details(project_id, component_id)
+        if not metadata:
+            raise ValueError(f"Component {component_id} not found")
+
+        if test_types is None:
+            test_types = ["unit", "integration"]
+
+        suite = TestSuite(
+            name=f"test_{metadata.name.lower()}",
+            component_name=metadata.name
+        )
+
+        # Generate function tests
+        if "unit" in test_types:
+            for func in metadata.functions:
+                test = self._generate_function_test(metadata, func)
+                suite.tests.append(test)
+
+        # Generate endpoint tests
+        if "api" in test_types or "integration" in test_types:
+            for endpoint in metadata.endpoints:
+                test = self._generate_endpoint_test(metadata, endpoint)
+                suite.tests.append(test)
+
+        # Generate query tests
+        if "integration" in test_types:
+            for query in metadata.queries:
+                test = self._generate_query_test(metadata, query)
+                suite.tests.append(test)
+
+        # Write to files if output_dir specified
+        if output_dir:
+            self._write_test_files(suite, output_dir)
+
+        # Save test records to database
+        if save_to_db and suite.tests:
+            self._save_component_tests(component_id, suite, output_dir)
+
+        return suite
+
+    def _save_component_tests(
+        self,
+        component_id: int,
+        suite: TestSuite,
+        output_dir: Optional[str] = None
+    ):
+        """Save generated tests to database"""
+        # Get the component
+        component = self.db.query(Component).filter(
+            Component.id == component_id
+        ).first()
+
+        if not component:
+            return
+
+        # Get existing tests for this component
+        existing = self.db.query(ComponentTest).filter(
+            ComponentTest.component_id == component_id
+        ).all()
+        existing_names = {t.test_name for t in existing}
+
+        tests_created = 0
+        for test in suite.tests:
+            if test.name not in existing_names:
+                # Build the full file path
+                file_path = test.file_path
+                if output_dir:
+                    file_path = str(Path(output_dir) / test.file_path)
+
+                component_test = ComponentTest(
+                    component_id=component_id,
+                    test_file=file_path,
+                    test_name=test.name,
+                    test_type=test.test_type,
+                    generated_at=datetime.now(),
+                    generated_by="auto",
+                    last_status="pending"
+                )
+                self.db.add(component_test)
+                tests_created += 1
+
+        # Update component test count
+        component.test_count = len(existing) + tests_created
+
+        self.db.commit()
+        logger.info(f"Created {tests_created} test records for component {component.name}")
+
+    def generate_tests_for_project(
+        self,
+        project_id: int,
+        base_path: Optional[str] = None,
+        output_dir: Optional[str] = None,
+        test_types: Optional[List[str]] = None
+    ) -> List[TestSuite]:
+        """
+        Generate tests for all components in a project
+
+        Args:
+            project_id: Project ID
+            base_path: Base path to scan for components
+            output_dir: Directory to write test files
+            test_types: List of test types to generate
+
+        Returns:
+            List of TestSuites for each component
+        """
+        # Discover components
+        components = self.discovery.discover_components(project_id, base_path)
+
+        suites = []
+        for metadata in components:
+            if not metadata.is_valid:
+                logger.warning(f"Skipping invalid component: {metadata.name}")
+                continue
+
+            suite = self._generate_suite_from_metadata(metadata, test_types)
+            if suite.tests:
+                suites.append(suite)
+
+        # Write to files if output_dir specified
+        if output_dir:
+            self._write_project_tests(suites, output_dir)
+
+        return suites
+
+    def generate_conftest(
+        self,
+        project_id: int,
+        output_dir: Optional[str] = None
+    ) -> str:
+        """Generate conftest.py with common fixtures"""
+        project = self.db.query(Project).filter(Project.id == project_id).first()
+        if not project:
+            raise ValueError(f"Project {project_id} not found")
+
+        content = self.templates.CONFTEST.format(
+            project_name=project.name
+        )
+
+        if output_dir:
+            path = Path(output_dir) / "conftest.py"
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(content, encoding='utf-8')
+
+        return content
+
+    def _generate_suite_from_metadata(
+        self,
+        metadata: ComponentMetadata,
+        test_types: Optional[List[str]] = None
+    ) -> TestSuite:
+        """Generate test suite from component metadata"""
+        if test_types is None:
+            test_types = ["unit", "integration"]
+
+        suite = TestSuite(
+            name=f"test_{self._sanitize_name(metadata.name)}",
+            component_name=metadata.name
+        )
+
+        # Generate function tests
+        if "unit" in test_types:
+            for func in metadata.functions:
+                test = self._generate_function_test(metadata, func)
+                suite.tests.append(test)
+
+        # Generate endpoint tests
+        if "api" in test_types or "integration" in test_types:
+            for endpoint in metadata.endpoints:
+                test = self._generate_endpoint_test(metadata, endpoint)
+                suite.tests.append(test)
+
+        # Generate query tests
+        if "integration" in test_types:
+            for query in metadata.queries:
+                test = self._generate_query_test(metadata, query)
+                suite.tests.append(test)
+
+        return suite
+
+    def _generate_function_test(
+        self,
+        metadata: ComponentMetadata,
+        func: DiscoveredFunction
+    ) -> GeneratedTest:
+        """Generate test for a function"""
+        func_name = self._sanitize_name(func.name)
+
+        # Build function call
+        if func.params:
+            param_values = ", ".join([f"{p}=sample_value" for p in func.params])
+            function_call = f"{func.name}({param_values})"
+        else:
+            function_call = f"{func.name}()"
+
+        # Generate assertions based on return type
+        assertions = []
+        if func.return_type:
+            if func.return_type.lower() in ['string', 'str']:
+                assertions.append("assert isinstance(result, str)")
+            elif func.return_type.lower() in ['int', 'integer', 'number']:
+                assertions.append("assert isinstance(result, (int, float))")
+            elif func.return_type.lower() in ['list', 'array']:
+                assertions.append("assert isinstance(result, list)")
+            elif func.return_type.lower() in ['dict', 'object']:
+                assertions.append("assert isinstance(result, dict)")
+            elif func.return_type.lower() in ['bool', 'boolean']:
+                assertions.append("assert isinstance(result, bool)")
+
+        content = self.templates.FUNCTION_TEST.format(
+            function_name=func_name,
+            setup_code="sample_value = 'test'",
+            function_call=function_call,
+            assertions="\n    ".join(assertions) if assertions else "# Add specific assertions"
+        )
+
+        return GeneratedTest(
+            name=f"test_{func_name}_basic",
+            description=f"Test {func.name} function",
+            test_type="unit",
+            file_path=f"test_{self._sanitize_name(metadata.name)}.py",
+            content=content,
+            component_name=metadata.name,
+            target_function=func.name,
+            assertions=assertions
+        )
+
+    def _generate_endpoint_test(
+        self,
+        metadata: ComponentMetadata,
+        endpoint: DiscoveredEndpoint
+    ) -> GeneratedTest:
+        """Generate test for an endpoint"""
+        # Sanitize endpoint name for test function
+        endpoint_name = self._sanitize_name(endpoint.path.replace('/', '_'))
+        method = endpoint.method.upper()
+        method_lower = method.lower()
+
+        # Choose template based on method
+        if method in ['POST', 'PUT', 'PATCH']:
+            content = self.templates.ENDPOINT_POST_TEST.format(
+                endpoint_name=endpoint_name,
+                method=method,
+                method_lower=method_lower,
+                path=endpoint.path,
+                setup_code="",
+                request_body='"key": "value"',
+                assertions=""
+            )
+        else:
+            content = self.templates.ENDPOINT_TEST.format(
+                endpoint_name=endpoint_name,
+                method=method,
+                method_lower=method_lower,
+                path=endpoint.path,
+                setup_code="",
+                assertions=""
+            )
+
+        return GeneratedTest(
+            name=f"test_{endpoint_name}_{method_lower}",
+            description=f"Test {method} {endpoint.path}",
+            test_type="api",
+            file_path=f"test_{self._sanitize_name(metadata.name)}.py",
+            content=content,
+            component_name=metadata.name,
+            target_endpoint=endpoint.path,
+            assertions=[]
+        )
+
+    def _generate_query_test(
+        self,
+        metadata: ComponentMetadata,
+        query: DiscoveredQuery
+    ) -> GeneratedTest:
+        """Generate test for a database query"""
+        query_name = self._sanitize_name(query.name)
+
+        content = self.templates.QUERY_TEST.format(
+            query_name=query_name,
+            setup_code="# Set up test data" if query.query_type != "select" else "",
+            assertions=""
+        )
+
+        return GeneratedTest(
+            name=f"test_{query_name}_query",
+            description=f"Test {query.name} query",
+            test_type="integration",
+            file_path=f"test_{self._sanitize_name(metadata.name)}.py",
+            content=content,
+            component_name=metadata.name,
+            assertions=[]
+        )
+
+    def _write_test_files(self, suite: TestSuite, output_dir: str):
+        """Write test suite to files"""
+        output_path = Path(output_dir)
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        # Group tests by file
+        file_tests: Dict[str, List[GeneratedTest]] = {}
+        for test in suite.tests:
+            if test.file_path not in file_tests:
+                file_tests[test.file_path] = []
+            file_tests[test.file_path].append(test)
+
+        # Write each file
+        for file_path, tests in file_tests.items():
+            full_path = output_path / file_path
+            content = self.templates.FILE_HEADER.format(
+                component_name=suite.component_name,
+                date=datetime.now().isoformat()
+            )
+            content += self.templates.FIXTURES
+
+            for test in tests:
+                content += test.content
+
+            full_path.write_text(content, encoding='utf-8')
+            logger.info(f"Generated test file: {full_path}")
+
+    def _write_project_tests(self, suites: List[TestSuite], output_dir: str):
+        """Write all test suites for a project"""
+        output_path = Path(output_dir)
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        for suite in suites:
+            suite_path = output_path / f"test_{self._sanitize_name(suite.component_name)}.py"
+
+            content = self.templates.FILE_HEADER.format(
+                component_name=suite.component_name,
+                date=datetime.now().isoformat()
+            )
+            content += self.templates.FIXTURES
+
+            for test in suite.tests:
+                content += test.content
+
+            suite_path.write_text(content, encoding='utf-8')
+            logger.info(f"Generated test file: {suite_path}")
+
+    def _sanitize_name(self, name: str) -> str:
+        """Sanitize a name for use in test function names"""
+        # Remove invalid characters
+        sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', name)
+        # Remove leading digits
+        sanitized = re.sub(r'^[0-9]+', '', sanitized)
+        # Convert to lowercase
+        sanitized = sanitized.lower()
+        # Remove consecutive underscores
+        sanitized = re.sub(r'_+', '_', sanitized)
+        # Remove trailing underscores
+        sanitized = sanitized.strip('_')
+        return sanitized or 'unnamed'
+
+    def get_test_coverage_estimate(
+        self,
+        project_id: int,
+        base_path: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Estimate test coverage based on component analysis
+
+        Returns:
+            Dict with coverage estimates
+        """
+        components = self.discovery.discover_components(project_id, base_path)
+
+        total_functions = 0
+        total_endpoints = 0
+        total_queries = 0
+        total_components = 0
+
+        for metadata in components:
+            if metadata.is_valid:
+                total_components += 1
+                total_functions += len(metadata.functions)
+                total_endpoints += len(metadata.endpoints)
+                total_queries += len(metadata.queries)
+
+        # Calculate potential test counts
+        potential_function_tests = total_functions
+        potential_endpoint_tests = total_endpoints
+        potential_query_tests = total_queries
+        potential_total = potential_function_tests + potential_endpoint_tests + potential_query_tests
+
+        return {
+            "total_components": total_components,
+            "total_functions": total_functions,
+            "total_endpoints": total_endpoints,
+            "total_queries": total_queries,
+            "potential_tests": {
+                "function_tests": potential_function_tests,
+                "endpoint_tests": potential_endpoint_tests,
+                "query_tests": potential_query_tests,
+                "total": potential_total
+            },
+            "recommendation": self._get_test_recommendation(
+                total_functions, total_endpoints, total_queries
+            )
+        }
+
+    def _get_test_recommendation(
+        self,
+        functions: int,
+        endpoints: int,
+        queries: int
+    ) -> str:
+        """Get testing recommendation based on component analysis"""
+        if endpoints > 0:
+            return "Focus on API/integration tests for endpoints first"
+        elif functions > 0:
+            return "Focus on unit tests for functions"
+        elif queries > 0:
+            return "Focus on database integration tests"
+        else:
+            return "Add q:function or q:endpoint tags to components for better test coverage"
+
+    def update_test_results(
+        self,
+        component_id: int,
+        test_run_id: int,
+        test_results: List[Dict[str, Any]]
+    ):
+        """
+        Update component test records with results from a test run
+
+        Args:
+            component_id: Component ID
+            test_run_id: TestRun ID
+            test_results: List of test result dicts with name, status, duration, error
+        """
+        # Get the component
+        component = self.db.query(Component).filter(
+            Component.id == component_id
+        ).first()
+
+        if not component:
+            return
+
+        # Get all tests for this component
+        tests = self.db.query(ComponentTest).filter(
+            ComponentTest.component_id == component_id
+        ).all()
+
+        tests_map = {t.test_name: t for t in tests}
+
+        passed = 0
+        failed = 0
+
+        for result in test_results:
+            test_name = result.get("name")
+            if test_name and test_name in tests_map:
+                test = tests_map[test_name]
+                test.last_run_id = test_run_id
+                test.last_status = result.get("status", "unknown")
+                test.last_run_at = datetime.now()
+                test.last_duration_seconds = result.get("duration")
+                test.last_error_message = result.get("error")
+
+                if test.last_status == "passed":
+                    passed += 1
+                elif test.last_status in ["failed", "error"]:
+                    failed += 1
+
+        # Update component counters
+        component.tests_passing = passed
+        component.tests_failing = failed
+        component.last_test_run_id = test_run_id
+        component.last_test_run_at = datetime.now()
+
+        self.db.commit()
+        logger.info(f"Updated test results for component {component.name}: {passed} passed, {failed} failed")
+
+    def get_component_test_summary(
+        self,
+        component_id: int
+    ) -> Dict[str, Any]:
+        """Get test summary for a component"""
+        component = self.db.query(Component).filter(
+            Component.id == component_id
+        ).first()
+
+        if not component:
+            return {}
+
+        tests = self.db.query(ComponentTest).filter(
+            ComponentTest.component_id == component_id
+        ).all()
+
+        by_status = {"passed": 0, "failed": 0, "pending": 0, "skipped": 0, "error": 0}
+        by_type = {"unit": 0, "api": 0, "integration": 0, "e2e": 0}
+
+        for test in tests:
+            status = test.last_status or "pending"
+            if status in by_status:
+                by_status[status] += 1
+            if test.test_type in by_type:
+                by_type[test.test_type] += 1
+
+        return {
+            "component_id": component_id,
+            "component_name": component.name,
+            "total_tests": len(tests),
+            "tests_passing": component.tests_passing,
+            "tests_failing": component.tests_failing,
+            "by_status": by_status,
+            "by_type": by_type,
+            "last_run_at": component.last_test_run_at.isoformat() if component.last_test_run_at else None,
+            "coverage": component.test_coverage if hasattr(component, 'test_coverage') else 0
+        }
+
+
+# Singleton instance
+_test_generator = None
+
+
+def get_test_generator(db: Session = None) -> TestGeneratorService:
+    """Get test generator service instance"""
+    if db:
+        return TestGeneratorService(db)
+
+    global _test_generator
+    if _test_generator is None:
+        from .database import SessionLocal
+        _test_generator = TestGeneratorService(SessionLocal())
+    return _test_generator