QuantumChecker 0.2.8__tar.gz → 0.2.9__tar.gz

quantumchecker-0.2.9/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: QuantumChecker
+Version: 0.2.9
+Summary: A package to evaluate homework submissions in Python, SQL, PowerBI, and SSIS.
+Author: Qobiljon
+Author-email: qobiljonkhayrullayev@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.31.0
+Requires-Dist: tenacity>=8.2.3
+Requires-Dist: pdf2image>=1.16.3
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: Pillow>=10.0.0
+Requires-Dist: PyPDF2>=3.0.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+Sample usage: 
+```
+import asyncio
+from your_evaluator_module import HomeworkEvaluator
+
+async def main():
+    evaluator = HomeworkEvaluator()
+    question_content = """
+Q1: What is a Python list? Explain with an example.
+
+Q2: Write an SQL query to select all records from a table named 'students'.
+"""
+    answer_path = "sample_submissions/student1_answer.py"
+    api_keys = ["your_api_key_1", "your_api_key_2"]
+    question_type = "python"
+
+    result = await evaluator.evaluate_from_content(
+        question_content=question_content,
+        answer_path=answer_path,
+        api_keys=api_keys,
+        question_type=question_type
+    )
+    print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```

quantumchecker-0.2.9/QuantumCheck/main.py

@@ -0,0 +1,222 @@
+import logging
+import os
+import zipfile
+import random
+from datetime import datetime
+from typing import List, Dict, Optional
+from .python_evaluator import PythonEvaluator
+from .sql_evaluator import SQLEvaluator
+from .powerbi_evaluator import PowerBIEvaluator
+from .ssis_evaluator import SSISEvaluator
+import asyncio
+
+_logger_cache = {}
+
+class HomeworkEvaluator:
+    EVALUATOR_REGISTRY = {
+        "python": PythonEvaluator,
+        "sql": SQLEvaluator,
+        "powerbi": PowerBIEvaluator,
+        "ssis": SSISEvaluator
+    }
+
+    EXTENSION_TO_TYPE = {
+        ".py": "python",
+        ".sql": "sql",
+        ".pbit": "powerbi",
+        ".pdf": "powerbi",
+        ".dtsx": "ssis",
+        ".DTSX": "ssis",
+        ".txt": "text",
+        ".md": "text"
+    }
+
+    API_NAME_MAPPING = {
+        "python": "Google Gemini API",
+        "sql": "Google Gemini API",
+        "powerbi": "Google Gemini API",
+        "ssis": "Google Gemini API",
+        "text": "Google Gemini API"
+    }
+
+    def __init__(self, log_level: int = logging.INFO):
+        self.log_level = log_level
+        self._successful_key_cache = {}
+        self._rate_limit_delay = {}  # Track delay per key
+
+    def _get_logger(self, log_type: str) -> logging.Logger:
+        log_name = f"{log_type}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
+        if log_name not in _logger_cache:
+            logger = logging.getLogger(log_name)
+            logger.setLevel(self.log_level)
+            if not logger.handlers:
+                handler = logging.StreamHandler()
+                handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+                logger.addHandler(handler)
+            _logger_cache[log_name] = logger
+        return _logger_cache[log_name]
+
+    def parse_questions(self, content: str) -> List[str]:
+        logger = self._get_logger("QuantumCheck.main")
+        questions = [q.strip() for q in content.split("\n\n") if q.strip()]
+        logger.info(f"Parsed {len(questions)} questions from content")
+        if not questions:
+            raise ValueError("No valid questions found in content")
+        return questions
+
+    def _detect_zip_content_type(self, zip_path: str, logger: logging.Logger) -> str:
+        try:
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                extensions = {os.path.splitext(name)[1].lower() for name in zip_ref.namelist()}
+                file_types = [self.EXTENSION_TO_TYPE.get(ext, "text") for ext in extensions if ext]
+                logger.info(f"Detected extensions in ZIP {zip_path}: {extensions}, types: {file_types}")
+                if "python" in file_types:
+                    logger.info(f"Selected file type: python from extension: .py in ZIP: {zip_path}")
+                    return "python"
+                elif "sql" in file_types:
+                    logger.info(f"Selected file type: sql from extension: .sql in ZIP: {zip_path}")
+                    return "sql"
+                elif "powerbi" in file_types:
+                    logger.info(f"Selected file type: powerbi from extension: .pbit or .pdf in ZIP: {zip_path}")
+                    return "powerbi"
+                elif "ssis" in file_types:
+                    logger.info(f"Selected file type: ssis from extension: .dtsx in ZIP: {zip_path}")
+                    return "ssis"
+                else:
+                    logger.info(f"Selected file type: text (default) in ZIP: {zip_path}")
+                    return "text"
+        except zipfile.BadZipFile:
+            logger.error(f"Invalid ZIP file: {zip_path}")
+            raise ValueError(f"Invalid ZIP file: {zip_path}")
+
+    async def evaluate_from_content(
+            self,
+            question_content: str,
+            answer_path: str,
+            api_keys: List[str],
+            question_type: str
+    ) -> Dict[str, any]:
+        try:
+            questions = self.parse_questions(question_content)
+        except ValueError as e:
+            logger = self._get_logger("QuantumCheck.main")
+            logger.error("Failed to parse question content: %s", str(e))
+            return {
+                "score": 0,
+                "feedback": f"Error parsing question content: {str(e)}",
+                "issues": [str(e)],
+                "recommendations": [],
+                "used_api_key_index": None,
+                "used_api_name": None
+            }
+
+        answer_path = answer_path.strip()
+        _, ext = os.path.splitext(answer_path)
+        ext = ext.lower()
+
+        # Determine file type, prioritizing question_type for evaluator selection
+        if ext == ".zip":
+            logger = self._get_logger("zip")
+            file_type = self._detect_zip_content_type(answer_path, logger)
+        else:
+            file_type = self.EXTENSION_TO_TYPE.get(ext, "text")
+            logger = self._get_logger(file_type)
+
+        # Use question_type if provided, else fallback to file_type
+        eval_type = question_type if question_type in self.EVALUATOR_REGISTRY else file_type
+        logger.info(f"Processing answer_path: {answer_path} with detected file type: {file_type}, evaluation type: {eval_type}")
+
+        if not os.path.exists(answer_path):
+            logger.error(f"Answer file not found: {answer_path}")
+            return {
+                "score": 0,
+                "feedback": f"Answer file not found: {answer_path}",
+                "issues": [f"Answer file not found: {answer_path}"],
+                "recommendations": [],
+                "used_api_key_index": None,
+                "used_api_name": None
+            }
+
+        evaluator_class = self.EVALUATOR_REGISTRY.get(eval_type, PythonEvaluator)
+        last_error_messages = []
+
+        # Shuffle keys for load balancing
+        key_order = [(i + 1, key) for i, key in enumerate(api_keys)]
+        random.shuffle(key_order)
+
+        # Try cached key with 30% probability to encourage rotation
+        cached_key_idx = self._successful_key_cache.get(eval_type)
+        if cached_key_idx is not None and cached_key_idx < len(api_keys) and random.random() < 0.3:
+            key_order.insert(0, (cached_key_idx + 1, api_keys[cached_key_idx]))
+
+        for idx, key in key_order:
+            # Check rate limit delay
+            if key in self._rate_limit_delay:
+                delay_until = self._rate_limit_delay[key]
+                current_time = datetime.now()
+                delay_until_time = datetime.fromtimestamp(delay_until)
+                if current_time < delay_until_time:
+                    logger.info(f"API key #{idx} is rate-limited until {delay_until_time}, skipping.")
+                    continue
+                else:
+                    del self._rate_limit_delay[key]
+
+            logger.info(f"Trying API key #{idx}")
+            evaluator = evaluator_class(key)
+            api_name = getattr(evaluator, 'get_api_name', lambda: self.API_NAME_MAPPING.get(eval_type, "Unknown API"))()
+            logger.info(f"Using API: {api_name} for evaluation type: {eval_type}")
+
+            try:
+                evaluation = evaluator.evaluate(questions, answer_path, temp_dir=f"temp_extract_{os.getpid()}_{idx}")
+
+                feedback = evaluation.get("feedback", "").lower()
+                issues = " ".join(evaluation.get("issues", [])).lower()
+
+                # Check for invalid API key
+                if any(phrase in feedback or phrase in issues for phrase in ["api key not valid", "api_key_invalid"]):
+                    logger.warning(f"API key #{idx} invalid, trying next key.")
+                    last_error_messages.append(f"API key #{idx} invalid.")
+                    continue
+
+                # Check for rate limit errors
+                if any(phrase in feedback or phrase in issues for phrase in ["429", "too many requests", "rate limit"]):
+                    logger.warning(f"API key #{idx} hit rate limit, applying delay.")
+                    last_error_messages.append(f"API key #{idx} rate limited.")
+                    self._rate_limit_delay[key] = datetime.now().timestamp() + 45  # 45s delay
+                    continue
+
+                # Check for invalid evaluation
+                if evaluation.get("score", 0) == 0 and "evaluation not returned" in feedback:
+                    logger.warning(f"API key #{idx} returned invalid evaluation, trying next key.")
+                    last_error_messages.append(f"API key #{idx} returned invalid evaluation.")
+                    continue
+
+                # Cache successful key
+                self._successful_key_cache[eval_type] = idx - 1
+                logger.info(f"Evaluation succeeded with API key #{idx}: Score = {evaluation.get('score')}")
+
+                return {
+                    "score": evaluation.get("score", 0),
+                    "feedback": evaluation.get("feedback", "No feedback provided"),
+                    "issues": evaluation.get("issues", []),
+                    "recommendations": evaluation.get("recommendations", []),
+                    "used_api_key_index": idx,
+                    "used_api_name": api_name
+                }
+
+            except Exception as e:
+                logger.error(f"Exception using API key #{idx}: {str(e)}")
+                last_error_messages.append(f"Exception with key #{idx}: {str(e)}")
+                if "429" in str(e) or "rate limit" in str(e).lower():
+                    self._rate_limit_delay[key] = datetime.now().timestamp() + 45
+                continue
+
+        logger.error("Evaluation failed with all API keys.")
+        return {
+            "score": 0,
+            "feedback": "Evaluation failed with all API keys.",
+            "issues": last_error_messages if last_error_messages else ["All API keys failed to evaluate the submission."],
+            "recommendations": [],
+            "used_api_key_index": None,
+            "used_api_name": None
+        }

{quantumchecker-0.2.8 → quantumchecker-0.2.9}/QuantumCheck/powerbi_evaluator.py

@@ -285,11 +285,11 @@ class PowerBIEvaluator:
         self.model = GeminiFlashModel(api_key)
         self.processor = PowerBIProcessor()
 
-    def evaluate(self, questions: List[str], answer_path: str) -> Dict[str, any]:
+    def evaluate(self, questions: List[str], answer_path: str, temp_dir: str = "temp_extract") -> Dict[str, any]:
         try:
             _, ext = os.path.splitext(answer_path)
             ext = ext.lower()
-            extract_path = os.path.join(os.path.dirname(answer_path), "temp_extract")
+            extract_path = temp_dir
             pbit_path = None
             pdf_path = None
             if ext == ".zip":
@@ -322,8 +322,8 @@ class PowerBIEvaluator:
                 }
                 if pdf_path:
                     try:
-                        self.processor.process_pdf(pdf_path)
-                        visual_result = self.model.evaluate_visuals(questions[0], "outputimages")
+                        image_paths = self.processor.process_pdf(pdf_path, output_dir=os.path.join(temp_dir, "outputimages"))
+                        visual_result = self.model.evaluate_visuals(questions[0], os.path.join(temp_dir, "outputimages"))
                         result["score"] = int(0.7 * dax_result["score"] + 0.3 * visual_result["score"])
                         result["visual_score"] = visual_result["score"]
                         result["feedback"] += f"\n\nVisual Feedback:\n{visual_result['feedback']}"
@@ -344,10 +344,10 @@ class PowerBIEvaluator:
                 logger.info("[Final] Score (70%% DAX, 30%% Visuals): %d/100", result["score"])
                 return result
             finally:
-                self.processor._cleanup(extract_path, "outputimages")
+                self.processor._cleanup(extract_path, os.path.join(temp_dir, "outputimages"))
         except Exception as e:
             logger.exception("Failed to evaluate Power BI file %s: %s", answer_path, str(e))
-            self.processor._cleanup(extract_path, "outputimages")
+            self.processor._cleanup(extract_path, os.path.join(temp_dir, "outputimages"))
             return {
                 "score": 0,
                 "feedback": f"Error processing file: {str(e)}",
@@ -359,4 +359,4 @@ class PowerBIEvaluator:
 
 
 class ProcessingError(Exception):
-    pass
+    pass

{quantumchecker-0.2.8 → quantumchecker-0.2.9}/QuantumCheck/python_evaluator.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import zipfile
+import shutil
 from pprint import pprint
 from typing import List, Dict
 
@@ -107,50 +108,60 @@ class PythonAnswerParser:
         return answers
 
     @staticmethod
-    def parse_zip_file(zip_path: str) -> List[str]:
+    def parse_zip_file(zip_path: str, temp_dir: str) -> List[str]:
+        """
+        Parse Python files from a ZIP file, extracting to the specified temp_dir.
+
+        Args:
+            zip_path: Path to the ZIP file
+            temp_dir: Directory to extract ZIP contents
+
+        Returns:
+            List of answer strings extracted from Python files
+        """
         combined_content = []
 
         try:
+            # Create temporary extraction directory
+            os.makedirs(temp_dir, exist_ok=True)
             with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                temp_dir = "temp_python_extract"
-                os.makedirs(temp_dir, exist_ok=True)
                 zip_ref.extractall(temp_dir)
 
-                python_files = sorted(
-                    [f for f in os.listdir(temp_dir) if f.endswith(".py")]
-                )
+            python_files = sorted(
+                [f for f in os.listdir(temp_dir) if f.endswith(".py")]
+            )
 
-                if not python_files:
-                    logger.warning("No Python files found in ZIP")
-                    return []
+            if not python_files:
+                logger.warning(f"No Python files found in ZIP: {zip_path}")
+                return []
 
-                for python_file in python_files:
-                    with open(
+            for python_file in python_files:
+                with open(
                         os.path.join(temp_dir, python_file),
                         "r",
                         encoding="utf-8",
-                    ) as f:
-                        content = f.read().strip()
-                        if content:
-                            combined_content.append(content)
-
-                for f in os.listdir(temp_dir):
-                    os.remove(os.path.join(temp_dir, f))
+                ) as f:
+                    content = f.read().strip()
+                    if content:
+                        combined_content.append(content)
 
-                os.rmdir(temp_dir)
+            if not combined_content:
+                logger.warning(f"No valid content found in Python files in ZIP: {zip_path}")
+                return []
 
-                if not combined_content:
-                    logger.warning("No valid content found in Python files")
-                    return []
-
-                combined_text = "\n\n".join(combined_content)
-                return [a.strip() for a in combined_text.split("\n\n") if a.strip()]
+            combined_text = "\n\n".join(combined_content)
+            return [a.strip() for a in combined_text.split("\n\n") if a.strip()]
         except zipfile.BadZipFile:
-            logger.error("Invalid ZIP file: %s", zip_path)
+            logger.error(f"Invalid ZIP file: {zip_path}")
             return []
         except Exception as e:
-            logger.error("Error processing ZIP file %s: %s", zip_path, str(e))
+            logger.error(f"Error processing ZIP file {zip_path}: {str(e)}")
             return []
+        finally:
+            # Clean up temporary directory
+            if os.path.exists(temp_dir):
+                shutil.rmtree(temp_dir, ignore_errors=True)
+                logger.info(f"Cleaned up temporary directory: {temp_dir}")
 
 
 class PythonEvaluator:
@@ -158,19 +169,30 @@ class PythonEvaluator:
         self.api_key = api_key
         self.model = GeminiFlashModel(api_key)
 
-    def evaluate(self, questions: List[str], answer_path: str) -> Dict[str, any]:
+    def evaluate(self, questions: List[str], answer_path: str, temp_dir: str = None) -> Dict[str, any]:
+        """
+        Evaluate a Python submission.
+
+        Args:
+            questions: List of questions to evaluate against
+            answer_path: Path to the answer file (ZIP or single file)
+            temp_dir: Optional directory for temporary ZIP extraction
+
+        Returns:
+            Dictionary containing score, feedback, issues, and recommendations
+        """
         try:
             if answer_path.endswith(".zip"):
-                answers = PythonAnswerParser.parse_zip_file(answer_path)
+                # Use provided temp_dir or generate a default one
+                temp_dir = temp_dir or f"temp_python_extract_{os.getpid()}"
+                answers = PythonAnswerParser.parse_zip_file(answer_path, temp_dir)
             else:
                 with open(answer_path, "r", encoding="utf-8") as file:
                     content = file.read()
                 answers = PythonAnswerParser.parse_single_file(content)
 
             logger.info(
-                "Processing %d questions and %d answers",
-                len(questions),
-                len(answers),
+                f"Processing {len(questions)} questions and {len(answers)} answers"
             )
             pprint(f"Processing {len(questions)} questions and {len(answers)} answers")
 
@@ -186,9 +208,10 @@ class PythonEvaluator:
 
             return self.model.evaluate(combined_raw_content)
         except Exception as e:
-            logger.error("Failed to process answers from %s: %s", answer_path, str(e))
+            logger.error(f"Failed to process answers from {answer_path}: {str(e)}")
             return {
                 "score": 0,
                 "feedback": f"Error processing answers: {str(e)}",
                 "issues": [str(e)],
-            }
+                "recommendations": []
+            }

{quantumchecker-0.2.8 → quantumchecker-0.2.9}/QuantumCheck/sql_evaluator.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import zipfile
+import shutil
 from pprint import pprint
 from typing import List, Dict
 
@@ -107,50 +108,60 @@ class SQLAnswerParser:
         return answers
 
     @staticmethod
-    def parse_zip_file(zip_path: str) -> List[str]:
+    def parse_zip_file(zip_path: str, temp_dir: str) -> List[str]:
+        """
+        Parse SQL files from a ZIP file, extracting to the specified temp_dir.
+
+        Args:
+            zip_path: Path to the ZIP file
+            temp_dir: Directory to extract ZIP contents
+
+        Returns:
+            List of answer strings extracted from SQL files
+        """
         combined_content = []
 
         try:
+            # Create temporary extraction directory
+            os.makedirs(temp_dir, exist_ok=True)
             with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                temp_dir = "temp_sql_extract"
-                os.makedirs(temp_dir, exist_ok=True)
                 zip_ref.extractall(temp_dir)
 
-                sql_files = sorted(
-                    [f for f in os.listdir(temp_dir) if f.endswith(".sql")]
-                )
+            sql_files = sorted(
+                [f for f in os.listdir(temp_dir) if f.endswith(".sql")]
+            )
 
-                if not sql_files:
-                    logger.warning("No SQL files found in ZIP")
-                    return []
+            if not sql_files:
+                logger.warning(f"No SQL files found in ZIP: {zip_path}")
+                return []
 
-                for sql_file in sql_files:
-                    with open(
+            for sql_file in sql_files:
+                with open(
                         os.path.join(temp_dir, sql_file),
                         "r",
                         encoding="utf-8",
-                    ) as f:
-                        content = f.read().strip()
-                        if content:
-                            combined_content.append(content)
-
-                for f in os.listdir(temp_dir):
-                    os.remove(os.path.join(temp_dir, f))
+                ) as f:
+                    content = f.read().strip()
+                    if content:
+                        combined_content.append(content)
 
-                os.rmdir(temp_dir)
+            if not combined_content:
+                logger.warning(f"No valid content found in SQL files in ZIP: {zip_path}")
+                return []
 
-                if not combined_content:
-                    logger.warning("No valid content found in SQL files")
-                    return []
-
-                combined_text = "\n\n".join(combined_content)
-                return [a.strip() for a in combined_text.split("\n\n") if a.strip()]
+            combined_text = "\n\n".join(combined_content)
+            return [a.strip() for a in combined_text.split("\n\n") if a.strip()]
         except zipfile.BadZipFile:
-            logger.error("Invalid ZIP file: %s", zip_path)
+            logger.error(f"Invalid ZIP file: {zip_path}")
             return []
         except Exception as e:
-            logger.error("Error processing ZIP file %s: %s", zip_path, str(e))
+            logger.error(f"Error processing ZIP file {zip_path}: {str(e)}")
             return []
+        finally:
+            # Clean up temporary directory
+            if os.path.exists(temp_dir):
+                shutil.rmtree(temp_dir, ignore_errors=True)
+                logger.info(f"Cleaned up temporary directory: {temp_dir}")
 
 
 class SQLEvaluator:
@@ -158,19 +169,30 @@ class SQLEvaluator:
         self.api_key = api_key
         self.model = GeminiFlashModel(api_key)
 
-    def evaluate(self, questions: List[str], answer_path: str) -> Dict[str, any]:
+    def evaluate(self, questions: List[str], answer_path: str, temp_dir: str = None) -> Dict[str, any]:
+        """
+        Evaluate an SQL submission.
+
+        Args:
+            questions: List of questions to evaluate against
+            answer_path: Path to the answer file (ZIP or single file)
+            temp_dir: Optional directory for temporary ZIP extraction
+
+        Returns:
+            Dictionary containing score, feedback, issues, and recommendations
+        """
         try:
             if answer_path.endswith(".zip"):
-                answers = SQLAnswerParser.parse_zip_file(answer_path)
+                # Use provided temp_dir or generate a default one
+                temp_dir = temp_dir or f"temp_sql_extract_{os.getpid()}"
+                answers = SQLAnswerParser.parse_zip_file(answer_path, temp_dir)
             else:
                 with open(answer_path, "r", encoding="utf-8") as file:
                     content = file.read()
                 answers = SQLAnswerParser.parse_single_file(content)
 
             logger.info(
-                "Processing %d questions and %d answers",
-                len(questions),
-                len(answers),
+                f"Processing {len(questions)} questions and {len(answers)} answers"
             )
             pprint(f"Processing {len(questions)} questions and {len(answers)} answers")
 
@@ -188,9 +210,10 @@ class SQLEvaluator:
 
             return self.model.evaluate(combined_raw_content)
         except Exception as e:
-            logger.error("Failed to process answers from %s: %s", answer_path, str(e))
+            logger.error(f"Failed to process answers from {answer_path}: {str(e)}")
             return {
                 "score": 0,
                 "feedback": f"Error processing answers: {str(e)}",
                 "issues": [str(e)],
-            }
+                "recommendations": []
+            }