QuantumChecker 0.2.9__tar.gz → 0.3.0__tar.gz

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: QuantumChecker
-Version: 0.2.9
+Version: 0.3.0
 Summary: A package to evaluate homework submissions in Python, SQL, PowerBI, and SSIS.
 Author: Qobiljon
 Author-email: qobiljonkhayrullayev@gmail.com

quantumchecker-0.3.0/QuantumCheck/main.py

@@ -0,0 +1,230 @@
+import logging
+import os
+import zipfile
+from datetime import datetime
+from typing import List, Dict
+from .python_evaluator import PythonEvaluator
+from .sql_evaluator import SQLEvaluator
+from .powerbi_evaluator import PowerBIEvaluator
+from .ssis_evaluator import SSISEvaluator
+import asyncio
+
+_logger_cache = {}
+
+class HomeworkEvaluator:
+    EVALUATOR_REGISTRY = {
+        "python": PythonEvaluator,
+        "sql": SQLEvaluator,
+        "powerbi": PowerBIEvaluator,
+        "ssis": SSISEvaluator
+    }
+
+    EXTENSION_TO_TYPE = {
+        ".py": "python",
+        ".sql": "sql",
+        ".pbit": "powerbi",
+        ".pdf": "powerbi",
+        ".dtsx": "ssis",
+        ".DTSX": "ssis",
+        ".txt": "text",
+        ".md": "text"
+    }
+
+    API_NAME_MAPPING = {
+        "python": "Google Gemini API",
+        "sql": "Google Gemini API",
+        "powerbi": "Google Gemini API",
+        "ssis": "Google Gemini API",
+        "text": "Google Gemini API"
+    }
+
+    def __init__(self, log_level: int = logging.INFO):
+        self.log_level = log_level
+        self._successful_key_cache = {}
+        self._rate_limit_delay = {}
+        self._invalid_key_cache = set()
+        self._lock = asyncio.Lock()
+        self._last_request_time = None
+
+    def _get_logger(self, log_type: str) -> logging.Logger:
+        log_name = f"{log_type}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
+        if log_name not in _logger_cache:
+            logger = logging.getLogger(log_name)
+            logger.setLevel(self.log_level)
+            if not logger.handlers:
+                handler = logging.StreamHandler()
+                handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+                logger.addHandler(handler)
+            _logger_cache[log_name] = logger
+        return _logger_cache[log_name]
+
+    def parse_questions(self, content: str) -> List[str]:
+        logger = self._get_logger("QuantumCheck.main")
+        questions = [q.strip() for q in content.split("\n\n") if q.strip()]
+        if not questions:
+            raise ValueError("No valid questions found in content")
+        return questions
+
+    def _detect_zip_content_type(self, zip_path: str, logger: logging.Logger) -> str:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            extensions = {os.path.splitext(name)[1].lower() for name in zip_ref.namelist()}
+            file_types = [self.EXTENSION_TO_TYPE.get(ext, "text") for ext in extensions if ext]
+            if "python" in file_types:
+                return "python"
+            elif "sql" in file_types:
+                return "sql"
+            elif "powerbi" in file_types:
+                return "powerbi"
+            elif "ssis" in file_types:
+                return "ssis"
+            else:
+                return "text"
+
+    async def evaluate_from_content(
+        self,
+        question_content: str,
+        answer_path: str,
+        api_keys: List[str],
+        question_type: str,
+        retry_count: int = 0
+    ) -> Dict[str, any]:
+        async with self._lock:
+            now = datetime.now()
+            if self._last_request_time:
+                elapsed = (now - self._last_request_time).total_seconds()
+                if elapsed < 30:
+                    await asyncio.sleep(30 - elapsed)
+            self._last_request_time = datetime.now()
+
+            try:
+                questions = self.parse_questions(question_content)
+            except ValueError as e:
+                logger = self._get_logger("QuantumCheck.main")
+                return {
+                    "score": 0,
+                    "feedback": f"Error parsing question content: {str(e)}",
+                    "issues": [str(e)],
+                    "recommendations": [],
+                    "used_api_key_index": None,
+                    "used_api_name": None
+                }
+
+            answer_path = answer_path.strip()
+            _, ext = os.path.splitext(answer_path)
+            ext = ext.lower()
+
+            if ext == ".zip":
+                logger = self._get_logger("zip")
+                file_type = self._detect_zip_content_type(answer_path, logger)
+            else:
+                file_type = self.EXTENSION_TO_TYPE.get(ext, "text")
+                logger = self._get_logger(file_type)
+
+            eval_type = question_type if question_type in self.EVALUATOR_REGISTRY else file_type
+
+            if not os.path.exists(answer_path):
+                return {
+                    "score": 0,
+                    "feedback": f"Answer file not found: {answer_path}",
+                    "issues": [f"Answer file not found: {answer_path}"],
+                    "recommendations": [],
+                    "used_api_key_index": None,
+                    "used_api_name": None
+                }
+
+            evaluator_class = self.EVALUATOR_REGISTRY.get(eval_type, PythonEvaluator)
+            last_error_messages = []
+
+            available_keys = [(i + 1, key) for i, key in enumerate(api_keys) if key not in self._invalid_key_cache]
+
+            cached_key_idx = self._successful_key_cache.get(eval_type)
+            if cached_key_idx is not None and cached_key_idx < len(api_keys):
+                cached_key = api_keys[cached_key_idx]
+                if cached_key not in self._invalid_key_cache:
+                    available_keys.insert(0, (cached_key_idx + 1, cached_key))
+
+            if not available_keys:
+                return {
+                    "score": 0,
+                    "feedback": "No valid API keys available.",
+                    "issues": ["All API keys are invalid or rate-limited."],
+                    "recommendations": [],
+                    "used_api_key_index": None,
+                    "used_api_name": None
+                }
+
+            for idx, key in available_keys:
+                if key in self._rate_limit_delay:
+                    delay_until = self._rate_limit_delay[key]
+                    current_time = datetime.now()
+                    delay_until_time = datetime.fromtimestamp(delay_until)
+                    if current_time < delay_until_time:
+                        continue
+                    else:
+                        del self._rate_limit_delay[key]
+
+                evaluator = evaluator_class(key)
+                api_name = getattr(evaluator, 'get_api_name', lambda: self.API_NAME_MAPPING.get(eval_type, "Unknown API"))()
+
+                try:
+                    evaluation = evaluator.evaluate(questions, answer_path, temp_dir=f"temp_extract_{os.getpid()}_{idx}")
+                    feedback = evaluation.get("feedback", "").lower()
+                    issues = " ".join(evaluation.get("issues", [])).lower()
+
+                    if any(phrase in feedback or phrase in issues for phrase in ["api key not valid", "api_key_invalid"]):
+                        last_error_messages.append(f"API key #{idx} invalid.")
+                        self._invalid_key_cache.add(key)
+                        continue
+
+                    if any(phrase in feedback or phrase in issues for phrase in ["429", "too many requests", "rate limit"]):
+                        last_error_messages.append(f"API key #{idx} rate limited.")
+                        self._rate_limit_delay[key] = datetime.now().timestamp() + 300
+                        continue
+
+                    if any(phrase in feedback or phrase in issues for phrase in ["503", "service unavailable"]):
+                        last_error_messages.append(f"API key #{idx} service unavailable.")
+                        self._rate_limit_delay[key] = datetime.now().timestamp() + 7200
+                        continue
+
+                    if evaluation.get("score", 0) == 0 and "evaluation not returned" in feedback:
+                        last_error_messages.append(f"API key #{idx} returned invalid evaluation.")
+                        continue
+
+                    self._successful_key_cache[eval_type] = idx - 1
+                    return {
+                        "score": evaluation.get("score", 0),
+                        "feedback": evaluation.get("feedback", "No feedback provided"),
+                        "issues": evaluation.get("issues", []),
+                        "recommendations": evaluation.get("recommendations", []),
+                        "used_api_key_index": idx,
+                        "used_api_name": api_name
+                    }
+
+                except Exception as e:
+                    last_error_messages.append(f"Exception with key #{idx}: {str(e)}")
+                    if "429" in str(e) or "rate limit" in str(e).lower():
+                        self._rate_limit_delay[key] = datetime.now().timestamp() + 300
+                    elif "503" in str(e) or "service unavailable" in str(e).lower():
+                        self._rate_limit_delay[key] = datetime.now().timestamp() + 7200
+                    continue
+
+            if retry_count < 3 and self._rate_limit_delay:
+                next_available_ts = min(self._rate_limit_delay.values())
+                wait_time = max(0, next_available_ts - datetime.now().timestamp())
+                await asyncio.sleep(wait_time + 1)
+                return await self.evaluate_from_content(
+                    question_content=question_content,
+                    answer_path=answer_path,
+                    api_keys=api_keys,
+                    question_type=question_type,
+                    retry_count=retry_count + 1
+                )
+
+            return {
+                "score": 0,
+                "feedback": "Evaluation failed with all API keys." if retry_count >= 3 else "All API keys are temporarily unavailable.",
+                "issues": last_error_messages if last_error_messages else ["All API keys failed to evaluate the submission."],
+                "recommendations": [],
+                "used_api_key_index": None,
+                "used_api_name": None
+            }

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/QuantumCheck/powerbi_evaluator.py

@@ -37,7 +37,7 @@ logging.basicConfig(
 
 
 class GeminiFlashModel:
-    def __init__(self, api_key: str, model_name: str = "gemini-1.5-flash"):
+    def __init__(self, api_key: str, model_name: str = "gemini-2.0-flash"):
         api_key = os.getenv("GEMINI_API_KEY") or api_key
         if not api_key:
             raise ValueError("API key not found in .env file or environment variables.")

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/QuantumCheck/python_evaluator.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 
 class GeminiFlashModel:
-    def __init__(self, api_key: str, model_name: str = "gemini-1.5-flash"):
+    def __init__(self, api_key: str, model_name: str = "gemini-2.0-flash"):
         if not api_key:
             raise ValueError("API key is required.")
         self.api_key = api_key
@@ -202,10 +202,6 @@ class PythonEvaluator:
                 f"Questions:\n{combined_questions}\n\nAnswers:\n{combined_answers}"
             )
 
-            final_prompt = prompt_text_python(combined_raw_content)
-            with open("combined_python_prompt.txt", "w", encoding="utf-8") as f:
-                f.write(final_prompt)
-
             return self.model.evaluate(combined_raw_content)
         except Exception as e:
             logger.error(f"Failed to process answers from {answer_path}: {str(e)}")

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/QuantumCheck/sql_evaluator.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 
 class GeminiFlashModel:
-    def __init__(self, api_key: str, model_name: str = "gemini-1.5-flash"):
+    def __init__(self, api_key: str, model_name: str = "gemini-2.0-flash"):
         if not api_key:
             raise ValueError("API key is required.")
         self.api_key = api_key
@@ -202,12 +202,6 @@ class SQLEvaluator:
                 f"Questions:\n{combined_questions}\n\nAnswers:\n{combined_answers}"
             )
 
-            final_prompt = prompt_text_sql(combined_raw_content)
-
-            logger.info(
-                "Saved full combined content and prompt to 'combined_sql_full.txt'"
-            )
-
             return self.model.evaluate(combined_raw_content)
         except Exception as e:
             logger.error(f"Failed to process answers from {answer_path}: {str(e)}")

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/QuantumCheck/ssis_evaluator.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 
 
 class GeminiFlashModel:
-    def __init__(self, api_key: str, model_name: str = "gemini-1.5-flash"):
+    def __init__(self, api_key: str, model_name: str = "gemini-2.0-flash"):
         if not api_key:
             raise ValueError("API key is required.")
         self.api_key = api_key
@@ -385,16 +385,8 @@ class SSISEvaluator:
                 f"Issues:\n{', '.join(issues) if issues else 'None'}"
             )
 
-            final_prompt = prompt_text_ssis(combined_raw_content)
 
-            # Save final prompt to txt file for debugging
-            with open("last_ssis_prompt.txt", "w", encoding="utf-8") as f:
-                f.write(final_prompt)
-            logger.debug("Generated prompt: %s", final_prompt[:500])
-
-            # Evaluate using Gemini model
             result = self.model.evaluate(combined_raw_content)
-            # Append parsing issues to result
             result["issues"] = result.get("issues", []) + issues
             return result
 

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/QuantumChecker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: QuantumChecker
-Version: 0.2.9
+Version: 0.3.0
 Summary: A package to evaluate homework submissions in Python, SQL, PowerBI, and SSIS.
 Author: Qobiljon
 Author-email: qobiljonkhayrullayev@gmail.com

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="QuantumChecker",
-    version="0.2.9",
+    version="0.3.0",
     author="Qobiljon",
     author_email="qobiljonkhayrullayev@gmail.com",
     description="A package to evaluate homework submissions in Python, SQL, PowerBI, and SSIS.",

{quantumchecker-0.2.9 → quantumchecker-0.3.0}/tests/test2.py

@@ -10,9 +10,9 @@ async def main():
     Q2: What is the difference between a list and a tuple in Python?
     """
 
-
     answer_path = "answer/python1.zip"
     api_keys = []
+
     question_type = "python"
 
     result = await evaluator.evaluate_from_content(
@@ -23,7 +23,8 @@ async def main():
     )
 
     print("Evaluation Result:")
-    print(result)
+    print(result["score"])
+    print(result["feedback"])
 
 
 if __name__ == "__main__":

quantumchecker-0.2.9/QuantumCheck/main.py

@@ -1,222 +0,0 @@
-import logging
-import os
-import zipfile
-import random
-from datetime import datetime
-from typing import List, Dict, Optional
-from .python_evaluator import PythonEvaluator
-from .sql_evaluator import SQLEvaluator
-from .powerbi_evaluator import PowerBIEvaluator
-from .ssis_evaluator import SSISEvaluator
-import asyncio
-
-_logger_cache = {}
-
-class HomeworkEvaluator:
-    EVALUATOR_REGISTRY = {
-        "python": PythonEvaluator,
-        "sql": SQLEvaluator,
-        "powerbi": PowerBIEvaluator,
-        "ssis": SSISEvaluator
-    }
-
-    EXTENSION_TO_TYPE = {
-        ".py": "python",
-        ".sql": "sql",
-        ".pbit": "powerbi",
-        ".pdf": "powerbi",
-        ".dtsx": "ssis",
-        ".DTSX": "ssis",
-        ".txt": "text",
-        ".md": "text"
-    }
-
-    API_NAME_MAPPING = {
-        "python": "Google Gemini API",
-        "sql": "Google Gemini API",
-        "powerbi": "Google Gemini API",
-        "ssis": "Google Gemini API",
-        "text": "Google Gemini API"
-    }
-
-    def __init__(self, log_level: int = logging.INFO):
-        self.log_level = log_level
-        self._successful_key_cache = {}
-        self._rate_limit_delay = {}  # Track delay per key
-
-    def _get_logger(self, log_type: str) -> logging.Logger:
-        log_name = f"{log_type}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
-        if log_name not in _logger_cache:
-            logger = logging.getLogger(log_name)
-            logger.setLevel(self.log_level)
-            if not logger.handlers:
-                handler = logging.StreamHandler()
-                handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
-                logger.addHandler(handler)
-            _logger_cache[log_name] = logger
-        return _logger_cache[log_name]
-
-    def parse_questions(self, content: str) -> List[str]:
-        logger = self._get_logger("QuantumCheck.main")
-        questions = [q.strip() for q in content.split("\n\n") if q.strip()]
-        logger.info(f"Parsed {len(questions)} questions from content")
-        if not questions:
-            raise ValueError("No valid questions found in content")
-        return questions
-
-    def _detect_zip_content_type(self, zip_path: str, logger: logging.Logger) -> str:
-        try:
-            with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                extensions = {os.path.splitext(name)[1].lower() for name in zip_ref.namelist()}
-                file_types = [self.EXTENSION_TO_TYPE.get(ext, "text") for ext in extensions if ext]
-                logger.info(f"Detected extensions in ZIP {zip_path}: {extensions}, types: {file_types}")
-                if "python" in file_types:
-                    logger.info(f"Selected file type: python from extension: .py in ZIP: {zip_path}")
-                    return "python"
-                elif "sql" in file_types:
-                    logger.info(f"Selected file type: sql from extension: .sql in ZIP: {zip_path}")
-                    return "sql"
-                elif "powerbi" in file_types:
-                    logger.info(f"Selected file type: powerbi from extension: .pbit or .pdf in ZIP: {zip_path}")
-                    return "powerbi"
-                elif "ssis" in file_types:
-                    logger.info(f"Selected file type: ssis from extension: .dtsx in ZIP: {zip_path}")
-                    return "ssis"
-                else:
-                    logger.info(f"Selected file type: text (default) in ZIP: {zip_path}")
-                    return "text"
-        except zipfile.BadZipFile:
-            logger.error(f"Invalid ZIP file: {zip_path}")
-            raise ValueError(f"Invalid ZIP file: {zip_path}")
-
-    async def evaluate_from_content(
-            self,
-            question_content: str,
-            answer_path: str,
-            api_keys: List[str],
-            question_type: str
-    ) -> Dict[str, any]:
-        try:
-            questions = self.parse_questions(question_content)
-        except ValueError as e:
-            logger = self._get_logger("QuantumCheck.main")
-            logger.error("Failed to parse question content: %s", str(e))
-            return {
-                "score": 0,
-                "feedback": f"Error parsing question content: {str(e)}",
-                "issues": [str(e)],
-                "recommendations": [],
-                "used_api_key_index": None,
-                "used_api_name": None
-            }
-
-        answer_path = answer_path.strip()
-        _, ext = os.path.splitext(answer_path)
-        ext = ext.lower()
-
-        # Determine file type, prioritizing question_type for evaluator selection
-        if ext == ".zip":
-            logger = self._get_logger("zip")
-            file_type = self._detect_zip_content_type(answer_path, logger)
-        else:
-            file_type = self.EXTENSION_TO_TYPE.get(ext, "text")
-            logger = self._get_logger(file_type)
-
-        # Use question_type if provided, else fallback to file_type
-        eval_type = question_type if question_type in self.EVALUATOR_REGISTRY else file_type
-        logger.info(f"Processing answer_path: {answer_path} with detected file type: {file_type}, evaluation type: {eval_type}")
-
-        if not os.path.exists(answer_path):
-            logger.error(f"Answer file not found: {answer_path}")
-            return {
-                "score": 0,
-                "feedback": f"Answer file not found: {answer_path}",
-                "issues": [f"Answer file not found: {answer_path}"],
-                "recommendations": [],
-                "used_api_key_index": None,
-                "used_api_name": None
-            }
-
-        evaluator_class = self.EVALUATOR_REGISTRY.get(eval_type, PythonEvaluator)
-        last_error_messages = []
-
-        # Shuffle keys for load balancing
-        key_order = [(i + 1, key) for i, key in enumerate(api_keys)]
-        random.shuffle(key_order)
-
-        # Try cached key with 30% probability to encourage rotation
-        cached_key_idx = self._successful_key_cache.get(eval_type)
-        if cached_key_idx is not None and cached_key_idx < len(api_keys) and random.random() < 0.3:
-            key_order.insert(0, (cached_key_idx + 1, api_keys[cached_key_idx]))
-
-        for idx, key in key_order:
-            # Check rate limit delay
-            if key in self._rate_limit_delay:
-                delay_until = self._rate_limit_delay[key]
-                current_time = datetime.now()
-                delay_until_time = datetime.fromtimestamp(delay_until)
-                if current_time < delay_until_time:
-                    logger.info(f"API key #{idx} is rate-limited until {delay_until_time}, skipping.")
-                    continue
-                else:
-                    del self._rate_limit_delay[key]
-
-            logger.info(f"Trying API key #{idx}")
-            evaluator = evaluator_class(key)
-            api_name = getattr(evaluator, 'get_api_name', lambda: self.API_NAME_MAPPING.get(eval_type, "Unknown API"))()
-            logger.info(f"Using API: {api_name} for evaluation type: {eval_type}")
-
-            try:
-                evaluation = evaluator.evaluate(questions, answer_path, temp_dir=f"temp_extract_{os.getpid()}_{idx}")
-
-                feedback = evaluation.get("feedback", "").lower()
-                issues = " ".join(evaluation.get("issues", [])).lower()
-
-                # Check for invalid API key
-                if any(phrase in feedback or phrase in issues for phrase in ["api key not valid", "api_key_invalid"]):
-                    logger.warning(f"API key #{idx} invalid, trying next key.")
-                    last_error_messages.append(f"API key #{idx} invalid.")
-                    continue
-
-                # Check for rate limit errors
-                if any(phrase in feedback or phrase in issues for phrase in ["429", "too many requests", "rate limit"]):
-                    logger.warning(f"API key #{idx} hit rate limit, applying delay.")
-                    last_error_messages.append(f"API key #{idx} rate limited.")
-                    self._rate_limit_delay[key] = datetime.now().timestamp() + 45  # 45s delay
-                    continue
-
-                # Check for invalid evaluation
-                if evaluation.get("score", 0) == 0 and "evaluation not returned" in feedback:
-                    logger.warning(f"API key #{idx} returned invalid evaluation, trying next key.")
-                    last_error_messages.append(f"API key #{idx} returned invalid evaluation.")
-                    continue
-
-                # Cache successful key
-                self._successful_key_cache[eval_type] = idx - 1
-                logger.info(f"Evaluation succeeded with API key #{idx}: Score = {evaluation.get('score')}")
-
-                return {
-                    "score": evaluation.get("score", 0),
-                    "feedback": evaluation.get("feedback", "No feedback provided"),
-                    "issues": evaluation.get("issues", []),
-                    "recommendations": evaluation.get("recommendations", []),
-                    "used_api_key_index": idx,
-                    "used_api_name": api_name
-                }
-
-            except Exception as e:
-                logger.error(f"Exception using API key #{idx}: {str(e)}")
-                last_error_messages.append(f"Exception with key #{idx}: {str(e)}")
-                if "429" in str(e) or "rate limit" in str(e).lower():
-                    self._rate_limit_delay[key] = datetime.now().timestamp() + 45
-                continue
-
-        logger.error("Evaluation failed with all API keys.")
-        return {
-            "score": 0,
-            "feedback": "Evaluation failed with all API keys.",
-            "issues": last_error_messages if last_error_messages else ["All API keys failed to evaluate the submission."],
-            "recommendations": [],
-            "used_api_key_index": None,
-            "used_api_name": None
-        }