QuantumChecker 0.2.7__tar.gz → 0.2.8__tar.gz

quantumchecker-0.2.8/PKG-INFO

@@ -0,0 +1,138 @@
+Metadata-Version: 2.4
+Name: QuantumChecker
+Version: 0.2.8
+Summary: A package to evaluate homework submissions in Python, SQL, PowerBI, and SSIS.
+Author: Qobiljon
+Author-email: qobiljonkhayrullayev@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.31.0
+Requires-Dist: tenacity>=8.2.3
+Requires-Dist: pdf2image>=1.16.3
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: Pillow>=10.0.0
+Requires-Dist: PyPDF2>=3.0.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# 📘 HomeworkEvaluator
+
+The **HomeworkEvaluator** is a Python-based evaluation engine designed to automatically assess student assignments across different technologies including Python, SQL, Power BI, and SSIS. It uses AI to parse and evaluate student-submitted answers against a set of markdown-formatted questions.
+
+---
+
+## ✨ Features
+
+- Supports multiple file types:
+  - `.py` → Python
+  - `.sql` → SQL
+  - `.zip` → Power BI
+  - `.dtsx` / `.DTSX` → SSIS
+  - `.txt` / `.md` → Plain Text
+- Smart evaluator routing based on file extension.
+- AI-powered feedback generation and scoring.
+- Logging for each evaluation by file type and timestamp.
+- Automatic fallback to backup API keys when quota is exceeded.
+
+---
+
+## 📦 Folder Structure
+
+```
+.
+├── homework_evaluator/
+│   ├── homework_evaluator.py         # Main evaluator class
+│   ├── python_evaluator.py           # Python evaluator logic
+│   ├── sql_evaluator.py              # SQL evaluator logic
+│   ├── powerbi_evaluator.py          # Power BI evaluator logic
+│   ├── ssis_evaluator.py             # SSIS evaluator logic
+│   └── logs/                         # Log files categorized by type and timestamp
+```
+
+---
+
+## 🔧 Installation
+
+Clone this repository and install the necessary dependencies:
+
+```bash
+git clone https://github.com/yourusername/homework-evaluator.git
+cd homework-evaluator
+pip install -r requirements.txt
+```
+
+---
+
+## 🧠 Usage
+
+```python
+from homework_evaluator import HomeworkEvaluator
+
+evaluator = HomeworkEvaluator()
+
+result = evaluator.evaluate_from_content(
+    question_content=markdown_questions,
+    answer_path="/path/to/answer/file.py",
+    api_key="your-main-api-key",
+    backup_api_keys=["backup-key-1", "backup-key-2"]
+)
+
+print(result["mark"])      # e.g., 85
+print(result["feedback"])  # Structured feedback
+```
+
+---
+
+## 🗂️ Question Format
+
+The evaluator expects `question_content` as a Markdown-formatted string where each question is separated by a **double newline** (`\n\n`). Example:
+
+```markdown
+### Q1
+Write a Python function to reverse a string.
+
+### Q2
+Explain the purpose of list comprehensions in Python.
+```
+
+---
+
+## 🛠️ Logging
+
+All evaluations are logged under the `logs/` directory, grouped by file type and timestamp.
+
+```
+logs/
+├── python/
+│   └── evaluation_2025-05-26_14-00-00.log
+├── sql/
+│   └── evaluation_2025-05-26_14-10-12.log
+```
+
+---
+
+## 🧪 Exception Handling
+
+- If a file is not found or questions are malformed, an informative error is raised.
+- If the API quota is exceeded (429 errors or rate limits), it retries using backup keys.
+
+---
+
+## 📄 License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## 🤝 Contributing
+
+Pull requests are welcome. For major changes, please open an issue first to discuss your ideas.

quantumchecker-0.2.8/QuantumCheck/main.py

@@ -0,0 +1,188 @@
+import logging
+import os
+import zipfile
+from datetime import datetime
+from typing import List, Dict, Optional
+from .python_evaluator import PythonEvaluator
+from .sql_evaluator import SQLEvaluator
+from .powerbi_evaluator import PowerBIEvaluator
+from .ssis_evaluator import SSISEvaluator
+from pprint import pprint
+
+logger = logging.getLogger(__name__)
+
+class HomeworkEvaluator:
+    EXTENSION_TO_TYPE = {
+        ".py": "python",
+        ".sql": "sql",
+        ".pbit": "powerbi",
+        ".pdf": "powerbi",
+        ".dtsx": "ssis",
+        ".DTSX": "ssis",
+        ".txt": "text",
+        ".md": "text"
+    }
+
+    def _setup_logger(self, file_type: str) -> logging.Logger:
+        base_log_dir = os.path.join(os.path.dirname(__file__), "logs")
+        type_log_dir = os.path.join(base_log_dir, file_type)
+        os.makedirs(type_log_dir, exist_ok=True)
+
+        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        log_file_path = os.path.join(type_log_dir, f"evaluation_{timestamp}.log")
+
+        logger = logging.getLogger(f"{file_type}_{timestamp}")
+        logger.setLevel(logging.INFO)
+
+        if not logger.handlers:
+            file_handler = logging.FileHandler(log_file_path, encoding="utf-8")
+            formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)
+
+        return logger
+
+    @staticmethod
+    def parse_questions(md_content: str) -> List[str]:
+        questions = [q.strip() for q in md_content.strip().split("\n\n") if q.strip()]
+        if not questions:
+            logger.error("No valid questions found in the question content")
+            raise ValueError("No valid questions found in the question content")
+        logger.info("Parsed %d questions from content", len(questions))
+        return questions
+
+    def _detect_zip_content_type(self, zip_path: str) -> str:
+        """Determine the file type based on ZIP contents."""
+        try:
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                files = [f for f in zip_ref.namelist() if not f.startswith('__MACOSX/')]
+                extensions = {os.path.splitext(f)[1].lower() for f in files if os.path.splitext(f)[1]}
+
+                if not extensions:
+                    logger.warning("No valid files found in ZIP: %s", zip_path)
+                    return "text"
+
+                # Check for specific file types in order of priority: sql, powerbi, ssis, python
+                for ext in [".sql", ".pbit", ".pdf", ".dtsx", ".DTSX", ".py"]:
+                    if ext in extensions and ext in self.EXTENSION_TO_TYPE:
+                        file_type = self.EXTENSION_TO_TYPE[ext]
+                        logger.info("Detected file type: %s from extension: %s in ZIP: %s", file_type, ext, zip_path)
+                        return file_type
+
+                # Fallback to text if only .txt or .md are present
+                if extensions.issubset({".txt", ".md"}):
+                    logger.info("Defaulting to text type for ZIP contents with extensions: %s", extensions)
+                    return "text"
+
+                logger.warning("No recognized specific file types in ZIP: %s, extensions: %s", zip_path, extensions)
+                return "text"
+        except zipfile.BadZipFile:
+            logger.error("Invalid ZIP file: %s", zip_path)
+            return "text"
+        except Exception as e:
+            logger.error("Error inspecting ZIP file %s: %s", zip_path, str(e))
+            return "text"
+
+    def evaluate_from_content(
+        self,
+        question_content: str,
+        answer_path: str,
+        api_key: str,
+        backup_api_keys: Optional[List[str]] = None,
+    ) -> Dict[str, any]:
+        if backup_api_keys is None:
+            backup_api_keys = []
+
+        try:
+            questions = self.parse_questions(question_content)
+        except Exception as e:
+            logger.error("Failed to parse question content: %s", str(e))
+            return {
+                "score": 0,
+                "feedback": f"Error parsing question content: {str(e)}",
+                "issues": [str(e)],
+                "recommendations": []
+            }
+
+        answer_path = answer_path.strip()
+        _, ext = os.path.splitext(answer_path)
+        ext = ext.lower()
+
+        # Determine file type
+        if ext == ".zip":
+            file_type = self._detect_zip_content_type(answer_path)
+        else:
+            file_type = self.EXTENSION_TO_TYPE.get(ext, "text")
+
+        eval_logger = self._setup_logger(file_type)
+        eval_logger.info("Processing answer_path: %s", answer_path)
+        eval_logger.info("Extracted extension: %s", ext)
+        eval_logger.info("Detected file type: %s for file: %s", file_type, answer_path)
+        pprint(f"Processing {len(questions)} questions for file type: {file_type}")
+
+        if not os.path.exists(answer_path):
+            eval_logger.error("Answer file not found: %s", answer_path)
+            return {
+                "score": 0,
+                "feedback": f"Answer file not found: {answer_path}",
+                "issues": [f"Answer file not found: {answer_path}"],
+                "recommendations": []
+            }
+
+        def create_evaluator(ftype, key):
+            if ftype == "python":
+                eval_logger.info("Using PythonEvaluator for file type: %s", ftype)
+                return PythonEvaluator(key)
+            elif ftype == "sql":
+                eval_logger.info("Using SQLEvaluator for file type: %s", ftype)
+                return SQLEvaluator(key)
+            elif ftype == "powerbi":
+                eval_logger.info("Using PowerBIEvaluator for file type: %s", ftype)
+                return PowerBIEvaluator(key)
+            elif ftype == "ssis":
+                eval_logger.info("Using SSISEvaluator for file type: %s", ftype)
+                return SSISEvaluator(key)
+            else:
+                eval_logger.warning("Unknown file type %s, defaulting to PythonEvaluator", ftype)
+                return PythonEvaluator(key)
+
+        keys_to_try = [api_key] + backup_api_keys[:5]
+
+        last_exception = None
+        for i, key in enumerate(keys_to_try):
+            evaluator = create_evaluator(file_type, key)
+            try:
+                evaluation = evaluator.evaluate(questions, answer_path)
+                eval_logger.info(f"Evaluation complete with API key #{i + 1}: Score = {evaluation.get('score')}")
+                return {
+                    "score": evaluation.get("score", 0),
+                    "feedback": evaluation.get("feedback", "No feedback provided"),
+                    "issues": evaluation.get("issues", []),
+                    "recommendations": evaluation.get("recommendations", [])
+                }
+            except Exception as e:
+                error_msg = str(e).lower()
+                if (
+                    "429" in error_msg
+                    or "rate limit" in error_msg
+                    or "quota exceeded" in error_msg
+                    or "daily limit exceeded" in error_msg
+                    or "quota" in error_msg
+                ):
+                    eval_logger.warning(f"API key #{i + 1} limited or quota exceeded. Trying next key if available.")
+                    last_exception = e
+                    continue
+                else:
+                    eval_logger.error(f"Evaluation failed with API key #{i + 1}: %s", str(e))
+                    return {
+                        "score": 0,
+                        "feedback": f"Evaluation failed: {str(e)}",
+                        "issues": [str(e)],
+                        "recommendations": []
+                    }
+        else:
+            eval_logger.error("All API keys exhausted and evaluation failed.")
+            return {
+                "score": 0,
+                "feedback": f"All API keys exhausted: {str(last_exception) if last_exception else 'Unknown error'}",
+            }

{quantumchecker-0.2.7 → quantumchecker-0.2.8}/QuantumCheck/powerbi_evaluator.py

@@ -15,7 +15,6 @@ import io
 import base64
 
 
-# Placeholder for prompts.py content
 def prompt_text_powerbi(combined_content: str) -> str:
     return f"""
     Evaluate the following Power BI DAX question-answer pairs for correctness, clarity, and appropriateness.
@@ -46,19 +45,20 @@ class GeminiFlashModel:
         self.model_name = model_name
         self.endpoint = f"https://generativelanguage.googleapis.com/v1beta/models/{model_name}:generateContent"
 
-    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=4, max=10),
-           retry=retry_if_exception_type((requests.exceptions.RequestException,)))
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(min=4, max=10),
+        retry=retry_if_exception_type((requests.exceptions.RequestException,))
+    )
     def evaluate(self, question_answer_pairs: List[Dict[str, str]]) -> Dict[str, any]:
         logger.info("Starting evaluation of %d Power BI question-answer pairs", len(question_answer_pairs))
         combined_content = "\n\n".join(
             f"Question {i}:\n{qa['question']}\n\nAnswer {i}:\n{qa['answer']}\n"
             for i, qa in enumerate(question_answer_pairs, 1)
         )
-
         headers = {"Content-Type": "application/json"}
         data = {"contents": [{"parts": [{"text": prompt_text_powerbi(combined_content)}]}]}
         response = requests.post(f"{self.endpoint}?key={self.api_key}", headers=headers, json=data)
-
         if response.status_code != 200:
             logger.error("API request failed: Status %d, Response: %s", response.status_code, response.text)
             raise Exception(f"API call failed: {response.status_code} - {response.text}")
@@ -69,8 +69,11 @@ class GeminiFlashModel:
         generated_text = response_data["candidates"][0]["content"]["parts"][0]["text"]
         return self._parse_response(generated_text)
 
-    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=4, max=10),
-           retry=retry_if_exception_type((requests.exceptions.RequestException,)))
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(min=4, max=10),
+        retry=retry_if_exception_type((requests.exceptions.RequestException,))
+    )
     def evaluate_visuals(self, question: str, image_folder: str) -> Dict[str, any]:
         folder_path = Path(image_folder)
         images = list(folder_path.glob("*.png"))[:3]
@@ -80,12 +83,12 @@ class GeminiFlashModel:
             "Evaluate the Power BI report visuals based on the provided task. The visuals are professional dashboards designed for enterprise use.\n\n"
             f"Task: {question}\n\n"
             f"Screenshots: {[str(img.name) for img in images]}\n\n"
-            "Evaluate based on the following criteria, assigning a score out of 100:z\n"
+            "Evaluate based on the following criteria, assigning a score out of 100:\n"
             "- Clarity (30%): Are visuals clear, with readable labels, titles, and legends?\n"
             "- Appropriateness (30%): Are chart types (e.g., bar, line, pie) suitable for the data and task?\n"
             "- Color Usage (20%): Are colors consistent, accessible, and visually appealing? Consider contrast and colorblind accessibility.\n"
             "- Interactivity (20%): Do visible slicers, filters, or tooltips enhance usability and data exploration?\n\n"
-            "Provide a score (0-100) that reflects the overall quality, considering the enterprise context. Avoid overly harsh penalties for minor issues.\n"
+            "Provide a score for overall quality, considering the enterprise context. Avoid overly harsh penalties for minor issues.\n"
             "Provide concise, supportive feedback for beginners, highlighting strengths and areas for improvement.\n\n"
             "Structure the response as:\n"
             "Score: [SCORE]/100\n"
@@ -231,9 +234,7 @@ class PowerBIProcessor:
                     measures.append({
                         "Table": table["name"],
                         "Name": measure["name"],
-                        "Expression": " ".join(measure.get("expression", "")) if isinstance(measure.get("expression"),
-                                                                                            list) else measure.get(
-                            "expression", ""),
+                        "Expression": " ".join(measure.get("expression", "")) if isinstance(measure.get("expression"), list) else measure.get("expression", ""),
                         "FormatString": measure.get("formatString", "")
                     })
         return measures
@@ -242,19 +243,31 @@ class PowerBIProcessor:
     def _get_tables_and_columns(tables: List[Dict]) -> List[Dict]:
         table_info = []
         for table in tables:
-            columns = [{"Column Name": col["name"], "Data Type": col.get("dataType", "Unknown"),
-                        "Source Column": col.get("sourceColumn", "N/A"), "Calculated": col.get("type") == "calculated"}
-                       for col in table.get("columns", [])]
-            expressions = [part["source"]["expression"] for part in table.get("partitions", []) if
-                           part["source"].get("expression")]
+            columns = [
+                {
+                    "Column Name": col["name"],
+                    "Data Type": col.get("dataType", "Unknown"),
+                    "Source Column": col.get("sourceColumn", "N/A"),
+                    "Calculated": col.get("type") == "calculated"
+                }
+                for col in table.get("columns", [])
+            ]
+            expressions = [part["source"]["expression"] for part in table.get("partitions", []) if part["source"].get("expression")]
             table_info.append({"Table Name": table["name"], "Columns": columns, "Expressions": expressions})
         return table_info
 
     @staticmethod
     def _get_relationships(relationships: List[Dict]) -> List[Dict]:
-        return [{"From Table": rel["fromTable"], "From Column": rel["fromColumn"], "To Table": rel["toTable"],
-                 "To Column": rel["toColumn"], "Join Behavior": rel.get("joinOnDateBehavior", "N/A")} for rel in
-                relationships]
+        return [
+            {
+                "From Table": rel["fromTable"],
+                "From Column": rel["fromColumn"],
+                "To Table": rel["toTable"],
+                "To Column": rel["toColumn"],
+                "Join Behavior": rel.get("joinOnDateBehavior", "N/A")
+            }
+            for rel in relationships
+        ]
 
     @staticmethod
     def _cleanup(*paths: str):
@@ -279,8 +292,6 @@ class PowerBIEvaluator:
             extract_path = os.path.join(os.path.dirname(answer_path), "temp_extract")
             pbit_path = None
             pdf_path = None
-
-            # Handle input file type
             if ext == ".zip":
                 pbit_path, pdf_path = self.processor.extract_zip(answer_path, extract_path)
             elif ext == ".pbit":
@@ -296,57 +307,43 @@ class PowerBIEvaluator:
                     "dax_score": 0,
                     "visual_score": 0
                 }
-
             try:
-                # Extract and process the data model from .pbit
                 data_model = self.processor.extract_datamodel(pbit_path)
                 model_data = self.processor.extract_model_data(data_model)
                 answers = [json.dumps(model_data)] * len(questions)
                 dax_result = self.model.evaluate([{"question": q, "answer": a} for q, a in zip(questions, answers)])
-
-                # Initialize result with DAX evaluation
                 result = {
                     "score": 0,
                     "feedback": f"DAX Feedback:\n{dax_result['feedback']}",
                     "issues": dax_result["issues"],
                     "recommendations": dax_result["recommendations"],
-                    "dax_score": dax_result["score"],  # Store DAX score
-                    "visual_score": 0  # Default visual score
+                    "dax_score": dax_result["score"],
+                    "visual_score": 0
                 }
-
-                # Process PDF and evaluate visuals if present
                 if pdf_path:
                     try:
                         self.processor.process_pdf(pdf_path)
                         visual_result = self.model.evaluate_visuals(questions[0], "outputimages")
-                        # Apply 70% DAX, 30% visuals scoring
                         result["score"] = int(0.7 * dax_result["score"] + 0.3 * visual_result["score"])
-                        result["visual_score"] = visual_result["score"]  # Store visual score
+                        result["visual_score"] = visual_result["score"]
                         result["feedback"] += f"\n\nVisual Feedback:\n{visual_result['feedback']}"
                         result["issues"].extend([f"Visual: {i}" for i in visual_result.get("issues", [])])
                         result["recommendations"].extend(visual_result.get("recommendations", []))
                     except ProcessingError as e:
                         logger.warning("Failed to process PDF, proceeding with DAX evaluation only: %s", str(e))
-                        # Use DAX score only, weighted at 100% if no visuals
                         result["score"] = dax_result["score"]
                         result["issues"].append(f"Visual evaluation skipped: {str(e)}")
-                        result["recommendations"].append(
-                            "Ensure a valid PDF is provided for visual evaluation if intended")
+                        result["recommendations"].append("Ensure a valid PDF is provided for visual evaluation if intended")
                 else:
-                    # No PDF provided, use DAX score only
                     result["score"] = dax_result["score"]
                     result["feedback"] += "\n\nVisual Feedback:\nNo visuals provided for evaluation."
                     result["issues"].append("No PDF provided for visual evaluation")
                     result["recommendations"].append("Include a PDF with report visuals for complete evaluation")
-
-                # Print scores with text labels to terminal
                 logger.info("[DAX] Score: %d/100", result["dax_score"])
                 logger.info("[Visual] Score: %d/100", result["visual_score"])
                 logger.info("[Final] Score (70%% DAX, 30%% Visuals): %d/100", result["score"])
-
                 return result
             finally:
-                # Cleanup temporary files and directories
                 self.processor._cleanup(extract_path, "outputimages")
         except Exception as e:
             logger.exception("Failed to evaluate Power BI file %s: %s", answer_path, str(e))