QuizGenerator 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

QuizGenerator/generate.py

@@ -1,16 +1,19 @@
 #!env python
 import argparse
+from datetime import datetime
 import os
 import random
 import shutil
 import subprocess
 import tempfile
+import traceback
 import re
 from pathlib import Path
 from dotenv import load_dotenv
 from QuizGenerator.canvas.canvas_interface import CanvasInterface
 
 from QuizGenerator.quiz import Quiz
+from QuizGenerator.question import QuestionRegistry
 
 import logging
 log = logging.getLogger(__name__)
@@ -44,6 +47,12 @@ def parse_args():
   parser.add_argument("--num_pdfs", default=0, type=int, help="How many PDF quizzes to create")
   parser.add_argument("--latex", action="store_false", dest="typst", help="Use Typst instead of LaTeX for PDF generation")
 
+  # Testing flags
+  parser.add_argument("--test_all", type=int, default=0, metavar="N",
+                     help="Generate N variations of ALL registered questions to test they work correctly")
+  parser.add_argument("--strict", action="store_true",
+                     help="With --test_all, skip PDF/Canvas generation if any questions fail")
+
   subparsers = parser.add_subparsers(dest='command')
   test_parser = subparsers.add_parser("TEST")
 
@@ -63,8 +72,155 @@ def test():
   print("\n" + "="*60)
   print("TEST COMPLETE")
   print("="*60)
-  
-  
+
+
+def test_all_questions(
+    num_variations: int,
+    generate_pdf: bool = False,
+    use_typst: bool = True,
+    canvas_course=None,
+    strict: bool = False
+):
+  """
+  Test all registered questions by generating N variations of each.
+
+  This helps verify that all question types work correctly and can generate
+  valid output without errors.
+
+  Args:
+    num_variations: Number of variations to generate for each question type
+    generate_pdf: If True, generate a PDF with all successful questions
+    use_typst: If True, use Typst for PDF generation; otherwise use LaTeX
+    canvas_course: If provided, push a test quiz to this Canvas course
+    strict: If True, skip PDF/Canvas generation if any questions fail
+  """
+  # Ensure all premade questions are loaded
+  QuestionRegistry.load_premade_questions()
+
+  registered_questions = QuestionRegistry._registry
+  total_questions = len(registered_questions)
+
+  # Test defaults for questions that require external input
+  # These are "template" questions that can't work without content
+  TEST_DEFAULTS = {
+    'fromtext': {'text': 'Test question placeholder text.'},
+    'fromgenerator': {'generator': 'return "Generated test content"'},
+  }
+
+  print(f"\nTesting {total_questions} registered question types with {num_variations} variations each...")
+  print("=" * 70)
+
+  failed_questions = []
+  successful_questions = []
+  # Collect question instances for PDF/Canvas generation
+  test_question_instances = []
+
+  for i, (question_name, question_class) in enumerate(sorted(registered_questions.items()), 1):
+    print(f"\n[{i}/{total_questions}] Testing: {question_name}")
+    print(f"  Class: {question_class.__name__}")
+
+    question_failures = []
+
+    for variation in range(num_variations):
+      seed = variation * 1000  # Use different seeds for each variation
+      try:
+        # Get any test defaults for this question type
+        extra_kwargs = TEST_DEFAULTS.get(question_name, {})
+
+        # Create question instance with minimal required params
+        question = question_class(
+          name=f"{question_name} (v{variation+1})",
+          points_value=1.0,
+          **extra_kwargs
+        )
+
+        # Generate the question (this calls refresh and builds the AST)
+        question_ast = question.get_question(rng_seed=seed)
+
+        # Try rendering to both formats to catch format-specific issues
+        try:
+          question_ast.render("html")
+        except Exception as e:
+          tb = traceback.format_exc()
+          question_failures.append(f"  Variation {variation+1}: HTML render failed - {e}\n{tb}")
+          continue
+
+        try:
+          question_ast.render("typst")
+        except Exception as e:
+          tb = traceback.format_exc()
+          question_failures.append(f"  Variation {variation+1}: Typst render failed - {e}\n{tb}")
+          continue
+
+        # If we got here, the question works - save the instance
+        test_question_instances.append(question)
+
+      except Exception as e:
+        tb = traceback.format_exc()
+        question_failures.append(f"  Variation {variation+1}: Generation failed - {e}\n{tb}")
+
+    if question_failures:
+      print(f"  FAILED ({len(question_failures)}/{num_variations} variations)")
+      for failure in question_failures:
+        print(failure)
+      failed_questions.append((question_name, question_failures))
+    else:
+      print(f"  OK ({num_variations}/{num_variations} variations)")
+      successful_questions.append(question_name)
+
+  # Summary
+  print("\n" + "=" * 70)
+  print("TEST SUMMARY")
+  print("=" * 70)
+  print(f"Total question types: {total_questions}")
+  print(f"Successful: {len(successful_questions)}")
+  print(f"Failed: {len(failed_questions)}")
+
+  if failed_questions:
+    print("\nFailed questions:")
+    for name, failures in failed_questions:
+      print(f"  - {name}: {len(failures)} failures")
+
+  print("=" * 70)
+
+  # Generate PDF and/or push to Canvas if requested
+  if strict and failed_questions:
+    print("\n[STRICT MODE] Skipping PDF/Canvas generation due to failures")
+  elif (generate_pdf or canvas_course) and test_question_instances:
+    print(f"\nCreating test quiz with {len(test_question_instances)} questions...")
+
+    # Create a Quiz object with all successful questions
+    test_quiz = Quiz(
+      name="Test All Questions",
+      questions=test_question_instances,
+      practice=True
+    )
+
+    if generate_pdf:
+      print("Generating PDF...")
+      pdf_seed = 12345  # Fixed seed for reproducibility
+      if use_typst:
+        typst_text = test_quiz.get_quiz(rng_seed=pdf_seed).render("typst")
+        generate_typst(typst_text, remove_previous=True, name_prefix="test_all_questions")
+      else:
+        latex_text = test_quiz.get_quiz(rng_seed=pdf_seed).render_latex()
+        generate_latex(latex_text, remove_previous=True, name_prefix="test_all_questions")
+      print("PDF generated in out/ directory")
+
+    if canvas_course:
+      print("Pushing to Canvas...")
+      quiz_title = f"Test All Questions ({int(datetime.now().timestamp())} : {datetime.now().strftime('%b %d %I:%M%p')})"
+      canvas_course.push_quiz_to_canvas(
+        test_quiz,
+        num_variations=1,
+        title=quiz_title,
+        is_practice=True
+      )
+      print(f"Quiz '{quiz_title}' pushed to Canvas")
+
+  return len(failed_questions) == 0
+
+
 def generate_latex(latex_text, remove_previous=False, name_prefix=None):
   """
   Generate PDF from LaTeX source code.
@@ -269,6 +425,22 @@ def main():
     test()
     return
 
+  if args.test_all > 0:
+    # Set up Canvas course if course_id provided
+    canvas_course = None
+    if args.course_id:
+      canvas_interface = CanvasInterface(prod=args.prod)
+      canvas_course = canvas_interface.get_course(course_id=args.course_id)
+
+    success = test_all_questions(
+      args.test_all,
+      generate_pdf=True,
+      use_typst=getattr(args, 'typst', True),
+      canvas_course=canvas_course,
+      strict=args.strict
+    )
+    exit(0 if success else 1)
+
   # Clear any previous metrics
   PerformanceTracker.clear_metrics()
 

QuizGenerator/misc.py

@@ -21,9 +21,3 @@ def fix_negative_zero(value):
   if isinstance(value, (int, float)):
     return 0.0 if value == 0 else value
   return value
-
-
-# Backward compatibility: Answer and MatrixAnswer have moved to ContentAST
-# Re-export them here so existing imports continue to work
-Answer = ContentAST.Answer
-MatrixAnswer = ContentAST.MatrixAnswer

QuizGenerator/mixins.py

@@ -6,7 +6,6 @@ These mixins provide reusable patterns for common question structures.
 
 import abc
 from typing import Dict, List, Any, Union
-from QuizGenerator.misc import Answer
 from QuizGenerator.contentast import ContentAST
 
 
@@ -68,13 +67,13 @@ class TableQuestionMixin:
     """
     answer_columns = answer_columns or []
     
-    def format_cell(row_data: Dict, column: str) -> Union[str, Answer]:
+    def format_cell(row_data: Dict, column: str) -> Union[str, ContentAST.Answer]:
       """Format a cell based on whether it should be an answer or plain data"""
       value = row_data.get(column, "")
 
       # If this column should contain answers and the value is an Answer object
       # Answer extends ContentAST.Leaf, so it can be used directly
-      if column in answer_columns and isinstance(value, Answer):
+      if column in answer_columns and isinstance(value, ContentAST.Answer):
         return value
       # If this column should contain answers but we have the answer key
       elif column in answer_columns and isinstance(value, str) and hasattr(self, 'answers'):
@@ -150,11 +149,11 @@ class TableQuestionMixin:
         ContentAST.Table with multiple answer blanks
     """
     
-    def process_cell_value(value: Any) -> Union[str, Answer]:
+    def process_cell_value(value: Any) -> Union[str, ContentAST.Answer]:
       """Convert cell values to appropriate display format"""
       # If it's already an Answer object, use it directly
       # Answer extends ContentAST.Leaf so it can be used in the AST
-      if isinstance(value, Answer):
+      if isinstance(value, ContentAST.Answer):
         return value
       # If it's a string that looks like an answer key, try to resolve it
       elif isinstance(value, str) and value.startswith("answer__") and hasattr(self, 'answers'):
@@ -369,9 +368,9 @@ class MultiPartQuestionMixin:
     Example:
         # For a 3-part question
         {
-            'a': Answer.integer('a', 5),
-            'b': Answer.integer('b', 12),
-            'c': Answer.integer('c', -3)
+            'a': ContentAST.Answer.integer('a', 5),
+            'b': ContentAST.Answer.integer('b', 12),
+            'c': ContentAST.Answer.integer('c', -3)
         }
     """
     if not self.is_multipart():

QuizGenerator/premade_questions/basic.py

@@ -6,7 +6,7 @@ from typing import List, Dict, Any, Tuple
 import logging
 
 from QuizGenerator.contentast import *
-from QuizGenerator.question import Question, QuestionRegistry, Answer
+from QuizGenerator.question import Question, QuestionRegistry
 from QuizGenerator.mixins import TableQuestionMixin
 
 log = logging.getLogger(__name__)
@@ -25,8 +25,8 @@ class FromText(Question):
     
     return ContentAST.Section([ContentAST.Text(self.text)])
   
-  def get_answers(self, *args, **kwargs) -> Tuple[Answer.AnswerKind, List[Dict[str,Any]]]:
-    return Answer.AnswerKind.ESSAY, []
+  def get_answers(self, *args, **kwargs) -> Tuple[ContentAST.Answer.CanvasAnswerKind, List[Dict[str,Any]]]:
+    return ContentAST.Answer.CanvasAnswerKind.ESSAY, []
 
 
 @QuestionRegistry.register()

QuizGenerator/premade_questions/cst334/languages.py

@@ -6,9 +6,9 @@ import enum
 import itertools
 from typing import List, Dict, Optional, Tuple, Any
 
-from QuizGenerator.question import QuestionRegistry, Question, Answer
+from QuizGenerator.question import QuestionRegistry, Question
 
-from QuizGenerator.contentast import ContentAST
+from QuizGenerator.contentast import ContentAST, AnswerTypes
 
 import logging
 log = logging.getLogger(__name__)
@@ -265,60 +265,55 @@ class ValidStringsInLanguageQuestion(LanguageQuestion):
       self._select_random_grammar()
 
     self.answers = {}
-    
-    self.answers.update(
-      {
-        "answer_good" : Answer(
-          f"answer_good",
-          self.grammar_good.generate(self.include_spaces),
-          Answer.AnswerKind.MULTIPLE_ANSWER,
-          correct=True
-        )
-      }
+
+    # Create answers with proper ContentAST.Answer signature
+    # value is the generated string, correct indicates if it's a valid answer
+    good_string = self.grammar_good.generate(self.include_spaces)
+    self.answers["answer_good"] = ContentAST.Answer(
+      value=good_string,
+      kind=ContentAST.Answer.CanvasAnswerKind.MULTIPLE_ANSWER,
+      correct=True
     )
-    
-    self.answers.update(
-      {
-        "answer_bad":
-          Answer(
-            f"answer_bad",
-            self.grammar_bad.generate(self.include_spaces),
-            Answer.AnswerKind.MULTIPLE_ANSWER,
-            correct=False
-          )
-      })
-    self.answers.update({
-      "answer_bad_early":
-        Answer(
-          f"answer_bad_early",
-          self.grammar_bad.generate(self.include_spaces, early_exit=True),
-          Answer.AnswerKind.MULTIPLE_ANSWER,
-          correct=False
-        )
-    })
-    
+
+    bad_string = self.grammar_bad.generate(self.include_spaces)
+    self.answers["answer_bad"] = ContentAST.Answer(
+      value=bad_string,
+      kind=ContentAST.Answer.CanvasAnswerKind.MULTIPLE_ANSWER,
+      correct=False
+    )
+
+    bad_early_string = self.grammar_bad.generate(self.include_spaces, early_exit=True)
+    self.answers["answer_bad_early"] = ContentAST.Answer(
+      value=bad_early_string,
+      kind=ContentAST.Answer.CanvasAnswerKind.MULTIPLE_ANSWER,
+      correct=False
+    )
+
     answer_text_set = {a.value for a in self.answers.values()}
     num_tries = 0
     while len(self.answers) < 10 and num_tries < self.MAX_TRIES:
-      
+
       correct = self.rng.choice([True, False])
       if not correct:
         early_exit = self.rng.choice([True, False])
       else:
         early_exit = False
-      new_answer = Answer(
-        f"answer_{num_tries}",
-        (
-          self.grammar_good
-          if correct or early_exit
-          else self.grammar_bad
-        ).generate(self.include_spaces, early_exit=early_exit),
-        Answer.AnswerKind.MULTIPLE_ANSWER,
-        correct= correct and not early_exit
-      )
-      if len(new_answer.value) < self.MAX_LENGTH and new_answer.value not in answer_text_set:
-        self.answers.update({new_answer.key : new_answer})
-        answer_text_set.add(new_answer.value)
+
+      generated_string = (
+        self.grammar_good
+        if correct or early_exit
+        else self.grammar_bad
+      ).generate(self.include_spaces, early_exit=early_exit)
+
+      is_correct = correct and not early_exit
+
+      if len(generated_string) < self.MAX_LENGTH and generated_string not in answer_text_set:
+        self.answers[f"answer_{num_tries}"] = ContentAST.Answer(
+          value=generated_string,
+          kind=ContentAST.Answer.CanvasAnswerKind.MULTIPLE_ANSWER,
+          correct=is_correct
+        )
+        answer_text_set.add(generated_string)
       num_tries += 1
     
     # Generate answers that will be used only for the latex version.
@@ -336,7 +331,6 @@ class ValidStringsInLanguageQuestion(LanguageQuestion):
         ])()
       )
   
-  
   def _get_body(self, *args, **kwargs):
     """Build question body and collect answers."""
     answers = list(self.answers.values())
@@ -373,7 +367,7 @@ class ValidStringsInLanguageQuestion(LanguageQuestion):
     # For Latex-only, ask students to generate some more.
     body.add_element(
       ContentAST.OnlyLatex([
-        ContentAST.AnswerBlock([Answer.string(f"blank_line_{i}", "", label="") for i in range(self.num_answer_blanks)])
+        ContentAST.AnswerBlock([AnswerTypes.String("", label="") for i in range(self.num_answer_blanks)])
       ])
     )
 
@@ -400,6 +394,6 @@ class ValidStringsInLanguageQuestion(LanguageQuestion):
     explanation, _ = self._get_explanation(*args, **kwargs)
     return explanation
 
-  def get_answers(self, *args, **kwargs) -> Tuple[Answer.AnswerKind, List[Dict[str,Any]]]:
+  def get_answers(self, *args, **kwargs) -> Tuple[ContentAST.Answer.CanvasAnswerKind, List[Dict[str,Any]]]:
     
-    return Answer.AnswerKind.MULTIPLE_ANSWER, list(itertools.chain(*[a.get_for_canvas() for a in self.answers.values()]))
+    return ContentAST.Answer.CanvasAnswerKind.MULTIPLE_ANSWER, list(itertools.chain(*[a.get_for_canvas() for a in self.answers.values()]))

QuizGenerator/premade_questions/cst334/math_questions.py

@@ -3,8 +3,8 @@ import abc
 import logging
 import math
 
-from QuizGenerator.question import Question, QuestionRegistry, Answer
-from QuizGenerator.contentast import ContentAST
+from QuizGenerator.question import Question, QuestionRegistry
+from QuizGenerator.contentast import ContentAST, AnswerTypes
 from QuizGenerator.constants import MathRanges
 
 log = logging.getLogger(__name__)
@@ -31,10 +31,10 @@ class BitsAndBytes(MathQuestion):
     self.num_bytes = int(math.pow(2, self.num_bits))
     
     if self.from_binary:
-      self.answers = {"answer": Answer.integer("num_bytes", self.num_bytes,
+      self.answers = {"answer": AnswerTypes.Int(self.num_bytes,
                                                label="Address space size", unit="Bytes")}
     else:
-      self.answers = {"answer": Answer.integer("num_bits", self.num_bits,
+      self.answers = {"answer": AnswerTypes.Int(self.num_bits,
                                                label="Number of bits in address", unit="bits")}
   
   def _get_body(self, **kwargs):
@@ -111,10 +111,10 @@ class HexAndBinary(MathQuestion):
     self.binary_val = f"0b{self.value:0{4*self.number_of_hexits}b}"
     
     if self.from_binary:
-      self.answers['answer'] = Answer.string("hex_val", self.hex_val,
+      self.answers['answer'] = AnswerTypes.String(self.hex_val,
                                              label="Value in hex")
     else:
-      self.answers['answer'] = Answer.string("binary_val", self.binary_val,
+      self.answers['answer'] = AnswerTypes.String(self.binary_val,
                                              label="Value in binary")
   
   def _get_body(self, **kwargs):
@@ -223,8 +223,7 @@ class AverageMemoryAccessTime(MathQuestion):
     self.amat = self.hit_rate * self.hit_latency + (1 - self.hit_rate) * self.miss_latency
     
     self.answers = {
-      "amat": Answer.float_value("answer__amat", self.amat,
-                                 label="Average Memory Access Time", unit="cycles")
+      "amat": AnswerTypes.Float(self.amat, label="Average Memory Access Time", unit="cycles")
     }
     
     # Finally, do the self.rngizing of the question, to avoid these being non-deterministic
@@ -244,7 +243,7 @@ class AverageMemoryAccessTime(MathQuestion):
       ContentAST.Paragraph([
         ContentAST.Text("Please calculate the Average Memory Access Time given the below information. "),
         ContentAST.Text(
-          f"Please round your answer to {Answer.DEFAULT_ROUNDING_DIGITS} decimal points. ",
+          f"Please round your answer to {ContentAST.Answer.DEFAULT_ROUNDING_DIGITS} decimal points. ",
           hide_from_latex=True
         )
       ])
@@ -295,7 +294,7 @@ class AverageMemoryAccessTime(MathQuestion):
         lhs="AMAT",
         rhs=[
           r"(hit\_rate)*(hit\_cost) + (1 - hit\_rate)*(miss\_cost)",
-          f"({self.hit_rate: 0.{Answer.DEFAULT_ROUNDING_DIGITS}f})*({self.hit_latency}) + ({1 - self.hit_rate: 0.{Answer.DEFAULT_ROUNDING_DIGITS}f})*({self.miss_latency}) = {self.amat: 0.{Answer.DEFAULT_ROUNDING_DIGITS}f}\\text{{cycles}}"
+          f"({self.hit_rate: 0.{ContentAST.Answer.DEFAULT_ROUNDING_DIGITS}f})*({self.hit_latency}) + ({1 - self.hit_rate: 0.{ContentAST.Answer.DEFAULT_ROUNDING_DIGITS}f})*({self.miss_latency}) = {self.amat: 0.{ContentAST.Answer.DEFAULT_ROUNDING_DIGITS}f}\\text{{cycles}}"
         ]
       )
     )