excel2moodle 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

excel2moodle/__init__.py

@@ -56,9 +56,8 @@ if __package__ is not None:
 settings = Settings()
 logfile = Path(settings.get(SettingsKey.LOGFILE)).resolve()
 e2mMetadata["logfile"] = logfile
-if logfile.exists():
+if logfile.exists() and logfile.is_file():
     logfile.replace(f"{logfile}.old")
 
 mainLogger = logging.getLogger(__name__)
 logConfig.dictConfig(config=loggerConfig)
-qSignalLogger = LogWindowHandler()

excel2moodle/__main__.py

@@ -4,14 +4,13 @@ import signal
 
 from PySide6 import QtWidgets, sys
 
-from excel2moodle import e2mMetadata, mainLogger, qSignalLogger
+from excel2moodle import e2mMetadata, mainLogger
 from excel2moodle.core import dataStructure
 from excel2moodle.ui import appUi as ui
 from excel2moodle.ui.settings import Settings
 
 
 def main() -> None:
-    mainLogger.addHandler(qSignalLogger)
     signal.signal(signal.SIGINT, signal.SIG_DFL)
     app = QtWidgets.QApplication(sys.argv)
     settings = Settings()

excel2moodle/core/category.py

@@ -1,21 +1,18 @@
 import logging
+from typing import TYPE_CHECKING
 
 import lxml.etree as ET
 import pandas as pd
 
-from excel2moodle.core.parser import (
-    MCQuestionParser,
-    NFMQuestionParser,
-    NFQuestionParser,
-    QNotParsedException,
-)
-from excel2moodle.core.question import Question
-from excel2moodle.logger import LogAdapterQuestionID
+if TYPE_CHECKING:
+    from excel2moodle.core.question import Question
 
 loggerObj = logging.getLogger(__name__)
 
 
 class Category:
+    """Category stores a list of question. And holds shared information for all."""
+
     def __init__(
         self,
         n: int,
@@ -25,6 +22,7 @@ class Category:
         points: float = 0,
         version: int = 0,
     ) -> None:
+        """Instantiate a new Category object."""
         self.n = n
         self.NAME = name
         self.desc = str(description)
@@ -51,42 +49,6 @@ class Category:
             return self.NAME == other.NAME
         return False
 
-    def parseQ(
-        self,
-        q: Question,
-        questionData: dict | None = None,
-        xmlTree: ET._Element | None = None,
-    ) -> bool:
-        """Parse the given question."""
-        logger = LogAdapterQuestionID(loggerObj, {"qID": q.id})
-        if q.element is not None:
-            logger.info("Question already parsed")
-            return True
-        if q.qtype == "NF":
-            parser = NFQuestionParser(q, questionData)
-            logger.debug("setup a new NF parser ")
-        elif q.qtype == "MC":
-            parser = MCQuestionParser(q, questionData)
-            logger.debug("setup a new MC parser ")
-        elif q.qtype == "NFM":
-            parser = NFMQuestionParser(q, questionData)
-            logger.debug("setup a new NFM parser ")
-        else:
-            logger.error("couldn't setup Parser")
-            return False
-        try:
-            parser.parse(xmlTree=xmlTree)
-            return True
-        except QNotParsedException as e:
-            logger.critical(
-                "Question couldn't be parsed",
-                exc_info=e,
-                stack_info=True,
-            )
-            return False
-        finally:
-            del parser
-
     def getCategoryHeader(self) -> ET.Element:
         """Insert an <question type='category'> before all Questions of this Category."""
         header = ET.Element("question", type="category")

excel2moodle/core/dataStructure.py

@@ -4,19 +4,26 @@ At the heart is the class ``xmlTest``
 """
 
 import logging
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import lxml.etree as ET  # noqa: N812
 import pandas as pd
 from PySide6 import QtWidgets
+from PySide6.QtCore import QObject, Signal
 
 from excel2moodle.core import stringHelpers
 from excel2moodle.core.category import Category
 from excel2moodle.core.exceptions import InvalidFieldException, QNotParsedException
+from excel2moodle.core.globals import DFIndex
 from excel2moodle.core.question import Question
-from excel2moodle.core.questionValidator import Validator
-from excel2moodle.logger import QSignaler
+from excel2moodle.core.validator import Validator
+from excel2moodle.logger import LogAdapterQuestionID
+from excel2moodle.question_types import QuestionTypeMapping
+from excel2moodle.question_types.mc import MCQuestion, MCQuestionParser
+from excel2moodle.question_types.nf import NFQuestion, NFQuestionParser
+from excel2moodle.question_types.nfm import NFMQuestion, NFMQuestionParser
 from excel2moodle.ui.dialogs import QuestionVariantDialog
 from excel2moodle.ui.settings import Settings, SettingsKey
 from excel2moodle.ui.treewidget import QuestionItem
@@ -27,88 +34,247 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class QuestionDBSignals(QObject):
+    categoryReady = Signal(Category)
+    categoryQuestionsReady = Signal(Category)
+
+
+def processSheet(sheetPath: str, categoryName: str) -> pd.DataFrame:
+    """Parse `categoryName` from the file ``sheetPath`` into the dataframe.
+
+    This Function is meant to be run asynchron for increased speed.
+    """
+    return pd.read_excel(
+        Path(sheetPath),
+        sheet_name=str(categoryName),
+        index_col=0,
+        header=None,
+    )
+
+
 class QuestionDB:
-    """oberste Klasse für den Test."""
+    """The QuestionDB is the main class for processing the Spreadsheet.
+
+    It provides the functionality, for setting up the categories and Questions.
+    Any interaction with the questions are done by its methods.
+    """
 
-    dataChanged = QSignaler()
+    signals = QuestionDBSignals()
+    validator: Validator = Validator()
+    nfParser: NFQuestionParser = NFQuestionParser()
+    nfmParser: NFMQuestionParser = NFMQuestionParser()
+    mcParser: MCQuestionParser = MCQuestionParser()
 
     def __init__(self, settings: Settings) -> None:
         self.settings = settings
         self.window: QMainWindow | None = None
         self.version = None
-        self.categoriesMetaData = pd.DataFrame()
-        self.categories: dict[str, Category] = {}
+        self.categoriesMetaData: pd.DataFrame
+        self.categories: dict[str, Category]
 
-    def readSpreadsheetData(self, sheet: Path) -> None:
+    def readCategoriesMetadata(self, sheetPath: Path) -> None:
         """Read the metadata and questions from the spreadsheet.
 
-        This method gathers this information and stores it in the
+        Get the category data from the spreadsheet and stores it in the
         ``categoriesMetaData`` dataframe
-        It also reads the question data and stores it in ``self.categories = {}``
+        Setup the categories and store them  in ``self.categories = {}``
+        Pass the question data to the categories.
         """
         logger.info("Start Parsing the Excel Metadata Sheet\n")
-        with Path(sheet).open("rb") as f:
-            excelFile = pd.ExcelFile(f)
+        with Path(sheetPath).open("rb") as f:
+            settingDf = pd.read_excel(
+                f,
+                sheet_name="settings",
+                index_col=0,
+            )
+            logger.debug("Found the settings: \n\t%s", settingDf)
+            self._setProjectSettings(settingDf)
+        with Path(sheetPath).open("rb") as f:
             self.categoriesMetaData = pd.read_excel(
                 f,
-                sheet_name="Kategorien",
+                sheet_name=self.settings.get(SettingsKey.CATEGORIESSHEET),
                 usecols=["Kategorie", "Beschreibung", "Punkte", "Version"],
                 index_col=0,
             )
             logger.info("Sucessfully read categoriesMetaData")
-            self.categories = {}
-            for sh in excelFile.sheet_names:
-                if sh.startswith("KAT"):
-                    n = int(sh[4:])
-                    katDf = pd.read_excel(
-                        f,
-                        sheet_name=str(sh),
-                        index_col=0,
-                        header=None,
-                    )
-                    if not katDf.empty:
-                        p = self.categoriesMetaData["Punkte"].iloc[n - 1]
-                        points = p if not pd.isna(p) else 1
-                        v = self.categoriesMetaData["Version"].iloc[n - 1]
-                        version = v if not pd.isna(v) else 0
-                        self.categories[sh] = Category(
-                            n,
-                            sh,
-                            self.categoriesMetaData["Beschreibung"].iloc[n - 1],
-                            dataframe=katDf,
-                            points=points,
-                            version=version,
-                        )
-        # self.dataChanged.signal.emit("whoo")
-
-    def parseAll(self) -> None:
-        self.mainTree = ET.Element("quiz")
-        for c in self.categories.values():
-            validator = Validator(c)
-            for q in c.dataframe.columns:
-                logger.debug(f"Starting to check Validity of {q}")
-                qdat = c.dataframe[q]
-                if isinstance(qdat, pd.Series):
-                    validator.setup(qdat, q)
-                    check = False
-                    try:
-                        check = validator.validate()
-                    except InvalidFieldException as e:
-                        logger.exception(
-                            f"Question {c.id}{q:02d} is invalid.",
-                            exc_info=e,
-                        )
-                    if check:
-                        c.questions[q] = validator.question
-                        try:
-                            c.parseQ(c.questions[q], validator.qdata)
-                        except QNotParsedException as e:
-                            logger.exception(
-                                f"Frage {
-                                    c.questions[q].id
-                                } konnte nicht erstellt werden",
-                                exc_info=e,
-                            )
+
+    def _setProjectSettings(self, settings: pd.DataFrame) -> None:
+        for tag, value in settings.iterrows():
+            self.settings.set(tag, value.iloc[0], local=True)
+
+    def initAllCategories(self, sheetPath: Path) -> None:
+        """Read all category sheets and initialize all Categories."""
+        if not hasattr(self, "categoriesMetaData"):
+            logger.error("Can't process the Categories without Metadata")
+            return
+        if hasattr(self, "categories"):
+            self.categories.clear()
+        else:
+            self.categories: dict[str, Category] = {}
+        with Path(sheetPath).open("rb") as f:
+            excelFile = pd.ExcelFile(f)
+            for categoryName in excelFile.sheet_names:
+                logger.debug("Starting to read category %s", categoryName)
+                if categoryName.startswith("KAT"):
+                    self.initCategory(sheetPath, categoryName)
+
+    def asyncInitAllCategories(self, sheetPath: Path) -> None:
+        """Read all category sheets asynchron and initialize all Categories.
+
+        It does the same as `initAllCategories` but the parsing of the excelfile
+        is done asynchron via `concurrent.futures.ProcessPoolExecutor`
+        """
+        if not hasattr(self, "categoriesMetaData"):
+            logger.error("Can't process the Categories without Metadata")
+            return
+        if hasattr(self, "categories"):
+            self.categories.clear()
+        else:
+            self.categories: dict[str, Category] = {}
+        sheet_names = []
+        with Path(sheetPath).open("rb") as f:
+            excel_file = pd.ExcelFile(f)
+            sheet_names = [
+                name for name in excel_file.sheet_names if name.startswith("KAT_")
+            ]
+        logger.debug("found those caetegory sheets: \n %s ", sheet_names)
+        with ProcessPoolExecutor() as executor:
+            futures = {
+                executor.submit(processSheet, str(sheetPath), sheet): sheet
+                for sheet in sheet_names
+            }
+            for future in as_completed(futures):
+                categoryName = futures[future]
+                try:
+                    categoryDataF = future.result()
+                    categoryNumber = int(categoryName[4:])
+                    self._setupCategory(categoryDataF, categoryName, categoryNumber)
+                    logger.debug("Finished processing %s", categoryName)
+                except Exception as e:
+                    logger.exception("Error processing sheet %s: %s", categoryName, e)
+                    logger.debug("Future exception: %s", future.exception())
+
+    def initCategory(self, sheetPath: Path, categoryName: str) -> None:
+        """Read `categoryName` from the file ``sheetPath`` and initialize the category."""
+        categoryNumber = int(categoryName[4:])
+        katDf = pd.read_excel(
+            sheetPath,
+            sheet_name=str(categoryName),
+            index_col=0,
+            header=None,
+        )
+        if not katDf.empty:
+            logger.debug("Sucessfully read the Dataframe for cat %s", categoryName)
+            self._setupCategory(katDf, categoryName, categoryNumber)
+
+    def _setupCategory(
+        self, categoryDf: pd.DataFrame, categoryName: str, categoryNumber: int
+    ) -> None:
+        """Setup the category from the ``dataframe``.
+        :emits: categoryReady(self) Signal.
+        """  # noqa: D401
+        p = self.categoriesMetaData["Punkte"].iloc[categoryNumber - 1]
+        points = p if not pd.isna(p) else self.settings.get(SettingsKey.POINTS)
+        v = self.categoriesMetaData["Version"].iloc[categoryNumber - 1]
+        version = v if not pd.isna(v) else self.settings.get(SettingsKey.VERSION)
+        category = Category(
+            categoryNumber,
+            categoryName,
+            self.categoriesMetaData["Beschreibung"].iloc[categoryNumber - 1],
+            dataframe=categoryDf,
+            points=points,
+            version=version,
+        )
+        self.categories[categoryName] = category
+        logger.debug("Category %s is initialized", categoryName)
+        self.signals.categoryReady.emit(category)
+
+    def parseAllQuestions(self) -> None:
+        """Parse all question from all categories.
+
+        The categories need to be initialized first.
+        """
+        for category in self.categories.values():
+            self.parseCategoryQuestions(category)
+
+    def parseCategoryQuestions(self, category: Category) -> None:
+        """Parse all questions inside ``category``.
+
+        The category has to be initialized first.
+        """
+        for qNum in category.dataframe.columns:
+            try:
+                self.parseQuestion(category, qNum)
+            except InvalidFieldException as e:
+                logger.exception(
+                    "Question %s%02d is invalid.",
+                    category.id,
+                    qNum,
+                    exc_info=e,
+                )
+            except QNotParsedException as e:
+                logger.exception(
+                    "Frage %s konnte nicht erstellt werden",
+                    category.questions[qNum].id,
+                    exc_info=e,
+                )
+        self.signals.categoryQuestionsReady.emit(category)
+
+    @classmethod
+    def parseQuestion(cls, category: Category, qNumber: int) -> None:
+        """Check if the Question Data is valid. Then parse it.
+
+        The Question data is accessed from  `category.dataframe` via its number
+        First it is checked if all mandatory fields for the given question type
+        are provided.
+        Then in checks, weather the data has the correct type.
+        If the data is valid, the corresponding parser is fed with the data and run.
+
+        Raises
+        ------
+        QNotParsedException
+            If the parsing of the question is not possible this is raised
+        InvalidFieldException
+            If the data of the question is invalid.
+            This gives more information wheather a missing field, or the invalid type
+            caused the Exception.
+
+        """
+        locallogger = LogAdapterQuestionID(
+            logger, {"qID": f"{category.id}{qNumber:02d}"}
+        )
+        locallogger.debug("Starting to check Validity")
+        qdat = category.dataframe[qNumber]
+        if not isinstance(qdat, pd.Series):
+            locallogger.error("cannot validate data that isn't a pd.Series")
+            msg = "cannot validate data that isn't a pd.Series"
+            raise QNotParsedException(msg, f"{category.id}{qNumber}")
+        cls.validator.setup(qdat, qNumber)
+        cls.validator.validate()
+        validData = cls.validator.getQuestionRawData()
+        qtype: str = str(validData.get(DFIndex.TYPE))
+        category.questions[qNumber] = QuestionTypeMapping[qtype].create(
+            category, validData
+        )
+        question = category.questions[qNumber]
+        if question.element is not None:
+            locallogger.info("Question already parsed")
+            return
+        if isinstance(question, NFQuestion):
+            cls.nfParser.setup(question)
+            cls.nfParser.parse()
+            locallogger.debug("setup a new NF parser ")
+        elif isinstance(question, MCQuestion):
+            cls.mcParser.setup(question)
+            cls.mcParser.parse()
+            locallogger.debug("setup a new MC parser ")
+        elif isinstance(question, NFMQuestion):
+            cls.nfmParser.setup(question)
+            cls.nfmParser.parse()
+            locallogger.debug("setup a new NFM parser ")
+        else:
+            msg = "couldn't setup Parser"
+            raise QNotParsedException(msg, question.id)
 
     def appendQuestions(
         self, questions: list[QuestionItem], file: Path | None = None
@@ -143,18 +309,15 @@ class QuestionDB:
             logger.debug(f"Appended a new category item {cat=}")
         variant: int = self.settings.get(SettingsKey.QUESTIONVARIANT)
         for q in qList:
-            if cat.parseQ(q):
-                if q.variants is not None:
-                    if variant == 0 or variant > q.variants:
-                        dialog = QuestionVariantDialog(self.window, q)
-                        if dialog.exec() == QtWidgets.QDialog.Accepted:
-                            variant = dialog.variant
-                            logger.debug(f"Die Fragen-Variante {variant} wurde gewählt")
-                            q.assemble(variant)
-                        else:
-                            pass
-                else:
-                    q.assemble()
-                tree.append(q.element)
+            if q.variants is not None:
+                if variant == 0 or variant > q.variants:
+                    dialog = QuestionVariantDialog(self.window, q)
+                    if dialog.exec() == QtWidgets.QDialog.Accepted:
+                        variant = dialog.variant
+                        logger.debug("Die Fragen-Variante %s wurde gewählt", variant)
+                        q.assemble(variant)
+                    else:
+                        logger.warning("Keine Fragenvariante wurde gewählt.")
             else:
-                logger.warning(f"Frage {q} wurde nicht erstellt")
+                q.assemble()
+            tree.append(q.element)

excel2moodle/core/globals.py

@@ -120,23 +120,3 @@ feedbackStr = {
     "wrong": "falsch",
     "right1Percent": "Gratultaion, die Frage wurde im Rahmen der Toleranz richtig beantwortet",
 }
-
-parserSettings = {
-    "Parser": {
-        "standards": {
-            "hidden": 0,
-        },
-    },
-    "MCParser": {
-        "standards": {
-            "single": "false",
-            "shuffleanswers": "true",
-            "answernumbering": "abc",
-            "showstandardinstruction": "0",
-            "shownumcorrect": "",
-        },
-    },
-    "NFParser": {
-        "standards": {},
-    },
-}