qparse 0.1.0__py3-none-any.whl

qparse/markdwon_document/nodes/base_node.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup,element
+from typing import Union
+import copy
+class BaseNode():
+    def __init__(self,node:Union[element.Tag,element.NavigableString],render_option:dict):
+        self._soup = node
+        self._render_option = render_option
+        
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        
+        return None
+    def dump_markdwon(self):
+        return ''
+    @property
+    def ParentMarkdown(self):
+        return self._soup.find_parent("markdwon")
+    @property
+    def QuestionSoup(self):
+        return self._soup.find_parent("question")
+    @property
+    def depth(self)->int:
+        if p:=self.ParentMarkdown:
+           
+            return int( p.attrs.get('depth',0))
+        
+        return 0
+    def dump_soup(self):
+        pass
+    
+    @property
+    def attrs(self):
+        return copy.deepcopy(getattr(self._soup,'attrs',{}))
+    def dump_docx_buffer(self):
+        return []
+        
+    
+        
+    
+    
+    
+
+    
+
+
+
+    
+    
+    
+        

qparse/markdwon_document/nodes/img_node.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup,element
+from typing import Union
+import copy
+from .base_node import BaseNode
+
+class ImgNode(BaseNode):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if isinstance(node,element.Tag) and node.name=="img":
+            return cls(node=node,render_option=render_option)
+        return None
+    
+    def dump_soup(self):
+        return copy.deepcopy(self._soup)
+    def dump_markdwon(self):
+        alt = self._soup.attrs.get("alt", "")
+        src = self._soup.attrs.get("src", "")
+        title = self._soup.attrs.get("title")
+
+        if not src:
+            return ""
+
+        if title:
+            return f'![{alt}]({src} "{title}")'
+        return f'![{alt}]({src})'
+    
+    
+    

qparse/markdwon_document/nodes/text_node.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+from mistletoe import Document
+from mistletoe.html_renderer import HtmlRenderer
+from mistletoe.markdown_renderer import MarkdownRenderer
+from mistletoe.block_token import Heading
+from mistletoe.span_token import RawText
+from bs4 import BeautifulSoup,element
+from typing import Union
+from .base_node import BaseNode
+from qparse.utils.html_full_protector import HtmlFullProtector
+from qparse.render_option import SafeGetRenderOption,render_template
+class TextNode(BaseNode):
+    htmlParser = HtmlFullProtector()
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+
+    @classmethod
+    def create_by_node(cls,node:element.NavigableString,render_option:dict):
+        text = node.get_text(strip=True)
+        
+        if isinstance(node,element.NavigableString):
+            return cls(node=node,render_option=render_option)
+        return None
+    
+    def __repr__(self):
+        return '<{} text="{}...">'.format(self.__class__.__name__,self._soup.get_text(strip=True)[:10])
+   
+    @property
+    def text(self):
+        return self._soup.get_text()
+
+    @property
+    def unicode_text(self):
+        return self.htmlParser.decode_to_unicode(self.text)
+        
+    def dump_markdwon(self):
+        
+       
+        doc: Document = Document(self.unicode_text)
+        stack = [doc]
+        while len(stack):
+            block = stack.pop(0)
+            if isinstance(block, Heading):
+                block.level = self.depth + block.level
+            if children:=getattr(block,'children'):
+                stack = [ *children,*stack]
+                
+        with MarkdownRenderer() as r:
+            text=r.render(doc)
+            text = text.replace("\n",'')
+        return self.htmlParser.decode_to_unicode(text) 
+    def dump_soup(self):
+        markdwon_text = self.dump_markdwon()
+        doc: Document = Document(markdwon_text)
+        with HtmlRenderer() as r:
+            soup_text = r.render(doc)
+            return BeautifulSoup(soup_text,features='html.parser')
+        
+    def dump_docx_buffer(self):
+        return [{'type':'markdwonText','text':self.dump_markdwon()}]
+    
+class BrNode(BaseNode):
+    @classmethod
+    def create_by_node(cls, node: element.Tag, render_option: dict):
+        if isinstance(node, element.Tag) and node.name == "br":
+            return cls(node=node, render_option=render_option)
+        return None
+
+    def dump_markdwon(self):
+        return "\n"
+
+    def dump_soup(self):
+        return element.Tag(name="br")
+    
+    def dump_docx_buffer(self):
+        
+        return [{'type':'markdwonText','text':self.dump_markdwon()}]
+        
+
+class FirstTextVisibleNode(TextNode):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+        
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        
+        if isinstance(node,element.Tag) and node.name=="first-text-visible":
+            
+            return cls(node=node.children.__next__(),render_option=render_option)
+        return None
+    @property
+    def template(self):
+        return SafeGetRenderOption(
+            self._render_option,
+            [
+                "question",
+                self._soup.parent.attrs['include-by'],
+                "::first-text-visible",
+                'template'
+            ],
+            r"[% text %]"
+        )
+    
+   
+    
+    def dump_markdwon(self):
+        text =  super().dump_markdwon()
+        # print(text)
+        doc: Document = Document(text)
+        stack = [doc]
+        text_index = 0
+        while len(stack):
+            block = stack.pop(0)
+            # print(type(block) )
+            if children:=getattr(block,'children'):
+                stack = [ *children,*stack]
+                
+            if isinstance(block,RawText) and text_index==0:
+                
+                text_index +=1
+                block.content = render_template(template_text=self.template,context={
+                    'text':block.content,
+                    **self.QuestionSoup.attrs
+                })
+            
+            
+        with MarkdownRenderer() as r:
+            text=  r.render(doc)
+            text = text.replace("\n",'')
+        return self.htmlParser.decode_to_unicode(text) 
+        
+    
+    
+    
+        
+                
+    
+    
+    

qparse/markdwon_document/question_document.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup,element
+from typing import Dict,Union
+from .base_document import BaseDocument
+
+class QuestionDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+    
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="question":
+            return cls(node=node,render_option=render_option)
+        return None
+    
+    
+

qparse/markdwon_document/stem_document.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup,element
+from typing import Dict,Union
+from .base_document import BaseDocument
+from qparse.utils.html_full_protector import HtmlFullProtector
+from qparse.render_option import SafeGetRenderOption, render_template
+from markupsafe import Markup
+class StemDocument(BaseDocument):
+    htmlParser = HtmlFullProtector()
+    def __init__(self, node, render_option):
+        self.show = SafeGetRenderOption(option=render_option,path=['question','stem','show'] ,default=False)
+        
+        super().__init__(node, render_option)
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+       
+        if node.name=="stem":
+            # return cls(node=node,render_option=render_option)
+            for n in cls._iter_nodes(node):
+                if isinstance(n,element.NavigableString):
+                    text = cls.htmlParser.decode_to_unicode(n.get_text())
+                    if text.strip():
+                        
+                        # s = element.NavigableString()
+                        tag = element.Tag(name="first-text-visible")
+                        tag.append(n.get_text())
+                        tag.attrs['include-by']= "stem"
+                        n.replace_with(tag)
+                        
+                        break
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_markdwon(self):
+        if not self.show:
+            return ''
+
+        text = super().dump_markdwon()
+        question = self._soup.find_parent('question')
+        answer_types = question.attrs.get('answer_types', '') if question else ''
+        template = SafeGetRenderOption(
+            self._render_option,
+            ['question', 'stem', f'[answer_types="{answer_types}"]', 'template'],
+            default='[%text%]'
+        )
+        return render_template(
+            template_text=template,
+            context={
+                'text': Markup(text),
+                **(question.attrs if question else {})
+            }
+        )
+    def dump_docx_buffer(self):
+        if not self.show:
+            return []
+        keep_together= SafeGetRenderOption(self.render_option,['question', 'stem']).get('keep_together',True)
+        node = {'type':"document",'keep_together':True,'children':[]}
+        if not keep_together:
+            return super().dump_docx_buffer()
+        for c in self.children:
+            node['children'].extend(c.dump_docx_buffer())
+        return [node]
+        
+        
+

qparse/markdwon_document/table_document.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup,element
+from typing import Dict,Union
+from .base_document import BaseDocument
+import copy
+
+
+class TdDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+        
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="td":
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_soup(self):
+        tag = element.Tag(name='td',attrs=copy.deepcopy(self._soup.attrs) )
+        
+        for child in self.children:
+            sub_soup = child.dump_soup()
+            if sub_soup.name!="br":
+                tag.append(child.dump_soup())
+        return tag
+    
+    
+class ThDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+        
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="th":
+            return cls(node=node,render_option=render_option)
+        return None
+class TrDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+        
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="tr":
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_soup(self):
+        
+        tag = element.Tag(name='tr',attrs=copy.deepcopy(self._soup.attrs) )
+        
+        for child in self.children:
+            if isinstance(child,TdDocument):
+                tag.append(child.dump_soup())
+        return tag
+
+
+
+class TBodyDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+    
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="tbody":
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_soup(self):
+        tag = element.Tag(name='tbody',attrs=copy.deepcopy(self._soup.attrs))
+        for child in self.children:
+            if isinstance(child,TrDocument):
+                tag.append(child.dump_soup())
+        return tag
+    
+    def dump_markdwon(self):
+        soup = self.dump_soup()
+        return soup.prettify()
+    
+    
+class THeadDocument(BaseDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+    
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="thead":
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_soup(self):
+        tag = element.Tag(name='thead',attrs=copy.deepcopy(self._soup.attrs))
+        for child in self.children:
+            if isinstance(child,TrDocument):
+                tag.append(child.dump_soup())
+        return tag
+                
+    
+    
+class TableDocument(TBodyDocument):
+    def __init__(self, node, render_option):
+        super().__init__(node, render_option)
+    
+    @classmethod
+    def create_by_node(cls,node:element.Tag,render_option:dict):
+        if node.name=="table":
+            return cls(node=node,render_option=render_option)
+        return None
+    def dump_soup(self):
+        tag = element.Tag(name='table',attrs=copy.deepcopy(self._soup.attrs))
+        for child in self.children:
+            
+            if isinstance(child,TBodyDocument) or isinstance(child,TrDocument):
+                if sub_soup:= child.dump_soup() :
+                    if  sub_soup.name!="br":
+                        tag.append(sub_soup)
+        return tag
+                
+    def dump_markdwon(self):
+        soup = self.dump_soup()
+        # print(soup)
+        # return ''
+        return "\n"+soup.prettify()+"\n"
+    def dump_docx_buffer(self):
+        return [{"type":'tableHtmlText',"text":self.dump_markdwon()}]
+    
+THeadDocument.register_node_parser('tr',TrDocument)
+TBodyDocument.register_node_parser('tr',TrDocument)
+
+TrDocument.register_node_parser('td',TdDocument)
+TrDocument.register_node_parser('th',ThDocument)

qparse/markdwon_loader.py

@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+from cgitb import text
+import re,copy
+from pathlib import Path
+
+from bs4 import BeautifulSoup, element
+
+from .utils import HtmlFullProtector
+
+
+class MarkdownLoader:
+    def __init__(self, src: str):
+        self._src = Path(src)
+        self._protector = HtmlFullProtector()
+        self._reference_answer = BeautifulSoup("<div></div>", "html.parser").div
+
+    def _read_text(self, path: Path) -> str:
+        return path.read_text(encoding="utf-8")
+
+    def _to_absolute_url(self, raw_url: str, base_dir: Path) -> str:
+        url = raw_url.strip().strip('"').strip("'")
+        if re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", url) or url.startswith("/"):
+            return url
+        return str((base_dir / url).resolve())
+
+    def _replace_images(self, text: str, base_dir: Path) -> str:
+        pattern = re.compile(r'!\[([^\]]*)\]\(([^)]+)\)')
+
+        def repl(match: re.Match) -> str:
+            alt = match.group(1)
+            url = self._to_absolute_url(match.group(2), base_dir)
+            return f'<img alt="{alt}" src="{url}" />'
+
+        return pattern.sub(repl, text)
+
+    def _replace_imports(self, text: str, base_dir: Path) -> str:
+        pattern = re.compile(r'^\s*@import\s+["\']([^"\']+)["\']\s*$', re.MULTILINE)
+
+        def repl(match: re.Match) -> str:
+            url = self._to_absolute_url(match.group(1), base_dir)
+            return f'<markdwon src="{url}"></markdwon>'
+
+        return pattern.sub(repl, text)
+
+    def _load_markdown(self, path: Path) -> str:
+        base_dir = path.resolve().parent
+        text = self._read_text(path.resolve())
+        text = self._replace_images(text, base_dir)
+        text = self._replace_imports(text, base_dir)
+        return text
+
+    def _expand_imports(self, soup: BeautifulSoup, current_path: Path, stack: set[Path] | None = None, depth: int = 0) -> None:
+        stack = set() if stack is None else stack
+        current_resolved = current_path.resolve()
+        if current_resolved in stack:
+            return
+        stack.add(current_resolved)
+
+        # 给根 soup 加上 depth 属性！如果是根的话 depth 是 0！
+        if hasattr(soup, "markdwon_depth"):
+            soup.markdwon_depth = depth
+        else:
+            # 对于 BeautifulSoup 根，我们给根的标签（如果是 markdwon 的话）加 depth
+            if soup.contents and hasattr(soup.contents[0], "name") and soup.contents[0].name == "markdwon":
+                soup.contents[0]["depth"] = depth
+
+        for node in list(soup.find_all("markdwon")):
+            src = node.get("src")
+            if not src:
+                continue
+
+            imported_path = Path(src).resolve()
+            if imported_path in stack:
+                node["error"] = "circular import"
+                continue
+
+            if not imported_path.exists():
+                node["error"] = "import not found"
+                continue
+
+            # 这个 markdwon 标签应该是 depth + 1
+            node["depth"] = depth + 1
+            imported_markdown = self._load_markdown(imported_path)
+            protected = self._protector.encode(imported_markdown)
+            decoded = self._protector.decode_to_html_entity(protected)
+            imported_soup = BeautifulSoup(decoded, "html.parser")
+            self._expand_imports(imported_soup, imported_path, stack, depth + 1)
+            node.clear()
+            node.extend(list(imported_soup.contents))
+
+        stack.remove(current_resolved)
+
+    def _decorate_questions(self, soup: BeautifulSoup) -> None:
+        answer_class_map = {
+            "chiose": "choice",
+            "multiple-choice": "multiple-choice",
+            "multiple-chiose": "multiple-choice",
+            "blank": "blank",
+            "answer": "answer",
+        }
+
+        for question_id, question in enumerate(soup.find_all("question"), start=1):
+            answer_types = []
+            question["question_id"] = str(question_id)
+
+            for span in question.find_all("span"):
+                class_list = span.get("class", [])
+                for class_name in class_list:
+                    answer_type = answer_class_map.get(class_name)
+                    if not answer_type:
+                        continue
+                    span["type"] = "answer"
+                    if answer_type not in answer_types:
+                        answer_types.append(answer_type)
+
+            if answer_types:
+                question["answer_types"] = ",".join(answer_types)
+
+    def get_soup(self) -> BeautifulSoup:
+        markdown = self._load_markdown(self._src)
+        protected = self._protector.encode(markdown)
+        decoded = self._protector.decode_to_html_entity(protected)
+        root_src = str(self._src.resolve())
+        wrapped = f'<markdwon src="{root_src}" depth="0">{decoded}</markdwon>'
+        soup = BeautifulSoup(wrapped, "html.parser")
+        self._expand_imports(soup, self._src, set(), depth=0)
+        self._decorate_questions(soup)
+        
+        return copy.deepcopy(soup.select_one('markdwon'))
+
+    def get_reference_answer_markdwon(self) -> str:
+        lines = []
+        table_header = []
+        table_answers = []
+
+        def flush_table():
+            nonlocal table_header, table_answers
+            if not table_header:
+                return
+            lines.append("| " + " | ".join(table_header) + " |")
+            lines.append("| " + " | ".join(["---"] * len(table_header)) + " |")
+            lines.append("| " + " | ".join(table_answers) + " |")
+            lines.append("")
+            table_header = []
+            table_answers = []
+
+        soup = self.get_soup()
+
+        for question in soup.find_all("question"):
+            answer_nodes = question.find_all(
+                "span",
+                attrs={"type": "answer"}
+            )
+            if not answer_nodes:
+                continue
+
+            answer = "".join(
+                answer_node.get_text()
+                for answer_node in answer_nodes
+            )
+            answer_question = answer_nodes[0].find_parent("question")
+            if answer_question is None:
+                continue
+
+            question_id = answer_question.get("question_id", "")
+            answer_types = set(
+                answer_type.strip()
+                for answer_type in answer_question.get(
+                    "answer_types",
+                    ""
+                ).split(",")
+                if answer_type.strip()
+            )
+
+            if "answer" in answer_types:
+                flush_table()
+                lines.append(f"{question_id}、{answer}")
+                lines.append("")
+                continue
+
+            table_header.append(question_id)
+            table_answers.append(answer)
+
+        flush_table()
+        return "\n".join(lines).rstrip() + "\n"
+
+