parseidon-headings-plugin 1.2__py3-none-any.whl

parseidon_headings_plugin/remove_headings.py

@@ -0,0 +1,83 @@
+import re
+
+import pluggy
+from parseidon.handlers.content import PagePositionContext
+
+hookimpl = pluggy.HookimplMarker(project_name="parseidon")
+
+
+class RemoveHeadingPlugin:
+    @hookimpl
+    def post_process_content(self, output):
+        """
+        Hook for post processing the output from content_handler
+
+        Parameters:
+            output: list of tuples containing documents and their contents
+
+        Returns:
+            The post processed content_handler output, without
+            IPv4-like headings
+        """
+        for _, list_of_contents in output:
+            potential_headings = set()
+            for content in list_of_contents:
+                if isinstance(content.context, PagePositionContext):
+                    content, potential_headings = self.process_wordlist(
+                        content, potential_headings
+                    )
+
+                content, potential_headings = self.process_content_string(
+                    content, potential_headings
+                )
+
+        return output
+
+    def process_wordlist(self, content, potential_headings):
+        """
+        Helper for processing words in text-output mode
+
+        Parameters:
+            content: Content
+
+        Returns: Content without IPv4-like headings
+        """
+        words = content.context.words
+        heading_re = re.compile(r"(\d{1,2}\.)*\d{1,2}\.?")
+        prev = "\n"
+        for ix, word in enumerate(words):
+            word_text = word[0].strip().strip(".")
+            if heading_re.match(word_text) and prev.endswith("\n"):
+                parent = word_text.rsplit(".", 1)[0]
+                if parent == word_text:
+                    potential_headings.add(word_text)
+                    continue
+                if parent in potential_headings:
+                    potential_headings.add(word_text)
+                    words[ix] = ("", [])
+            prev = word[0]
+        return content, potential_headings
+
+    def process_content_string(self, content, potential_headings):
+        """
+        Helper for processing words in text-output mode
+
+        Parameters:
+            content: Content
+
+        Returns: Content without IPv4-like headings
+        """
+        text = content.content
+        heading_re = re.compile(r"(?:(?<=\n)|^)(\d{1,2}\.)*\d{1,2}(?=(\.?(\s+|$)))")
+        for match in heading_re.finditer(text):
+            word = match.group()
+            word_text = word.strip().strip(".")
+            parent = word_text.rsplit(".", 1)[0]
+            if word_text == parent:
+                potential_headings.add(word_text)
+                continue
+            if parent in potential_headings:
+                potential_headings.add(word_text)
+                text = re.sub(word, "\u200b" * len(word), text, 1)
+        content.content = text
+        return content, potential_headings

parseidon_headings_plugin-1.2.dist-info/METADATA

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: parseidon-headings-plugin
+Version: 1.2
+Summary: Plugin removing IPv4-like headings in documents for parseidon
+Author-email: Parseidon Dev Team <parseidon@foi.se>
+License: MIT
+Project-URL: Homepage, https://github.com/CrateOrg/parseidon
+Project-URL: Repository, https://github.com/CrateOrg/parseidon
+Requires-Python: >=3.9
+Requires-Dist: parseidon>=2.2.6

parseidon_headings_plugin-1.2.dist-info/RECORD

@@ -0,0 +1,6 @@
+parseidon_headings_plugin/remove_headings.py,sha256=f4pNWX7RGR1HOYhQiOClUMfoGsy-z9ACTHWNw582zOc,2841
+parseidon_headings_plugin-1.2.dist-info/METADATA,sha256=nsD-koTymlHqAd2Boe3McHspHwCkaxrs7FwmbIjbN8g,382
+parseidon_headings_plugin-1.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+parseidon_headings_plugin-1.2.dist-info/entry_points.txt,sha256=xFe7uIzv4UxVaLtH3Pqa7xIH1jxRq8fayiycZgjLAg8,100
+parseidon_headings_plugin-1.2.dist-info/top_level.txt,sha256=ZO3NekDA1gCKA0Qd_DfCTbocfJLghc0l53fGCl5K_FM,26
+parseidon_headings_plugin-1.2.dist-info/RECORD,,

parseidon_headings_plugin-1.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

parseidon_headings_plugin-1.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[parseidon.plugins]
+headings_plugin = parseidon_headings_plugin.remove_headings:RemoveHeadingPlugin

parseidon_headings_plugin-1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+parseidon_headings_plugin