webpageinfo 2022.5.27__tar.gz

webpageinfo-2022.5.27/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Vincent Poulailleau
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

webpageinfo-2022.5.27/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: webpageinfo
+Version: 2022.5.27
+Summary: Web page information (title, content, tags…)
+Home-page: https://github.com/vpoulailleau/webpageinfo
+License: BSD-3-Clause
+Keywords: web,page,information
+Author: Vincent Poulailleau
+Author-email: vpoulailleau@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Utilities
+Requires-Dist: beautifulsoup4 (>=4,<5)
+Requires-Dist: httpx (>=0,<1)
+Requires-Dist: pdfminer.six (>=20220524,<20220525)
+Requires-Dist: pillow (>=9,<10)
+Requires-Dist: splinter[selenium4] (>=0,<1)
+Project-URL: Documentation, https://webpageinfo.readthedocs.io/en/latest/
+Project-URL: Repository, https://github.com/vpoulailleau/webpageinfo
+Description-Content-Type: text/markdown
+
+# WebPageInfo
+
+Web page information (title, content, tags…)
+
+Internal tool, no public documentation…
+

webpageinfo-2022.5.27/README.md

@@ -0,0 +1,5 @@
+# WebPageInfo
+
+Web page information (title, content, tags…)
+
+Internal tool, no public documentation…

webpageinfo-2022.5.27/pyproject.toml

@@ -0,0 +1,53 @@
+[tool.poetry]
+name = "webpageinfo"
+version = "2022.5.27"
+description = "Web page information (title, content, tags…)"
+classifiers=[
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Utilities",
+]
+authors = ["Vincent Poulailleau <vpoulailleau@gmail.com>"]
+readme = "README.md"
+repository = "https://github.com/vpoulailleau/webpageinfo"
+homepage = "https://github.com/vpoulailleau/webpageinfo"
+documentation = "https://webpageinfo.readthedocs.io/en/latest/"
+keywords = ["web", "page", "information"]
+license = "BSD-3-Clause"
+include = ["webpageinfo/**/*.py"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<4.0"
+beautifulsoup4 = "^4"
+httpx = "^0"
+"pdfminer.six" = "^20220524"
+pillow = "^9"
+splinter = {version = "^0", extras = ["selenium4"]}
+
+[tool.poetry.dev-dependencies]
+python-dev-tools = "^2022"
+
+#[tool.poetry.scripts]
+#webpageinfo = "webpageinfo.cli:main"
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+addopts = "-ra -q  -s -vv --cov=webpageinfo"
+testpaths = [
+    "tests",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]
+
+[build-system]
+requires = ["poetry_core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

webpageinfo-2022.5.27/setup.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+packages = \
+['webpageinfo', 'webpageinfo.config']
+
+package_data = \
+{'': ['*']}
+
+install_requires = \
+['beautifulsoup4>=4,<5',
+ 'httpx>=0,<1',
+ 'pdfminer.six>=20220524,<20220525',
+ 'pillow>=9,<10',
+ 'splinter[selenium4]>=0,<1']
+
+setup_kwargs = {
+    'name': 'webpageinfo',
+    'version': '2022.5.27',
+    'description': 'Web page information (title, content, tags…)',
+    'long_description': '# WebPageInfo\n\nWeb page information (title, content, tags…)\n\nInternal tool, no public documentation…\n',
+    'author': 'Vincent Poulailleau',
+    'author_email': 'vpoulailleau@gmail.com',
+    'maintainer': None,
+    'maintainer_email': None,
+    'url': 'https://github.com/vpoulailleau/webpageinfo',
+    'packages': packages,
+    'package_data': package_data,
+    'install_requires': install_requires,
+    'python_requires': '>=3.10,<4.0',
+}
+
+
+setup(**setup_kwargs)

webpageinfo-2022.5.27/webpageinfo/browser.py

@@ -0,0 +1,89 @@
+from collections.abc import Iterable
+
+import splinter
+from splinter.driver.webdriver import BaseWebDriver
+
+
+class WebElementNotFound(Exception):
+    pass
+
+
+def _find_elements_by(
+    root, tag: str = "", class_name: str = "", css: str = "", wait_time: float = 0
+) -> Iterable["WebElement"]:
+    if tag:
+        for elem in root.find_by_tag(tag, wait_time):
+            yield WebElement(elem)
+    if class_name:
+        for elem in root.find_by_css(f".{class_name}", wait_time):
+            yield WebElement(elem)
+    if css:
+        for elem in root.find_by_css(css, wait_time):
+            yield WebElement(elem)
+
+
+class _Findable:
+    @property
+    def _root(self):
+        raise NotImplemented
+
+    def find_elements_by(
+        self, tag: str = "", class_name: str = "", css: str = "", wait_time: float = 0
+    ) -> Iterable["WebElement"]:
+        yield from _find_elements_by(self._root, tag, class_name, css, wait_time)
+
+    def find_element_by(
+        self, tag: str = "", class_name: str = "", css: str = "", wait_time: float = 0
+    ) -> "WebElement":
+        try:
+            return next(
+                iter(_find_elements_by(self._root, tag, class_name, css, wait_time))
+            )
+        except StopIteration:
+            raise WebElementNotFound
+
+    def is_element_present_by(self, tag: str = "", wait_time: float = 0) -> bool:
+        if tag:
+            return self._root.is_element_present_by_tag(tag)
+
+
+class WebElement(_Findable):
+    def __init__(self, implementation_element):
+        self._element = implementation_element
+
+    @property
+    def _root(self):
+        return self._element
+
+    def __getattr__(self, name):
+        """Proxy for internal implementation."""
+        return getattr(self._element, name)
+
+    def __getitem__(self, key):
+        """Proxy for internal implementation."""
+        return self._element[key]
+
+
+class Browser(_Findable):
+    def __init__(self, *args, **kwargs):
+        profile_preferences = {"intl.accept_languages": "fr-FR,en-US,en-GB"}
+        self.driver: BaseWebDriver = splinter.Browser(
+            "firefox",
+            *args,
+            incognito=True,
+            profile_preferences=profile_preferences,
+            **kwargs,
+        )
+        self.selenium_driver.set_window_size(1024, 1024)
+
+    @property
+    def _root(self):
+        return self.driver
+
+    @property
+    def selenium_driver(self):
+        return self.driver.driver
+
+    def __getattr__(self, name):
+        """Proxy for internal implementation."""
+        return getattr(self.driver, name)

webpageinfo-2022.5.27/webpageinfo/config/ignored_html_tags.py

@@ -0,0 +1,13 @@
+ignored_html_tags = {
+    "article",
+    "aside",
+    "figure",
+    "footer",
+    "header",
+    "label",
+    "main",
+    "section",
+    "show",
+    "time",
+    "tt",
+}

webpageinfo-2022.5.27/webpageinfo/config/ignored_word_in_tags.py

@@ -0,0 +1,63 @@
+ignored_word_in_tags = {
+    "about",
+    "apprenants",
+    "available",
+    "cette",
+    "command",
+    "créé",
+    "créée",
+    "default",
+    "des",
+    "different",
+    "directory",
+    "elles",
+    "est",
+    "était",
+    "example",
+    "exemple",
+    "first",
+    "following",
+    "for",
+    "from",
+    "how",
+    "however",
+    "http",
+    "https",
+    "ils",
+    "import",
+    "included",
+    "information",
+    "jusqu",
+    "learn",
+    "les",
+    "name",
+    "named",
+    "never",
+    "note",
+    "other",
+    "petit",
+    "project",
+    "readme",
+    "should",
+    "section",
+    "source",
+    "specify",
+    "super",
+    "sur",
+    "system",
+    "that",
+    "the",
+    "there",
+    "these",
+    "this",
+    "un",
+    "une",
+    "use",
+    "using",
+    "version",
+    "vous",
+    "which",
+    "with",
+    "you",
+    "your",
+}

webpageinfo-2022.5.27/webpageinfo/config/rejected_content_class.py

@@ -0,0 +1,10 @@
+rejected_content_class = (
+    "author-description",
+    "author-info",
+    "d-md-none",
+    "d-none",
+    "footer",
+    "jp-relatedposts",
+    "modal",
+    "sharedaddy",
+)

webpageinfo-2022.5.27/webpageinfo/config/rejected_content_id.py

@@ -0,0 +1,5 @@
+rejected_content_id = (
+    "author",
+    "reader-comments",
+    "sidebar-toc",
+)

webpageinfo-2022.5.27/webpageinfo/config/removed_html_tags.py

@@ -0,0 +1,13 @@
+removed_html_tags = {
+    "button",
+    "canvas",
+    "form",
+    "hr",
+    "input",
+    "link",
+    "meta",
+    "nav",
+    "script",
+    "style",
+    "video-js",
+}

webpageinfo-2022.5.27/webpageinfo/config/simple_html_tags.py

@@ -0,0 +1,41 @@
+simple_html_tags = {
+    "a",
+    "b",
+    "bdi",
+    "blockquote",
+    "br",
+    "code",
+    "del",
+    "details",
+    "div",
+    "em",
+    "figcaption",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "h7",
+    "i",
+    "ins",
+    "li",
+    "ol",
+    "p",
+    "picture",
+    "pre",
+    "small",
+    "source",
+    "span",
+    "strong",
+    "sub",
+    "summary",
+    "sup",
+    "table",
+    "td",
+    "textarea",
+    "th",
+    "tr",
+    "u",
+    "ul",
+}

webpageinfo-2022.5.27/webpageinfo/config/svg_tags.py

@@ -0,0 +1,20 @@
+svg_tags = {
+    "area",
+    "circle",
+    "defs",
+    "fecolormatrix",
+    "fecomponenttransfer",
+    "fecomposite",
+    "fefunca",
+    "fefuncb",
+    "fefuncg",
+    "fefuncr",
+    "filter",
+    "g",
+    "map",
+    "path",
+    "polygon",
+    "polyline",
+    "svg",
+    "symbol",
+}

webpageinfo-2022.5.27/webpageinfo/page.py

@@ -0,0 +1,267 @@
+"""Page information."""
+# TODO get canonical url: <link rel="canonical" href="https://leszexpertsfle.com/ressources-fle/raconter-une-histoire-damour-b1-et/" />
+# TODO regarder dans les iframe pour trouver des liens et du contenu et des PDF
+
+import math
+import re
+import urllib.parse
+import uuid
+from collections import Counter
+from contextlib import suppress
+from pathlib import Path
+from typing import Iterable
+
+import httpx
+import pdfminer.pdfparser
+from pdfminer.high_level import extract_text
+from PIL import Image
+
+from webpageinfo.browser import Browser, WebElementNotFound
+from webpageinfo.config.ignored_word_in_tags import ignored_word_in_tags
+from webpageinfo.safe_html import html2text, safe_html
+
+
+def get_random_temp_file_name(extension: str = "") -> str:
+    """Return a random filename in /tmp."""
+    return f"/tmp/{uuid.uuid4()}.{extension}"
+
+
+def extract_tags_from_words(words: list[str]) -> Iterable[str]:
+    counter = Counter(words)
+    nb_words = len(words)
+    threshold = math.sqrt(nb_words) / 5
+    # print(threshold)
+    result = []
+    result_lower = set()
+    for word, nb_apparitions in sorted(
+        counter.most_common(),
+        key=lambda c: c[1],
+        reverse=True,
+    ):
+        if nb_apparitions < max(threshold, 5.1):
+            break
+        if len(word) <= 4 and not word[0].isupper():
+            continue
+        if len(word) <= 2:
+            continue
+        if word.lower() in ignored_word_in_tags:
+            continue
+        if word.lower() not in result_lower:
+            result_lower.add(word.lower())
+            result.append(word)
+        # print(word, nb_apparitions)
+    return result
+
+
+class Page:
+    """Page information."""
+
+    def _fetch_title(self: "Page", browser: Browser) -> None:
+        # wordpress
+        with suppress(WebElementNotFound):
+            title = browser.find_element_by(class_name="entry-title").text
+            if title:
+                self.title = title
+                return
+
+        with suppress(WebElementNotFound):
+            head = browser.find_element_by(tag="head")
+            title = title = head.find_element_by(tag="title").text
+            if title:
+                self.title = title
+                return
+
+        self.title = browser.title
+
+    def _fetch_links(self: "Page", browser: Browser) -> None:
+        # avoid javascript, read from source
+        html = httpx.get(self.url, follow_redirects=True)
+        html.raise_for_status()
+        for match in re.finditer(r'<a[^>]*href="([^"]+)"', html.text, re.MULTILINE):
+            self.links.add(self.to_absolute_url(match[1]))
+
+        # use javascript
+        for a_tag in browser.find_elements_by(tag="a"):
+            href = a_tag["href"]
+            if href:
+                self.links.add(self.to_absolute_url(href))
+
+    def _fetch_tags(self: "Page", browser: Browser) -> None:
+        for link in browser.find_elements_by(tag="a"):
+            if "tag" in link["rel"]:
+                self.tags.add(link.text)
+        for tag in browser.find_elements_by(class_name="tag__link"):
+            self.tags.add(tag.text)
+
+    def _fetch_description(self: "Page", browser: Browser) -> None:
+        metas = list(browser.find_elements_by(tag="meta"))
+
+        # should be ordered by priority, first is most relevant
+        meta_keywords = (
+            "description",
+            "og:description",
+            "twitter:description",
+            "keywords",
+        )
+
+        for keyword in meta_keywords:
+            for meta in metas:
+                name = meta["name"] or meta["property"]
+                if name == keyword and meta["content"].strip():
+                    self.description = meta["content"].strip()
+                    return
+
+    def _fetch_content(self: "Page", browser: Browser) -> None:
+        content_element = browser.find_element_by(tag="body")
+        try:
+            content_elements = list(browser.find_elements_by(tag="article"))
+            if len(content_elements) == 1:
+                content_element = content_elements[0]
+        except WebElementNotFound:
+            pass
+
+        content_classes = (
+            "et_pb_post_content",
+            "blog-post-content",
+            "entry-content",
+            "post-article",
+        )
+        for class_ in content_classes:
+            try:
+                content_element = browser.find_element_by(class_name=class_)
+                break
+            except WebElementNotFound:
+                pass
+
+        self.content = content_element.html
+
+    def _fetch_screenshot(self: "Page", browser: Browser) -> None:
+        self.screenshot = get_random_temp_file_name(extension="png")
+        browser.selenium_driver.save_screenshot(self.screenshot)
+        image = Image.open(self.screenshot)
+        image.thumbnail((1000, 1000))
+        image.save(self.screenshot, optimize=True, quality=40)
+
+    def _fetch_pdf_content(self: "Page") -> None:
+        for pdf_url in self.attached_pdf:
+            print("Downloading", pdf_url)
+            pdf = httpx.get(pdf_url, follow_redirects=True)
+            pdf.raise_for_status()
+            tmp_file_path = get_random_temp_file_name(extension="pdf")
+            Path(tmp_file_path).write_bytes(pdf.content)
+            print("Saved in", tmp_file_path)
+            with suppress(pdfminer.pdfparser.PDFSyntaxError):
+                self.pdf_content += f"{extract_text(tmp_file_path)} "
+
+    def _fetch_information(self: "Page") -> None:
+        browser = Browser(headless=True)
+        browser.visit(self.url)
+        if not browser.is_element_present_by(tag="title", wait_time=10):
+            raise ValueError("No internet access")
+        if not browser.is_element_present_by(tag="body", wait_time=10):
+            raise ValueError("No internet access")
+        # browser.find_by_name('btnK').click()
+        # if browser.is_text_present('splinter.readthedocs.io'):
+        # browser.is_text_present('splinter', wait_time=10) # True, using wait_time
+        # browser.is_element_present_by_tag('h1')
+        # screenshot_path = browser.screenshot('absolute_path/your_screenshot.png', full=True)
+        # time.sleep(10)
+        self._fetch_title(browser)
+        self._fetch_links(browser)
+        self._fetch_tags(browser)
+        self._fetch_description(browser)
+        self._fetch_content(browser)
+        self._fetch_screenshot(browser)
+        self._fetch_pdf_content()
+
+        self.safe_html = safe_html(self.content, self.server)
+        self.text = html2text(self.safe_html)
+        self.total_text = re.sub(
+            r"[\n\s\W_]+",
+            " ",
+            f"{self.text} {self.pdf_content} {self.description} {' '.join(self.tags)}",
+            flags=re.DOTALL | re.MULTILINE,
+        )
+        self.words = self.total_text.split()
+        self.words_tags = extract_tags_from_words(self.words)
+        browser.quit()
+
+    def __init__(self: "Page", url: str) -> None:
+        self.url: str = url
+        url = urllib.parse.urlparse(self.url)
+        self.server = f"http://{url.netloc}"
+        self.title: str = ""
+        self.links: set[str] = set()
+        self.tags: set[str] = set()
+        self.description: str = ""
+        self.content: str = ""
+        self.safe_html: str = ""
+        self.text: str = ""
+        self.total_text: str = ""
+        self.words: list[str] = []
+        self.words_tags: list[str] = []
+        self.screenshot: str = ""
+        self.pdf_content: str = ""
+        self._fetch_information()
+
+    def to_absolute_url(self, url: str) -> str:
+        if url.startswith("/"):
+            return self.server + url
+        if url.startswith(("#", "?")):
+            return (
+                self.url + url
+            )  # TODO remove fragment + query from self.url (urlparse)
+        return url.replace("http://", "https://")
+
+    @property
+    def attached_pdf(self) -> Iterable[str]:
+        return [
+            self.to_absolute_url(link) for link in self.links if link.endswith(".pdf")
+        ]
+
+    @property
+    def all_tags(self) -> list[str]:
+        result = []
+        result_lower = set()
+        for tag in self.tags:
+            if tag.lower() not in result_lower:
+                result_lower.add(tag.lower())
+                result.append(tag)
+        for tag in self.words_tags:
+            if tag.lower() not in result_lower:
+                result_lower.add(tag.lower())
+                result.append(tag)
+        return result
+
+
+if __name__ == "__main__":
+    pages = [
+        # "https://leszexpertsfle.com/ressources-fle/raconter-une-histoire-damour-b1-et/",
+        # "https://littefle.wordpress.com/2019/10/16/le-prix-nobel-de-litterature-en-5-minutes/",
+        # "https://jancovici.com/changement-climatique/aspects-physiques/variation-du-climat-et-augmentation-de-leffet-de-serre-due-a-lhomme-cest-pareil/",
+        # "https://adamj.eu/tech/2022/03/30/how-to-make-django-error-for-undefined-template-variables/",
+        # "https://realpython.com/pypi-publish-python-package/",
+        "https://agi.to/enseigner/ressource/carte-didentite-de-super-heros/",
+    ]
+    for url in pages:
+        print("=" * 80)
+        page = Page(url)
+        print("URL", page.url)
+        print("title", page.title)
+        # print("links", page.links)
+        print("tags", page.tags)
+        print("words tags", page.words_tags)
+        print("all_tags", page.all_tags)
+        print("description:")
+        print(page.description)
+        print("pdfs", page.attached_pdf)
+        print("screenshot", page.screenshot)
+        # print("pdf content", page.pdf_content)
+        print("text", page.total_text[:2000].replace("\n", " "))
+
+    with open("index.html", "w", encoding="utf-8") as f:
+        f.write(
+            '<html><head><link rel="stylesheet" href="https://www.lecalamar.fr/css/main.css"></head><body style="max-width: 17cm;">'
+        )
+        f.write(page.safe_html)
+        f.write("</body></html>")

webpageinfo-2022.5.27/webpageinfo/safe_html.py

@@ -0,0 +1,103 @@
+import bs4
+
+from webpageinfo.config.ignored_html_tags import ignored_html_tags
+from webpageinfo.config.rejected_content_class import rejected_content_class
+from webpageinfo.config.rejected_content_id import rejected_content_id
+from webpageinfo.config.removed_html_tags import removed_html_tags
+from webpageinfo.config.simple_html_tags import simple_html_tags
+from webpageinfo.config.svg_tags import svg_tags
+
+# TODO mettre in id sur les titres si pas déjà existant
+# TODO conserver les id (pour les liens intradocuments)
+# TODO display none https://stackoverflow.com/questions/47862023/splinter-find-by-style
+
+_result = ""
+_server_url = ""
+
+BS4Element = (
+    bs4.element.Tag
+    | bs4.BeautifulSoup
+    | bs4.element.NavigableString
+    | bs4.element.Comment
+)
+
+
+def _manage_element_start(element: BS4Element) -> bool:
+    """Return False to ignore."""
+    global _result
+    if isinstance(element, bs4.element.Tag):
+        if "class" in element.attrs and any(
+            class_ in element["class"] for class_ in rejected_content_class
+        ):
+            return False
+        if "id" in element.attrs and any(
+            id_ == element["id"] for id_ in rejected_content_id
+        ):
+            return False
+        if element.name == "a" and "href" in element.attrs:
+            url = element["href"]
+            if url.startswith("/"):
+                url = _server_url + url
+            _result += f'<a href="{url}">'
+        elif element.name == "img":
+            url = element.attrs.get("src")
+            if url is not None:
+                if url.startswith("/"):
+                    url = _server_url + url
+                _result += f'<img src="{url}"'
+                if "alt" in element.attrs:
+                    _result += f' alt="{element["alt"]}"'
+                if "width" in element.attrs:
+                    _result += f' width="{element["width"]}"'
+                if "height" in element.attrs:
+                    _result += f' height="{element["height"]}"'
+                _result += ">"
+        elif element.name in simple_html_tags:
+            _result += f"<{element.name}>"
+        elif element.name in svg_tags:
+            pass  # TODO l'utiliser tel quel
+        elif element.name == ("iframe"):
+            return False  # TODO utiliser son contenu
+        elif element.name in removed_html_tags:
+            return False
+        elif element.name in ignored_html_tags:
+            return True
+        elif element.name == "[document]":
+            pass  # Beautiful soup root
+        else:
+            raise ValueError(f"Unknown tag: {element.name}, {element}, {element.attrs}")
+    elif isinstance(element, bs4.element.Comment):
+        return False
+    elif isinstance(element, bs4.element.NavigableString):
+        _result += str(element)
+        return False
+    return True
+
+
+def _manage_element_end(element: BS4Element):
+    global _result
+    if isinstance(element, bs4.element.Tag):
+        if element.name == "a" and "href" in element.attrs:
+            _result += "</a>"
+        elif element.name in simple_html_tags:
+            _result += f"</{element.name}>"
+
+
+def _visit(element):
+    if _manage_element_start(element):
+        for child in getattr(element, "children", []):
+            _visit(child)
+        _manage_element_end(element)
+
+
+def safe_html(html: str, server_url: str) -> str:
+    global _result, _server_url
+    _server_url = server_url
+    _result = ""
+    soup = bs4.BeautifulSoup(html, features="html.parser")
+    _visit(soup)
+    return _result.replace("<div></div>", "").replace("<p></p>", "")
+
+
+def html2text(safe_html: str) -> str:
+    return bs4.BeautifulSoup(safe_html, features="html.parser").text