lino 24.11.0__py3-none-any.whl → 24.11.1__py3-none-any.whl

lino/modlib/uploads/utils.py

@@ -0,0 +1,107 @@
+# -*- coding: UTF-8 -*-
+# Copyright 2010-2024 Rumma & Ko Ltd
+# License: GNU Affero General Public License v3 (see file COPYING for details)
+
+from os.path import splitext
+from django.conf import settings
+from django.utils.text import format_lazy
+from lino.api import dd, rt, _
+from lino.utils import needs_update
+
+if (with_thumbnails := dd.get_plugin_setting('uploads', 'with_thumbnails', False)):
+    from PIL import Image  # pip install Pillow
+    import pymupdf    # pip install PyMuPDF
+
+class UploadMediaFile:
+
+    def __init__(self, url):
+        self.url = url
+        assert url is not None
+        root, suffix = splitext(url)
+        self.suffix = suffix.lower()
+
+    def get_image_name(self):
+        if self.suffix not in previewer.PREVIEW_SUFFIXES:
+            return None
+        if previewer.base_dir is None:
+            if self.suffix == ".pdf":
+                return None
+            return self.url
+        url = self.url
+        if self.suffix == ".pdf":
+            url += ".png"
+        return previewer.base_dir + "/" + url
+
+    def get_mimetype_description(self):
+        if self.suffix == ".pdf":
+            return _("PDF file")
+        if self.get_image_name():
+            return _("picture")
+        return _("media file")
+
+    def get_image_url(self):
+        url = self.get_image_name()
+        if url is not None:
+            return settings.SITE.build_media_url(url)
+
+    def get_download_url(self):
+        return settings.SITE.build_media_url(self.url)
+
+
+class Previewer:
+    # The bare media previewer. It doesn't do any real work.
+    base_dir = None
+    max_width = None
+    PREVIEW_SUFFIXES = {'.png', '.jpg'}
+
+    def check_preview(self, obj, fix=False):
+        return []
+
+
+class FilePreviewer(Previewer):
+    # A media previewer that builds thumbnails in a separate directory tree
+    PREVIEW_SUFFIXES = {'.png', '.jpg', '.pdf'}
+
+    def __init__(self, base_dir=None, max_width=None):
+        self.base_dir = base_dir
+        self.max_width = max_width
+        super().__init__()
+
+    def check_preview(self, obj, fix=False):
+        mf = obj.get_media_file()
+        if mf is None:
+            return
+        if (dst := mf.get_image_name()) is None:
+            return
+        if dst == mf.url:
+            raise Exception("20241113 should never happen")
+            return
+        src = settings.SITE.media_root / mf.url
+        dst = settings.SITE.media_root / dst
+
+        if needs_update(src, dst):
+            yield (True, format_lazy(_("Must build thumbnail for {}"), mf.url))
+            if fix:
+                if src.suffix.lower() == ".pdf":
+                    doc = pymupdf.open(src)
+                    page = doc.load_page(0)
+                    pixmap = page.get_pixmap(dpi=120)
+                    pixmap.save(dst)
+                    return
+                with Image.open(src) as im:
+                    im.thumbnail((self.max_width, self.max_width))
+                    dst.parent.mkdir(parents=True, exist_ok=True)
+                    im.save(dst)
+
+if with_thumbnails:
+    previewer = FilePreviewer("thumbs", 720)
+else:
+    previewer = Previewer()
+
+# full = Previewer()
+# if with_thumbnails:
+#     small = FilePreviewer("thumbs", 720)
+# else:
+#     small = full
+
+# Lino currently offers only two previewers, "full" and "small"

lino/utils/__init__.py

@@ -98,6 +98,12 @@ def buildurl(root, *args, **kw):
     return url
 
 
+def needs_update(src, dest):
+    if dest.exists() and dest.stat().st_mtime >= src.stat().st_mtime:
+        return False
+    return True
+
+
 class AttrDict(dict):
     """
     Dictionary-like helper object.

lino/utils/html.py

@@ -11,7 +11,7 @@ from etgen.html import E, to_rst, fromstring, iselement, join_elems, forcetext,
 
 # from etgen.html import tostring as et_tostring
 from html2text import HTML2Text
-from django.utils.html import SafeString, mark_safe, escape
+from django.utils.html import SafeString, mark_safe, escape, format_html
 # from lino.utils import tostring
 
 SAFE_EMPTY = mark_safe("")

lino/utils/media.py

@@ -55,6 +55,5 @@ class MediaFile(object):
 class TmpMediaFile(MediaFile):
     def __init__(self, ar, fmt):
         ip = ar.request.META.get("REMOTE_ADDR", "unknown_ip")
-        super(TmpMediaFile, self).__init__(
-            False, "cache", "appy" + fmt, ip, str(ar.actor) + "." + fmt
-        )
+        super().__init__(
+            False, "cache", "appy" + fmt, ip, str(ar.actor) + "." + fmt)

lino/utils/soup.py

@@ -0,0 +1,311 @@
+# -*- coding: UTF-8 -*-
+# Copyright 2016-2024 Rumma & Ko Ltd
+# License: GNU Affero General Public License v3 (see file COPYING for details)
+
+# See https://dev.lino-framework.org/dev/bleach.html
+
+import re
+from bs4 import BeautifulSoup, NavigableString, Comment, Doctype
+from bs4.element import Tag
+import logging; logger = logging.getLogger(__file__)
+from lino.api import dd
+
+
+PARAGRAPH_TAGS = {
+    "p",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "h7",
+    "h8",
+    "h9",
+    "pre",
+    "li",
+    "div",
+}
+
+WHITESPACE_TAGS = PARAGRAPH_TAGS | {
+    "[document]",
+    "span",
+    "ul",
+    "html",
+    "head",
+    "body",
+    "base",
+}
+
+
+class Style:
+    # TODO: Extend rstgen.sphinxconf.sigal_image.Format to incoroporate this.
+    def __init__(self, s):
+        self._map = {}
+        if s:
+            for i in s.split(";"):
+                k, v = i.split(":", maxsplit=1)
+                self._map[k.strip()] = v.strip()
+        self.is_dirty = False
+
+    def __contains__(self, *args):
+        return self._map.__contains__(*args)
+
+    def __setitem__(self, k, v):
+        if k in self._map and self._map[k] == v:
+            return
+        self._map[k] = v
+        self.is_dirty = True
+
+    def __delitem__(self, k):
+        if k in self._map:
+            self.is_dirty = True
+        return self._map.__delitem__(k)
+
+    def adjust_size(self):
+        # if self['float'] == "none":
+        #     return
+        if "width" in self._map:
+            del self["width"]
+        self["height"] = dd.plugins.memo.short_preview_image_height
+
+    def as_string(self):
+        return ";".join(["{}:{}".format(*kv) for kv in self._map.items()])
+
+
+class TextCollector:
+    def __init__(self, max_length=None):
+        self.text = ""
+        self.sep = ""  # becomes "\n\n" after a PARAGRAPH_TAGS
+        self.remaining = max_length or settings.SITE.plugins.memo.short_preview_length
+        self.image = None
+
+    def add_chunk(self, ch):
+        # print("20230712 add_chunk", ch.name, ch)
+
+        if ch.name in WHITESPACE_TAGS:
+            for c in ch.children:
+                if not self.add_chunk(c):
+                    return False
+            if ch.name in PARAGRAPH_TAGS:
+                self.sep = "\n\n"
+            else:
+                self.sep = " "
+            return True
+
+        assert ch.name != "IMG"
+
+        if ch.name == "img":
+            if self.image is not None:
+                # Ignore all images except the first one.
+                self.text += self.sep
+                return True
+            style = Style(ch.get("style", None))
+            if not "float" in style:
+                style["float"] = "right"
+            style.adjust_size()
+            if style.is_dirty:
+                ch["style"] = style.as_string()
+            self.image = ch
+            # print("20231023 a", ch)
+
+        we_want_more = True
+        if ch.string is not None:
+            if len(ch.string) > self.remaining:
+                # print("20231023", len(ch.string), '>', self.remaining)
+                ch.string = ch.string[: self.remaining] + "..."
+                we_want_more = False
+                # print("20230927", ch.string, ch)
+                # self.text += str(ch.string) + "..."
+                # return False
+            self.remaining -= len(ch.string)
+
+        if isinstance(ch, NavigableString):
+            self.text += self.sep + ch.string
+        else:
+            self.text += self.sep + str(ch)
+
+        self.remaining -= len(self.sep)
+        self.sep = ""
+        return we_want_more
+
+
+def truncate_comment(html_str, max_length=300):
+    # Returns a single paragraph with a maximum number of visible chars.
+    # new implementation since 20230713
+    html_str = html_str.strip()  # remove leading or trailing newlines
+
+    if not html_str.startswith("<"):
+        # print("20231023 c", html_str)
+        if len(html_str) > max_length:
+            return html_str[:max_length] + "..."
+        return html_str
+
+    # if "choose one or the other" in html_str:
+    #     print(html_str)
+    #     raise Exception("20230928 {} {}".format(len(html_str), max_length))
+
+    soup = BeautifulSoup(html_str, features="html.parser")
+    tc = TextCollector(max_length)
+    tc.add_chunk(soup)
+    return tc.text
+
+
+
+def old_truncate_comment(html_str, max_p_len=None):
+    # returns a single paragraph with a maximum number of visible chars.
+    # No longer used. Replaced by new truncate_comment() below
+    if max_p_len is None:
+        max_p_len = settings.SITE.plugins.memo.short_preview_length
+    html_str = html_str.strip()  # remove leading or trailing newlines
+
+    if not html_str.startswith("<"):
+        if len(html_str) > max_p_len:
+            txt = html_str[:max_p_len] + "..."
+        else:
+            txt = html_str
+        return txt
+    soup = BeautifulSoup(html_str, "html.parser")
+    ps = soup.find_all(
+        ["p", "h1", "h2", "h3", "h4", "h5", "h6", "h7", "h8", "h9", "pre"]
+    )
+    if len(ps) > 0:
+        anchor_end = "</a>"
+        txt = ""
+        for p in ps:
+            text = ""
+            for c in p.contents:
+                if isinstance(c, Tag):
+                    if c.name == "a":
+                        text += str(c)
+                        max_p_len = max_p_len + len(text) - len(c.text)
+                    else:
+                        # text += str(c)
+                        text += c.text
+                else:
+                    text += str(c)
+
+            if len(txt) + len(text) > max_p_len:
+                txt += text
+                if anchor_end in txt:
+                    ae_index = txt.index(anchor_end) + len(anchor_end)
+                    if ae_index >= max_p_len:
+                        txt = txt[:ae_index]
+                        txt += "..."
+                        break
+                txt = txt[:max_p_len]
+                txt += "..."
+                break
+            else:
+                txt += text + "\n\n"
+        return txt
+    return html_str
+
+# remove these tags including their content.
+blacklist = frozenset(["script", "style", "head"])
+
+# unwrap these tags (remove the wrapper and leave the content)
+unwrap = frozenset(["html", "body"])
+
+useless_main_tags = frozenset(["p", "div", "span"])
+
+ALLOWED_TAGS = frozenset([
+    "a",
+    "b",
+    "i",
+    "em",
+    "ul",
+    "ol",
+    "li",
+    "strong",
+    "p",
+    "br",
+    "span",
+    "pre",
+    "def",
+    "div",
+    "img",
+    "table",
+    "th",
+    "tr",
+    "td",
+    "thead",
+    "tfoot",
+    "tbody",
+])
+
+
+# Map of allowed attributes by tag. Copied from bleach.sanitizer:
+ALLOWED_ATTRIBUTES = {
+    "a": {"href", "title"},
+    "abbr": {"title"},
+    "acronym": {"title"},
+}
+
+ALLOWED_ATTRIBUTES["span"] = {
+    "class",
+    "data-index",
+    "data-denotation-char",
+    "data-link",
+    "data-title",
+    "data-value",
+    "contenteditable",
+}
+ALLOWED_ATTRIBUTES["p"] = {"align"}
+
+def safe_css(attr, css):
+    if attr == "style":
+        return re.sub("(width|height):[^;]+;", "", css)
+    return css
+
+def sanitize(old):
+
+    # Inspired by https://chase-seibert.github.io/blog/2011/01/28/sanitize-html-with-beautiful-soup.html
+
+    old = old.strip()
+    if not old:
+        return old
+
+    try:
+        soup = BeautifulSoup(old, features="lxml")
+    except HTMLParseError as e:
+        logger.info("Could not sanitize %r : %s", old, e)
+        return f"Could not sanitize content ({e})"
+
+    for tag in soup.findAll():
+        # print(tag)
+        tag_name = tag.name.lower()
+        if tag_name in blacklist:
+            # blacklisted tags are removed in their entirety
+            tag.extract()
+        elif tag_name in unwrap:
+            tag.unwrap()
+        elif tag_name in ALLOWED_TAGS:
+            # tag is allowed. Make sure all the attributes are allowed.
+            allowed = ALLOWED_ATTRIBUTES.get(tag_name, None)
+            if allowed is None:
+                tag.attrs = dict()
+            else:
+                tag.attrs = {k: v for k, v in tag.attrs.items() if k in allowed}
+        else:
+            # print(tag.name)
+            # tag.decompose()
+            # tag.extract()
+            # not a whitelisted tag. I'd like to remove it from the tree
+            # and replace it with its children. But that's hard. It's much
+            # easier to just replace it with an empty span tag.
+            tag.name = "span"
+            tag.attrs = dict()
+
+    # remove all comments because they might contain scripts
+    comments = soup.findAll(text=lambda text:isinstance(text, (Comment, Doctype)))
+    for comment in comments:
+        comment.extract()
+
+    # remove the wrapper tag if it is useless
+    if len(soup.contents) == 1:
+        main_tag = soup.contents[0]
+        if main_tag.name in useless_main_tags and not main_tag.attrs:
+            main_tag.unwrap()
+
+    return str(soup).strip()

{lino-24.11.0.dist-info → lino-24.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lino
-Version: 24.11.0
+Version: 24.11.1
 Summary: A framework for writing desktop-like web applications using Django and ExtJS or React
 Project-URL: Homepage, https://www.lino-framework.org
 Project-URL: Repository, https://gitlab.com/lino-framework/lino
@@ -666,8 +666,6 @@ License:                     GNU AFFERO GENERAL PUBLIC LICENSE
         if any, to sign a "copyright disclaimer" for the program, if necessary.
         For more information on this, and how to apply and follow the GNU AGPL, see
         <https://www.gnu.org/licenses/>.
-License-File: AUTHORS.rst
-License-File: COPYING
 Keywords: Django,React,customized,framework
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Web Environment