wbnews 1.58.3__py2.py3-none-any.whl

wbnews/import_export/backends/news.py

@@ -0,0 +1,36 @@
+import json
+from datetime import datetime
+from io import BytesIO
+from typing import Generator
+
+import feedparser
+from django.db.models import QuerySet
+from slugify import slugify
+from wbcore.contrib.io.backends.abstract import AbstractDataBackend
+from wbcore.contrib.io.backends.utils import register
+
+from wbnews.models import NewsSource
+
+
+@register("News RSS Backend", save_data_in_import_source=True)
+class DataBackend(AbstractDataBackend):
+    def is_object_valid(self, obj: "NewsSource") -> bool:
+        return obj.type == NewsSource.Type.RSS and obj.is_active and obj.endpoint
+
+    def get_default_queryset(self) -> QuerySet["NewsSource"]:
+        return NewsSource.objects.filter(type=NewsSource.Type.RSS, is_active=True, endpoint__isnull=False)
+
+    def get_files(
+        self, execution_time: datetime, queryset=None, **kwargs
+    ) -> Generator[tuple[str, BytesIO], None, None] | None:
+        if queryset is not None:
+            for source in queryset:
+                data = feedparser.parse(source.endpoint)
+                if not data.get("bozo_exception"):
+                    data["news_source"] = source.id
+                    content_file = BytesIO()
+                    content_file.write(json.dumps(data).encode())
+                    file_name = (
+                        f"{slugify(source.title, separator='_')}_rss_file_{datetime.timestamp(execution_time)}.json"
+                    )
+                    yield file_name, content_file

wbnews/import_export/handlers/__init__.py

@@ -0,0 +1 @@
+from .news import NewsImportHandler

wbnews/import_export/handlers/news.py

@@ -0,0 +1,57 @@
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+import pytz
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.db import models
+from django.utils import timezone
+from slugify import slugify
+from wbcore.contrib.io.imports import ImportExportHandler
+
+if TYPE_CHECKING:
+    from wbnews.models import News
+
+
+class NewsImportHandler(ImportExportHandler):
+    MODEL_APP_LABEL = "wbnews.News"
+    model: "News"
+
+    def _deserialize(self, data: Dict[str, Any]):
+        from wbnews.models.sources import NewsSource
+
+        data["source"] = NewsSource.source_dict_to_model(data["source"])
+        if parsed_datetime := data.get("datetime", None):
+            data["datetime"] = pytz.utc.localize(datetime.strptime(parsed_datetime, "%Y-%m-%dT%H:%M:%S"))
+        else:
+            data["datetime"] = timezone.now()
+
+        data["default_guid"] = self.model.get_default_guid(data["title"], data.get("link", None))
+        if guid := data.get("guid", None):
+            data["guid"] = slugify(guid)
+        else:
+            data["guid"] = data["default_guid"]
+
+        # constrained fields to the max allowed size
+        if "title" in data:
+            data["title"] = data["title"][:500]
+
+        if "link" in data:
+            data["link"] = data["link"][:1024]
+
+    def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
+        default_guid = data.pop("default_guid")
+        instance = None
+        try:
+            instance = self.model.all_objects.get(models.Q(guid=data["guid"]) | models.Q(guid=default_guid))
+        except MultipleObjectsReturned:
+            instance = self.model.all_objects.get(models.Q(guid=data["guid"]))
+        except ObjectDoesNotExist:
+            pass
+
+        if instance:
+            self.import_source.log += f"\nFound existing news {instance.id} (guid: {instance.guid})"
+        return instance
+
+    def _create_instance(self, data: Dict[str, Any], **kwargs) -> models.Model:
+        self.import_source.log += "\nCreate News."
+        return self.model.objects.create(**data, import_source=self.import_source)

wbnews/import_export/parsers/emails/news.py

@@ -0,0 +1,39 @@
+import re
+from contextlib import suppress
+from datetime import datetime
+
+from django.conf.global_settings import LANGUAGES
+from wbcore.utils.importlib import import_from_dotted_path
+
+from .utils import EmlContentParser
+
+languages_dict = dict(LANGUAGES)
+
+
+def clean_string_with_paragraphs(string):
+    return re.sub(r"  +", " ", re.sub(r"(?<!\.)\\n", " ", string.strip()))
+
+
+def parse(import_source):
+    parser = EmlContentParser(
+        import_source.file.read(), encoding=import_source.source.import_parameters.get("email_encoding", "latin-1")
+    )
+    email_date = parser.date if parser.date else datetime.now()
+
+    # Source
+    html = parser.html
+
+    # If source define a custom html parser, we import it and convert the returned html
+    if html_parser_path := import_source.source.import_parameters.get("html_parser", None):
+        with suppress(ModuleNotFoundError):
+            html_parser = import_from_dotted_path(html_parser_path)
+            html = html_parser(html)
+
+    data = {
+        "datetime": email_date.strftime("%Y-%m-%dT%H:%M:%S"),
+        "title": parser.subject.replace(f"[{import_source.source.uuid}]", ""),
+        "description": html,
+        "source": parser.source,
+    }
+
+    return {"data": [data]}

wbnews/import_export/parsers/emails/utils.py

@@ -0,0 +1,65 @@
+import re
+from email import message, parser
+from email.utils import parseaddr, parsedate_to_datetime
+
+
+class EmlContentParser:
+    def __init__(self, email: bytes, encoding: str = "latin-1"):
+        self.message = parser.BytesParser().parsebytes(email)
+        self.encoding = encoding
+
+    @property
+    def date(self):
+        if date_str := self.message.get("date"):
+            return parsedate_to_datetime(date_str)
+
+    @property
+    def subject(self) -> str:
+        return self.message.get("subject", "")
+
+    @property
+    def html(self):
+        html = self.get_html(self.message)
+        return html.decode(self.encoding) if html else None
+
+    def get_html(self, parsed: message.Message) -> bytes | None:
+        if parsed.is_multipart():
+            for item in parsed.get_payload():  # type:message.Message
+                if html := self.get_html(item):
+                    return html
+        elif parsed.get_content_type() == "text/html":
+            return parsed.get_payload(decode=True)
+        return None
+
+    @property
+    def text(self):
+        text = self.get_text(self.message)
+        return text.decode(self.encoding) if text else None
+
+    @classmethod
+    def get_text(cls, parsed: message.Message) -> bytes | None:
+        if parsed.is_multipart():
+            for item in parsed.get_payload():
+                if text := cls.get_text(item):
+                    return text
+        elif parsed.get_content_type() == "text/plain":
+            return parsed.get_payload(decode=True)
+        return None
+
+    @property
+    def source(self) -> dict[str, any]:
+        match = re.search(r"From:(.*)<([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})>", self.text)
+        # we search first for forwarding info
+        if match:
+            name = match.group(1)
+            email = match.group(2)
+        else:
+            # otherwise we default to the From attribute
+            name, email = parseaddr(self.message["From"])
+
+        if not email:
+            raise ValueError("Couldn't find valid source data")
+        if not name:
+            name = email
+        source = {"title": name.split("@")[-1].strip().title(), "endpoint": email, "type": "EMAIL"}
+        return source

wbnews/import_export/parsers/rss/news.py

@@ -0,0 +1,58 @@
+import json
+from datetime import datetime
+from time import mktime
+
+from django.conf.global_settings import LANGUAGES
+from django.utils.html import strip_tags
+
+languages_dict = dict(LANGUAGES)
+
+
+def _get_source(d):
+    source = {"type": "RSS"}
+    if source_id := d.get("news_source"):
+        source["id"] = source_id
+    else:
+        if "title" in d["feed"]:
+            source["title"] = d["feed"]["title"]
+        if "author" in d["feed"]:
+            source["author"] = d["feed"]["author"]
+        if "image" in d["feed"]:
+            source["image"] = d["feed"]["image"]["href"]
+        if "href" in d["feed"]:
+            source["identifier"] = d["feed"]["href"]
+        if "link" in d["feed"]:
+            source["endpoint"] = d["feed"]["link"]
+    return source
+
+
+def parse(import_source):
+    content = json.load(import_source.file)
+    data = []
+    source = _get_source(content)
+
+    for entry in content["entries"]:
+        if summary := entry.get("summary", None):
+            description = entry.get("description", summary)
+            res = {
+                "description": description,
+                "summary": summary,
+                "source": source,
+                "title": strip_tags(entry.get("title", "")).strip(),
+                "link": entry.get("link", None),
+                "guid": entry.get("id", None),
+            }
+            if published_parsed := entry.get("published_parsed", None):
+                updated = datetime.fromtimestamp(mktime(tuple(published_parsed)))
+                res["datetime"] = updated.strftime("%Y-%m-%dT%H:%M:%S")
+            if enclosures := entry.get("enclosures", None):
+                res["enclosures"] = [e.get("href", "") for e in enclosures]
+            if (
+                (media_content := entry.get("media_content", []))
+                and isinstance(media_content, list)
+                and len(media_content) > 0
+                and (image_url := media_content[0].get("url", None))
+            ):
+                res["image_url"] = image_url
+            data.append(res)
+    return {"data": data}

wbnews/locale/de/LC_MESSAGES/django.po

@@ -0,0 +1,166 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+# Translators:
+# Kevin Decoster, 2025
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 11:37+0200\n"
+"PO-Revision-Date: 2025-05-30 09:40+0000\n"
+"Last-Translator: Kevin Decoster, 2025\n"
+"Language-Team: German (https://app.transifex.com/stainly/teams/171242/de/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: de\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: models/news.py:51 viewsets/display.py:54 viewsets/display.py:100
+#: viewsets/display.py:141
+msgid "Datetime"
+msgstr "Zeitpunkt"
+
+#: models/news.py:52 serializers.py:21 serializers.py:76
+#: viewsets/display.py:20 viewsets/display.py:55 viewsets/display.py:102
+#: viewsets/display.py:142
+msgid "Title"
+msgstr "Titel"
+
+#: models/news.py:54 serializers.py:24 serializers.py:77
+#: viewsets/display.py:23 viewsets/display.py:57 viewsets/display.py:104
+#: viewsets/display.py:143
+msgid "Description"
+msgstr "Beschreibung"
+
+#: models/news.py:55 serializers.py:78 viewsets/display.py:56
+#: viewsets/display.py:103
+msgid "Summary"
+msgstr "Zusammenfassung"
+
+#: models/news.py:56 viewsets/display.py:60 viewsets/display.py:144
+msgid "Language"
+msgstr "Sprache"
+
+#: models/news.py:57
+msgid "Link"
+msgstr "Link"
+
+#: models/news.py:61 viewsets/display.py:59 viewsets/display.py:106
+msgid "Source"
+msgstr "Quelle"
+
+#: models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: serializers.py:22
+msgid "Identifier"
+msgstr "Identifikator"
+
+#: serializers.py:25 viewsets/display.py:22
+msgid "Author"
+msgstr "Autor*in"
+
+#: serializers.py:26
+msgid "Updated"
+msgstr "Geändert"
+
+#: serializers.py:79
+msgid "Date"
+msgstr ""
+
+#: serializers.py:92 viewsets/menu.py:6 viewsets/titles.py:24
+msgid "News"
+msgstr "Neuigkeiten"
+
+#: viewsets/buttons.py:14
+msgid "Open News"
+msgstr "Neuigkeiten Öffnen"
+
+#: viewsets/buttons.py:21 viewsets/buttons.py:22 viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: viewsets/display.py:21
+msgid "RSS feed"
+msgstr "RSS Feed"
+
+#: viewsets/display.py:24
+msgid "Last Update"
+msgstr "Letztes Aktualisierung"
+
+#: viewsets/display.py:61
+msgid "Image"
+msgstr "Bild"
+
+#: viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: viewsets/menu.py:17 viewsets/titles.py:9
+msgid "Sources"
+msgstr "Quellen"
+
+#: viewsets/menu.py:23
+msgid "Create Source"
+msgstr "Quelle Erstellen"
+
+#: viewsets/titles.py:13
+#, python-brace-format
+msgid "Source: {source}"
+msgstr "Quelle: {source}"
+
+#: viewsets/titles.py:14
+msgid "News Source"
+msgstr ""
+
+#: viewsets/titles.py:19
+msgid "News Flow"
+msgstr "Nachrichtenfluss"
+
+#: viewsets/titles.py:30
+#, python-brace-format
+msgid "News from {source}"
+msgstr "Neuigkeiten von {source}"
+
+#: viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: viewsets/titles.py:37 viewsets/titles.py:44
+msgid "News Article"
+msgstr ""

wbnews/locale/de/LC_MESSAGES/django.po.translated

@@ -0,0 +1,173 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-29 13:33+0200\n"
+"PO-Revision-Date: 2022-10-17 14:51+0200\n"
+"Last-Translator: \n"
+"Language-Team: \n"
+"Language: de\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+"X-Generator: Poedit 3.1.1\n"
+
+#: wbnews/models/news.py:51 wbnews/viewsets/display.py:54
+#: wbnews/viewsets/display.py:100 wbnews/viewsets/display.py:141
+msgid "Datetime"
+msgstr "Zeitpunkt"
+
+#: wbnews/models/news.py:52 wbnews/serializers.py:21 wbnews/serializers.py:76
+#: wbnews/viewsets/display.py:20 wbnews/viewsets/display.py:55
+#: wbnews/viewsets/display.py:102 wbnews/viewsets/display.py:142
+msgid "Title"
+msgstr "Titel"
+
+#: wbnews/models/news.py:54 wbnews/serializers.py:24 wbnews/serializers.py:77
+#: wbnews/viewsets/display.py:23 wbnews/viewsets/display.py:57
+#: wbnews/viewsets/display.py:104 wbnews/viewsets/display.py:143
+msgid "Description"
+msgstr "Beschreibung"
+
+#: wbnews/models/news.py:55 wbnews/serializers.py:78
+#: wbnews/viewsets/display.py:56 wbnews/viewsets/display.py:103
+msgid "Summary"
+msgstr "Zusammenfassung"
+
+#: wbnews/models/news.py:56 wbnews/viewsets/display.py:60
+#: wbnews/viewsets/display.py:144
+msgid "Language"
+msgstr "Sprache"
+
+#: wbnews/models/news.py:57
+msgid "Link"
+msgstr "Link"
+
+#: wbnews/models/news.py:61 wbnews/viewsets/display.py:59
+#: wbnews/viewsets/display.py:106
+msgid "Source"
+msgstr "Quelle"
+
+#: wbnews/models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: wbnews/models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: wbnews/models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: wbnews/models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: wbnews/models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: wbnews/serializers.py:22
+msgid "Identifier"
+msgstr "Identifikator"
+
+#: wbnews/serializers.py:25 wbnews/viewsets/display.py:22
+msgid "Author"
+msgstr "Autor*in"
+
+#: wbnews/serializers.py:26
+msgid "Updated"
+msgstr "Geändert"
+
+#: wbnews/serializers.py:79
+#, fuzzy
+#| msgid "Datetime"
+msgid "Date"
+msgstr "Zeitpunkt"
+
+#: wbnews/serializers.py:92 wbnews/viewsets/menu.py:6
+#: wbnews/viewsets/titles.py:24 wbnews_config/menu.py:5
+msgid "News"
+msgstr "Neuigkeiten"
+
+#: wbnews/viewsets/buttons.py:14
+msgid "Open News"
+msgstr "Neuigkeiten Öffnen"
+
+#: wbnews/viewsets/buttons.py:21 wbnews/viewsets/buttons.py:22
+#: wbnews/viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: wbnews/viewsets/display.py:21
+msgid "RSS feed"
+msgstr "RSS Feed"
+
+#: wbnews/viewsets/display.py:24
+msgid "Last Update"
+msgstr "Letztes Aktualisierung"
+
+#: wbnews/viewsets/display.py:61
+msgid "Image"
+msgstr "Bild"
+
+#: wbnews/viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: wbnews/viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: wbnews/viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: wbnews/viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: wbnews/viewsets/menu.py:17 wbnews/viewsets/titles.py:9
+msgid "Sources"
+msgstr "Quellen"
+
+#: wbnews/viewsets/menu.py:23
+msgid "Create Source"
+msgstr "Quelle Erstellen"
+
+#: wbnews/viewsets/titles.py:13
+#, python-brace-format
+msgid "Source: {source}"
+msgstr "Quelle: {source}"
+
+#: wbnews/viewsets/titles.py:14
+#, fuzzy
+#| msgid "Source"
+msgid "News Source"
+msgstr "Quelle"
+
+#: wbnews/viewsets/titles.py:19
+msgid "News Flow"
+msgstr "Nachrichtenfluss"
+
+#: wbnews/viewsets/titles.py:30
+#, python-brace-format
+msgid "News from {source}"
+msgstr "Neuigkeiten von {source}"
+
+#: wbnews/viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: wbnews/viewsets/titles.py:37 wbnews/viewsets/titles.py:44
+#, fuzzy
+#| msgid "Source"
+msgid "News Article"
+msgstr "Quelle"