wbnews 1.46.12__py2.py3-none-any.whl → 1.58.0__py2.py3-none-any.whl

wbnews/admin.py

@@ -20,8 +20,11 @@ class NewsAdmin(admin.ModelAdmin):
 
     list_filter = ("source",)
 
+    def get_queryset(self, request):
+        return News.all_objects.select_related("source")
+
 
 @admin.register(NewsSource)
 class NewsSourceAdmin(admin.ModelAdmin):
-    search_fields = ("type", "title", "identifier", "description", "author", "url")
+    search_fields = ("type", "title", "identifier", "description", "author", "endpoint")
     list_filter = ("type",)

wbnews/factories.py

@@ -12,10 +12,10 @@ faker = Factory.create()
 class NewsSourceFactory(factory.django.DjangoModelFactory):
     title = factory.Sequence(lambda n: f"source_{n}")
     identifier = factory.Sequence(lambda n: f"http://myurl_{n}.com")
-    image = faker.url()
+    image = factory.Faker("url")
     description = factory.Faker("sentence", nb_words=32)
-    author = faker.name()
-    url = factory.Faker("url")
+    author = factory.Faker("name")
+    endpoint = factory.Faker("url")
 
     class Meta:
         model = NewsSource
@@ -23,12 +23,14 @@ class NewsSourceFactory(factory.django.DjangoModelFactory):
 
 class NewsFactory(factory.django.DjangoModelFactory):
     datetime = factory.LazyFunction(timezone.now)
-    title = factory.Sequence(lambda n: f"news_{n}")
+    title = factory.Faker("sentence", nb_words=32)
     description = factory.Faker("sentence", nb_words=32)
     summary = factory.Faker("sentence", nb_words=32)
     language = factory.Iterator(langs)
-    link = faker.url()
+    link = factory.Faker("url")
+    guid = factory.LazyAttribute(lambda o: News.get_default_guid(o.title, o.link))
     source = factory.SubFactory(NewsSourceFactory)
 
     class Meta:
         model = News
+        django_get_or_create = ("guid",)

wbnews/filters/__init__.py

@@ -1 +1 @@
-from .news import NewsFilterSet
+from .news import NewsFilterSet, NewsRelationshipFilterSet

wbnews/filters/news.py

@@ -1,9 +1,46 @@
 from wbcore import filters as wb_filters
 
-from wbnews.models import News
+from wbnews.models import News, NewsRelationship, NewsSource
 
 
 class NewsFilterSet(wb_filters.FilterSet):
+    datetime = wb_filters.DateTimeRangeFilter()
+
     class Meta:
         model = News
-        fields = {"title": ["icontains"], "datetime": ["lte", "gte"], "source": ["exact"], "language": ["exact"]}
+        fields = {"title": ["icontains"], "source": ["exact"], "language": ["exact"]}
+
+
+class NewsRelationshipFilterSet(wb_filters.FilterSet):
+    datetime = wb_filters.DateTimeRangeFilter(method="filter_datetime")
+    title = wb_filters.CharFilter(lookup_expr="icontains")
+    description = wb_filters.CharFilter(lookup_expr="icontains")
+    summary = wb_filters.CharFilter(lookup_expr="icontains")
+    source = wb_filters.ModelMultipleChoiceFilter(
+        label="Source",
+        queryset=NewsSource.objects.all(),
+        endpoint=NewsSource.get_representation_endpoint(),
+        value_key=NewsSource.get_representation_value_key(),
+        label_key=NewsSource.get_representation_label_key(),
+        method="filter_source",
+    )
+    content_type = wb_filters.CharFilter(method="fake_filter", hidden=True)
+    object_id = wb_filters.CharFilter(method="fake_filter", hidden=True)
+
+    def filter_datetime(self, queryset, name, value):
+        if value:
+            return queryset.filter(news__datetime__gte=value.lower, news__datetime__lte=value.upper)
+        return queryset
+
+    def filter_source(self, queryset, name, value):
+        if value:
+            return queryset.filter(news__source__in=value)
+        return queryset
+
+    class Meta:
+        model = NewsRelationship
+        fields = {
+            "analysis": ["icontains"],
+            "sentiment": ["exact"],
+            "important": ["exact"],
+        }

wbnews/import_export/backends/news.py

@@ -15,17 +15,17 @@ from wbnews.models import NewsSource
 @register("News RSS Backend", save_data_in_import_source=True)
 class DataBackend(AbstractDataBackend):
     def is_object_valid(self, obj: "NewsSource") -> bool:
-        return obj.type == NewsSource.Type.RSS and obj.is_active and obj.url
+        return obj.type == NewsSource.Type.RSS and obj.is_active and obj.endpoint
 
     def get_default_queryset(self) -> QuerySet["NewsSource"]:
-        return NewsSource.objects.filter(type=NewsSource.Type.RSS, is_active=True, url__isnull=False)
+        return NewsSource.objects.filter(type=NewsSource.Type.RSS, is_active=True, endpoint__isnull=False)
 
     def get_files(
         self, execution_time: datetime, queryset=None, **kwargs
     ) -> Generator[tuple[str, BytesIO], None, None] | None:
         if queryset is not None:
             for source in queryset:
-                data = feedparser.parse(source.url)
+                data = feedparser.parse(source.endpoint)
                 if not data.get("bozo_exception"):
                     data["news_source"] = source.id
                     content_file = BytesIO()

wbnews/import_export/handlers/news.py

@@ -1,24 +1,56 @@
 from datetime import datetime
-from typing import Any, Dict, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
 
 import pytz
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.db import models
 from django.utils import timezone
+from slugify import slugify
 from wbcore.contrib.io.imports import ImportExportHandler
 
+if TYPE_CHECKING:
+    from wbnews.models import News
+
 
 class NewsImportHandler(ImportExportHandler):
     MODEL_APP_LABEL = "wbnews.News"
+    model: "News"
 
     def _deserialize(self, data: Dict[str, Any]):
-        data["source"] = self.model.source_dict_to_model(data["source"])
+        from wbnews.models.sources import NewsSource
+
+        data["source"] = NewsSource.source_dict_to_model(data["source"])
         if parsed_datetime := data.get("datetime", None):
             data["datetime"] = pytz.utc.localize(datetime.strptime(parsed_datetime, "%Y-%m-%dT%H:%M:%S"))
         else:
             data["datetime"] = timezone.now()
 
+        data["default_guid"] = self.model.get_default_guid(data["title"], data.get("link", None))
+        if guid := data.get("guid", None):
+            data["guid"] = slugify(guid)
+        else:
+            data["guid"] = data["default_guid"]
+
+        # constrained fields to the max allowed size
+        if "title" in data:
+            data["title"] = data["title"][:500]
+
+        if "link" in data:
+            data["link"] = data["link"][:1024]
+
     def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
-        return self.model.objects.filter(source=data["source"], datetime=data["datetime"], title=data["title"]).first()
+        default_guid = data.pop("default_guid")
+        instance = None
+        try:
+            instance = self.model.all_objects.get(models.Q(guid=data["guid"]) | models.Q(guid=default_guid))
+        except MultipleObjectsReturned:
+            instance = self.model.all_objects.get(models.Q(guid=data["guid"]))
+        except ObjectDoesNotExist:
+            pass
+
+        if instance:
+            self.import_source.log += f"\nFound existing news {instance.id} (guid: {instance.guid})"
+        return instance
 
     def _create_instance(self, data: Dict[str, Any], **kwargs) -> models.Model:
         self.import_source.log += "\nCreate News."

wbnews/import_export/parsers/emails/news.py

@@ -3,7 +3,6 @@ from contextlib import suppress
 from datetime import datetime
 
 from django.conf.global_settings import LANGUAGES
-from langdetect import detect, lang_detect_exception
 from wbcore.utils.importlib import import_from_dotted_path
 
 from .utils import EmlContentParser
@@ -37,12 +36,4 @@ def parse(import_source):
         "source": parser.source,
     }
 
-    # Language
-    try:
-        language = detect(data["description"])
-        if language in languages_dict:
-            data["language"] = language
-    except lang_detect_exception.LangDetectException:
-        pass
-
     return {"data": [data]}

wbnews/import_export/parsers/emails/utils.py

@@ -1,3 +1,4 @@
+import re
 from email import message, parser
 from email.utils import parseaddr, parsedate_to_datetime
 
@@ -21,10 +22,10 @@ class EmlContentParser:
         html = self.get_html(self.message)
         return html.decode(self.encoding) if html else None
 
-    def get_html(cls, parsed: message.Message) -> bytes | None:
+    def get_html(self, parsed: message.Message) -> bytes | None:
         if parsed.is_multipart():
             for item in parsed.get_payload():  # type:message.Message
-                if html := cls.get_html(item):
+                if html := self.get_html(item):
                     return html
         elif parsed.get_content_type() == "text/html":
             return parsed.get_payload(decode=True)
@@ -47,15 +48,18 @@ class EmlContentParser:
 
     @property
     def source(self) -> dict[str, any]:
-        name, email = parseaddr(self.message["From"])
-        if not name:
-            name = "Generic"
+        match = re.search(r"From:(.*)<([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})>", self.text)
+        # we search first for forwarding info
+        if match:
+            name = match.group(1)
+            email = match.group(2)
+        else:
+            # otherwise we default to the From attribute
+            name, email = parseaddr(self.message["From"])
+
         if not email:
-            email = "generic"
-        source = {
-            "title": f"{name} Research Email",
-            "identifier": "research-email-" + email.lower(),
-            "author": name,
-            "url": email,
-        }
+            raise ValueError("Couldn't find valid source data")
+        if not name:
+            name = email
+        source = {"title": name.split("@")[-1].strip().title(), "endpoint": email, "type": "EMAIL"}
         return source

wbnews/import_export/parsers/rss/news.py

@@ -4,13 +4,12 @@ from time import mktime
 
 from django.conf.global_settings import LANGUAGES
 from django.utils.html import strip_tags
-from langdetect import detect, lang_detect_exception
 
 languages_dict = dict(LANGUAGES)
 
 
 def _get_source(d):
-    source = {}
+    source = {"type": "RSS"}
     if source_id := d.get("news_source"):
         source["id"] = source_id
     else:
@@ -23,7 +22,7 @@ def _get_source(d):
         if "href" in d["feed"]:
             source["identifier"] = d["feed"]["href"]
         if "link" in d["feed"]:
-            source["url"] = d["feed"]["link"]
+            source["endpoint"] = d["feed"]["link"]
     return source
 
 
@@ -41,13 +40,8 @@ def parse(import_source):
                 "source": source,
                 "title": strip_tags(entry.get("title", "")).strip(),
                 "link": entry.get("link", None),
+                "guid": entry.get("id", None),
             }
-            try:
-                language = detect(entry["summary"])
-                if language in languages_dict:
-                    res["language"] = language
-            except lang_detect_exception.LangDetectException:
-                pass
             if published_parsed := entry.get("published_parsed", None):
                 updated = datetime.fromtimestamp(mktime(tuple(published_parsed)))
                 res["datetime"] = updated.strftime("%Y-%m-%dT%H:%M:%S")

wbnews/locale/de/LC_MESSAGES/django.po

@@ -3,110 +3,164 @@
 # This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
+# Translators:
+# Kevin Decoster, 2025
+#
+#, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: \n"
+"Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-10-17 14:50+0200\n"
-"PO-Revision-Date: 2022-10-17 14:51+0200\n"
-"Last-Translator: \n"
-"Language-Team: \n"
-"Language: de\n"
+"POT-Creation-Date: 2025-05-30 11:37+0200\n"
+"PO-Revision-Date: 2025-05-30 09:40+0000\n"
+"Last-Translator: Kevin Decoster, 2025\n"
+"Language-Team: German (https://app.transifex.com/stainly/teams/171242/de/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Language: de\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
-"X-Generator: Poedit 3.1.1\n"
 
-#: wbnews/models.py:45 wbnews/viewsets/display.py:59
-#: wbnews/viewsets/display.py:86
+#: models/news.py:51 viewsets/display.py:54 viewsets/display.py:100
+#: viewsets/display.py:141
 msgid "Datetime"
 msgstr "Zeitpunkt"
 
-#: wbnews/models.py:46 wbnews/serializers.py:17 wbnews/viewsets/display.py:12
-#: wbnews/viewsets/display.py:60 wbnews/viewsets/display.py:87
+#: models/news.py:52 serializers.py:21 serializers.py:76
+#: viewsets/display.py:20 viewsets/display.py:55 viewsets/display.py:102
+#: viewsets/display.py:142
 msgid "Title"
 msgstr "Titel"
 
-#: wbnews/models.py:47 wbnews/serializers.py:20 wbnews/viewsets/display.py:15
-#: wbnews/viewsets/display.py:62 wbnews/viewsets/display.py:88
+#: models/news.py:54 serializers.py:24 serializers.py:77
+#: viewsets/display.py:23 viewsets/display.py:57 viewsets/display.py:104
+#: viewsets/display.py:143
 msgid "Description"
 msgstr "Beschreibung"
 
-#: wbnews/models.py:48 wbnews/viewsets/display.py:61
+#: models/news.py:55 serializers.py:78 viewsets/display.py:56
+#: viewsets/display.py:103
 msgid "Summary"
 msgstr "Zusammenfassung"
 
-#: wbnews/models.py:49 wbnews/viewsets/display.py:65
-#: wbnews/viewsets/display.py:89
+#: models/news.py:56 viewsets/display.py:60 viewsets/display.py:144
 msgid "Language"
 msgstr "Sprache"
 
-#: wbnews/models.py:50
+#: models/news.py:57
 msgid "Link"
 msgstr "Link"
 
-#: wbnews/models.py:54 wbnews/viewsets/display.py:64
+#: models/news.py:61 viewsets/display.py:59 viewsets/display.py:106
 msgid "Source"
 msgstr "Quelle"
 
-#: wbnews/serializers.py:18
+#: models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: serializers.py:22
 msgid "Identifier"
 msgstr "Identifikator"
 
-#: wbnews/serializers.py:21 wbnews/viewsets/display.py:14
+#: serializers.py:25 viewsets/display.py:22
 msgid "Author"
 msgstr "Autor*in"
 
-#: wbnews/serializers.py:22
+#: serializers.py:26
 msgid "Updated"
 msgstr "Geändert"
 
-#: wbnews/viewsets/buttons.py:8
+#: serializers.py:79
+msgid "Date"
+msgstr ""
+
+#: serializers.py:92 viewsets/menu.py:6 viewsets/titles.py:24
+msgid "News"
+msgstr "Neuigkeiten"
+
+#: viewsets/buttons.py:14
 msgid "Open News"
 msgstr "Neuigkeiten Öffnen"
 
-#: wbnews/viewsets/display.py:13
+#: viewsets/buttons.py:21 viewsets/buttons.py:22 viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: viewsets/display.py:21
 msgid "RSS feed"
 msgstr "RSS Feed"
 
-#: wbnews/viewsets/display.py:16
+#: viewsets/display.py:24
 msgid "Last Update"
 msgstr "Letztes Aktualisierung"
 
-#: wbnews/viewsets/display.py:33 wbnews/viewsets/menu.py:6
-#: wbnews/viewsets/titles.py:23 wbnews_config/menu.py:6
-msgid "News"
-msgstr "Neuigkeiten"
-
-#: wbnews/viewsets/display.py:66
+#: viewsets/display.py:61
 msgid "Image"
 msgstr "Bild"
 
-#: wbnews/viewsets/menu.py:11 wbnews/viewsets/titles.py:8
+#: viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: viewsets/menu.py:17 viewsets/titles.py:9
 msgid "Sources"
 msgstr "Quellen"
 
-#: wbnews/viewsets/menu.py:15
+#: viewsets/menu.py:23
 msgid "Create Source"
 msgstr "Quelle Erstellen"
 
-#: wbnews/viewsets/titles.py:12
+#: viewsets/titles.py:13
 #, python-brace-format
 msgid "Source: {source}"
 msgstr "Quelle: {source}"
 
-#: wbnews/viewsets/titles.py:13
-#, fuzzy
-#| msgid "Source"
+#: viewsets/titles.py:14
 msgid "News Source"
-msgstr "Quelle"
+msgstr ""
 
-#: wbnews/viewsets/titles.py:18
+#: viewsets/titles.py:19
 msgid "News Flow"
 msgstr "Nachrichtenfluss"
 
-#: wbnews/viewsets/titles.py:29
+#: viewsets/titles.py:30
 #, python-brace-format
 msgid "News from {source}"
 msgstr "Neuigkeiten von {source}"
+
+#: viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: viewsets/titles.py:37 viewsets/titles.py:44
+msgid "News Article"
+msgstr ""

wbnews/locale/de/LC_MESSAGES/django.po.translated

@@ -0,0 +1,173 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-29 13:33+0200\n"
+"PO-Revision-Date: 2022-10-17 14:51+0200\n"
+"Last-Translator: \n"
+"Language-Team: \n"
+"Language: de\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+"X-Generator: Poedit 3.1.1\n"
+
+#: wbnews/models/news.py:51 wbnews/viewsets/display.py:54
+#: wbnews/viewsets/display.py:100 wbnews/viewsets/display.py:141
+msgid "Datetime"
+msgstr "Zeitpunkt"
+
+#: wbnews/models/news.py:52 wbnews/serializers.py:21 wbnews/serializers.py:76
+#: wbnews/viewsets/display.py:20 wbnews/viewsets/display.py:55
+#: wbnews/viewsets/display.py:102 wbnews/viewsets/display.py:142
+msgid "Title"
+msgstr "Titel"
+
+#: wbnews/models/news.py:54 wbnews/serializers.py:24 wbnews/serializers.py:77
+#: wbnews/viewsets/display.py:23 wbnews/viewsets/display.py:57
+#: wbnews/viewsets/display.py:104 wbnews/viewsets/display.py:143
+msgid "Description"
+msgstr "Beschreibung"
+
+#: wbnews/models/news.py:55 wbnews/serializers.py:78
+#: wbnews/viewsets/display.py:56 wbnews/viewsets/display.py:103
+msgid "Summary"
+msgstr "Zusammenfassung"
+
+#: wbnews/models/news.py:56 wbnews/viewsets/display.py:60
+#: wbnews/viewsets/display.py:144
+msgid "Language"
+msgstr "Sprache"
+
+#: wbnews/models/news.py:57
+msgid "Link"
+msgstr "Link"
+
+#: wbnews/models/news.py:61 wbnews/viewsets/display.py:59
+#: wbnews/viewsets/display.py:106
+msgid "Source"
+msgstr "Quelle"
+
+#: wbnews/models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: wbnews/models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: wbnews/models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: wbnews/models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: wbnews/models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: wbnews/serializers.py:22
+msgid "Identifier"
+msgstr "Identifikator"
+
+#: wbnews/serializers.py:25 wbnews/viewsets/display.py:22
+msgid "Author"
+msgstr "Autor*in"
+
+#: wbnews/serializers.py:26
+msgid "Updated"
+msgstr "Geändert"
+
+#: wbnews/serializers.py:79
+#, fuzzy
+#| msgid "Datetime"
+msgid "Date"
+msgstr "Zeitpunkt"
+
+#: wbnews/serializers.py:92 wbnews/viewsets/menu.py:6
+#: wbnews/viewsets/titles.py:24 wbnews_config/menu.py:5
+msgid "News"
+msgstr "Neuigkeiten"
+
+#: wbnews/viewsets/buttons.py:14
+msgid "Open News"
+msgstr "Neuigkeiten Öffnen"
+
+#: wbnews/viewsets/buttons.py:21 wbnews/viewsets/buttons.py:22
+#: wbnews/viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: wbnews/viewsets/display.py:21
+msgid "RSS feed"
+msgstr "RSS Feed"
+
+#: wbnews/viewsets/display.py:24
+msgid "Last Update"
+msgstr "Letztes Aktualisierung"
+
+#: wbnews/viewsets/display.py:61
+msgid "Image"
+msgstr "Bild"
+
+#: wbnews/viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: wbnews/viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: wbnews/viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: wbnews/viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: wbnews/viewsets/menu.py:17 wbnews/viewsets/titles.py:9
+msgid "Sources"
+msgstr "Quellen"
+
+#: wbnews/viewsets/menu.py:23
+msgid "Create Source"
+msgstr "Quelle Erstellen"
+
+#: wbnews/viewsets/titles.py:13
+#, python-brace-format
+msgid "Source: {source}"
+msgstr "Quelle: {source}"
+
+#: wbnews/viewsets/titles.py:14
+#, fuzzy
+#| msgid "Source"
+msgid "News Source"
+msgstr "Quelle"
+
+#: wbnews/viewsets/titles.py:19
+msgid "News Flow"
+msgstr "Nachrichtenfluss"
+
+#: wbnews/viewsets/titles.py:30
+#, python-brace-format
+msgid "News from {source}"
+msgstr "Neuigkeiten von {source}"
+
+#: wbnews/viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: wbnews/viewsets/titles.py:37 wbnews/viewsets/titles.py:44
+#, fuzzy
+#| msgid "Source"
+msgid "News Article"
+msgstr "Quelle"