wbnews 1.45.0__py2.py3-none-any.whl → 1.58.1__py2.py3-none-any.whl

wbnews/models/relationships.py

@@ -1,21 +1,45 @@
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
+from django.utils.translation import gettext as _
 
 
 class NewsRelationship(models.Model):
+    class SentimentChoices(models.IntegerChoices):
+        POSITIVE = 4, _("Positive")
+        SLIGHTLY_POSITIVE = 3, _("Slightly Positive")
+        SLIGHTLY_NEGATIVE = 2, _("Slightly Negative")
+        NEGATIVE = 1, _("Negative")
+
+        def get_color(self):
+            colors = {
+                "POSITIVE": "#96DD99",
+                "SLIGHTLY_POSITIVE": "#FFEE8C",
+                "SLIGHTLY_NEGATIVE": "#FF964F",
+                "NEGATIVE": "#FF6961",
+            }
+            return colors[self.name]
+
     news = models.ForeignKey(to="wbnews.News", related_name="relationships", on_delete=models.CASCADE)
     content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
     object_id = models.PositiveIntegerField()
     content_object = GenericForeignKey("content_type", "object_id")
+    content_object_repr = models.CharField(max_length=512, default="")
 
     important = models.BooleanField(null=True, blank=True)
-    sentiment = models.PositiveIntegerField(null=True, blank=True)
+    sentiment = models.PositiveIntegerField(null=True, blank=True, choices=SentimentChoices.choices)
     analysis = models.TextField(null=True, blank=True)
 
+    def save(self, *args, **kwargs):
+        self.content_object_repr = str(self.content_object)
+        super().save(*args, **kwargs)
+
     def __str__(self) -> str:
         return f"{self.news.title} -> {self.content_object}"
 
     class Meta:
         verbose_name = "News Relationship"
         indexes = [models.Index(fields=["content_type", "object_id"])]
+        constraints = [
+            models.UniqueConstraint(name="unique_news_relationship", fields=["content_type", "object_id", "news"])
+        ]

wbnews/models/sources.py

@@ -1,7 +1,11 @@
+import re
+
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from wbcore.models import WBModel
 
+from wbnews.models.utils import endpoint_to_author
+
 
 class NewsSource(WBModel):
     class Type(models.TextChoices):
@@ -16,16 +20,17 @@ class NewsSource(WBModel):
     description = models.TextField(default="", blank=True)
     author = models.CharField(max_length=255, default="")
     clean_content = models.BooleanField(default=False)
-    url = models.URLField(
-        blank=True,
-        null=True,
-        unique=True,
-    )
+    endpoint = models.CharField(max_length=1024, unique=True)
     is_active = models.BooleanField(default=True)
 
     def __str__(self):
         return f"{self.title}"
 
+    def save(self, *args, **kwargs):
+        if not self.author and self.endpoint:
+            self.author = endpoint_to_author(self.endpoint)
+        super().save(*args, **kwargs)
+
     @classmethod
     def get_representation_endpoint(cls) -> str:
         return "wbnews:sourcerepresentation-list"
@@ -41,3 +46,28 @@ class NewsSource(WBModel):
     @classmethod
     def get_endpoint_basename(cls) -> str:
         return "wbnews:source"
+
+    @classmethod
+    def source_dict_to_model(cls, data: dict):
+        sources = NewsSource.objects.all()
+        endpoint = data.pop("endpoint", None)
+        if "id" in data:
+            return sources.get(id=data["id"])
+        if type := data.get("type"):
+            sources = sources.filter(type=type)
+        if identifier := data.get("identifier"):
+            sources = sources.filter(identifier=identifier)
+        elif endpoint:
+            for source in sources:
+                match = re.search(source.endpoint, endpoint)
+                if source.endpoint == endpoint or match:
+                    return source
+        if sources.count() == 1:
+            return sources.first()
+        else:
+            if endpoint:
+                # Pattern to capture and replace the local part of an email
+                pattern = r"^[^@]+"
+                # Replace the local part of an email with a wildcard regex
+                endpoint = re.sub(pattern, ".*", re.escape(endpoint))
+            return NewsSource.objects.create(**data, endpoint=endpoint)

wbnews/models/utils.py

@@ -0,0 +1,15 @@
+from contextlib import suppress
+from urllib.parse import urlparse
+
+
+def endpoint_to_author(endpoint: str) -> str:
+    author = endpoint
+    if "@" in endpoint:  # simplist way to check if the endpoint is an email address
+        author = author.replace("\\", "").split("@")[-1].split(".")
+        if len(author) > 1:
+            author = ".".join(author[:-1])
+    else:  # otherwise we consider it's a valid url and we extract only the domain part
+        with suppress(ValueError, IndexError):
+            author = urlparse(author).netloc.split(".")[-2]
+
+    return author.replace("_", " ").title()

wbnews/serializers.py

@@ -1,8 +1,12 @@
 from django.utils.translation import gettext_lazy as _
 from rest_framework.reverse import reverse
 from wbcore import serializers as wb_serializers
+from wbcore.content_type.serializers import (
+    ContentTypeRepresentationSerializer,
+    DynamicObjectIDRepresentationSerializer,
+)
 
-from .models import News, NewsSource
+from .models import News, NewsRelationship, NewsSource
 
 
 class SourceRepresentationSerializer(wb_serializers.RepresentationSerializer):
@@ -67,15 +71,52 @@ class NewsModelSerializer(wb_serializers.ModelSerializer):
 
 
 class NewsRelationshipModelSerializer(wb_serializers.ModelSerializer):
+    source = wb_serializers.PrimaryKeyCharField(read_only=True)
     _source = SourceRepresentationSerializer(source="source")
-    sentiment = wb_serializers.IntegerField(required=False)
-    analysis = wb_serializers.TextField(required=False)
-    important = wb_serializers.BooleanField(required=False)
+    title = wb_serializers.TextField(read_only=True, label=_("Title"))
+    description = wb_serializers.TextField(read_only=True, label=_("Description"))
+    summary = wb_serializers.TextField(read_only=True, label=_("Summary"))
+    datetime = wb_serializers.DateTimeField(read_only=True, label=_("Date"))
+    _content_type = ContentTypeRepresentationSerializer(source="content_type")
+    object_id = wb_serializers.CharField(label="Linked Object", required=False)
+    _object_id = DynamicObjectIDRepresentationSerializer(
+        content_type_field_name="content_type",
+        source="object_id",
+        optional_get_parameters={"content_type": "content_type"},
+        depends_on=[{"field": "content_type", "options": {}}],
+        filter_params={
+            "is_security": True
+        },  # TODO needs to find a way to not create a dependency to the wbfdm module here
+    )
+    news = wb_serializers.PrimaryKeyRelatedField(
+        queryset=News.objects.all(), read_only=lambda view: not view.new_mode, label=_("News")
+    )
+    _news = NewsRepresentationSerializer(source="news")
+
+    def validate(self, data):
+        if view := self.context["view"]:
+            if view.object_id:
+                data["object_id"] = view.object_id
+            if view.content_type:
+                data["content_type"] = view.content_type
+        return super().validate(data)
 
     class Meta:
-        model = News
+        model = NewsRelationship
+        read_only_fields = (
+            "content_object_repr",
+            "datetime",
+            "title",
+            "description",
+            "summary",
+            "content_type",
+            "_content_type",
+        )
         fields = (
             "id",
+            "news",
+            "_news",
+            "content_object_repr",
             "datetime",
             "sentiment",
             "analysis",
@@ -85,4 +126,9 @@ class NewsRelationshipModelSerializer(wb_serializers.ModelSerializer):
             "summary",
             "source",
             "_source",
+            "content_type",
+            "_content_type",
+            "object_id",
+            "_object_id",
+            "_additional_resources",
         )

wbnews/tasks.py

@@ -0,0 +1,16 @@
+from datetime import date, timedelta
+
+from celery import shared_task
+
+from wbnews.models import News
+
+
+@shared_task()
+def handle_daily_news_duplicates(
+    task_date: date | None = None,
+    day_interval: int = 7,
+):
+    if not task_date:
+        task_date = date.today()
+
+    News.handle_duplicates(task_date - timedelta(days=day_interval), task_date + timedelta(days=day_interval))

wbnews/tests/parsers/test_emails.py

@@ -0,0 +1,25 @@
+from unittest.mock import PropertyMock, patch
+
+import pytest
+
+from wbnews.import_export.parsers.emails.utils import EmlContentParser
+
+
+class TestEmlContentParser:
+    @pytest.fixture
+    def content_parser(self):
+        parser = EmlContentParser(b"")
+        parser.message = {"From": "main@acme.com"}
+        return parser
+
+    @patch.object(EmlContentParser, "text", new_callable=PropertyMock)
+    def test_source_from_in_text(self, mock_text, content_parser):
+        mock_text.return_value = (
+            "some random email content with a From field From: source name <email@test.com> and the rest of the email"
+        )
+        assert content_parser.source == {"title": "Source Name", "endpoint": "email@test.com", "type": "EMAIL"}
+
+    @patch.object(EmlContentParser, "text", new_callable=PropertyMock)
+    def test_source_from_in_text_alt(self, mock_text, content_parser):
+        mock_text.return_value = "some random email content without a From field"
+        assert content_parser.source == {"title": "Acme.Com", "endpoint": "main@acme.com", "type": "EMAIL"}

wbnews/tests/test_models.py

@@ -1,4 +1,13 @@
+from datetime import timedelta, timezone
+from unittest.mock import patch
+
 import pytest
+from django.utils import timezone as django_timezone
+from faker import Faker
+
+from wbnews.models import News, NewsSource
+
+fake = Faker()
 
 
 @pytest.mark.django_db
@@ -7,9 +16,65 @@ class TestSource:
     def test_str(self, news_source):
         assert str(news_source) == f"{news_source.title}"
 
+    def test_source_dict_to_model(self, news_source_factory):
+        ns1 = news_source_factory.create()
+        ns2 = news_source_factory.create()
+
+        assert NewsSource.source_dict_to_model({"id": ns1.id, "identifier": ns2.identifier}) == ns1  # priority to "id"
+        assert (
+            NewsSource.source_dict_to_model({"endpoint": ns1.endpoint, "identifier": ns2.identifier}) == ns2
+        )  # priority to "identifier"
+        assert NewsSource.source_dict_to_model({"endpoint": ns2.endpoint}) == ns2  # exact match on endpoint
+
+        ns1.endpoint = ".*@test.com"
+        ns1.save()
+        assert NewsSource.source_dict_to_model({"endpoint": "abc@test.com"}) == ns1  # regex match on endpoint
+
+        new_source = NewsSource.source_dict_to_model({"endpoint": "abc@main_source.com", "title": "New Source"})
+        assert new_source not in [ns1, ns2]
+        assert new_source.endpoint == r".*@main_source\.com"
+        assert new_source.title == "New Source"
+        assert new_source.author == "Main Source"
+
 
 @pytest.mark.django_db
 class TestNews:
     @pytest.mark.parametrize("news__title", ["new1"])
     def test_str(self, news):
         assert str(news) == f"{news.title} ({news.source.title})"
+
+    def test_mark_as_deplicates_not_in_default_queryset(self, news):
+        assert set(News.objects.all()) == {news}
+
+    def test_get_default_guid(self):
+        assert News.get_default_guid("This is a title", None) == "this-is-a-title"
+        assert (
+            News.get_default_guid("This is a title", "http://mylink.com") == "http://mylink.com"
+        )  # link takes precendence
+        assert News.get_default_guid("a" * 24, None, max_length=20) == "a" * 20
+
+    def test_future_news(self, news_factory):
+        # ensure a future datetime always default to now
+        now = django_timezone.now()
+        future_news = news_factory.create(datetime=now + timedelta(days=1))
+        assert (future_news.datetime - now).seconds < 1  # we do that to account for clock difference
+
+    @patch("wbnews.models.news.detect_near_duplicates")
+    def test_handle_duplicates(self, mock_fct, news_factory):
+        val_date = fake.date_time(tzinfo=timezone.utc)
+        n0 = news_factory.create(
+            datetime=val_date - timedelta(days=1)
+        )  # we exclude this news from the duplicate search
+        n1 = news_factory.create(datetime=val_date)
+        n2 = news_factory.create(datetime=val_date)
+        n3 = news_factory.create(datetime=val_date)
+
+        mock_fct.return_value = [
+            n0.id,
+            n3.id,
+        ]  # n0 is considered as duplicate but does not fall within the specified daterange so it will not be marked
+        News.handle_duplicates(val_date, val_date)
+
+        n3.refresh_from_db()
+        assert n3.mark_as_duplicate is True
+        assert set(News.objects.all()) == {n0, n1, n2}

wbnews/tests/test_utils.py

@@ -0,0 +1,7 @@
+from wbnews.models.utils import endpoint_to_author
+
+
+def test_endpoint_to_author():
+    assert endpoint_to_author("test@test_test\\.com") == "Test Test"
+    assert endpoint_to_author("http://somesubdomain.domain.com") == "Domain"
+    assert endpoint_to_author("test") == "Test"

wbnews/urls.py

@@ -22,9 +22,4 @@ urlpatterns = [
         views.NewsModelViewSet.as_view({"get": "list"}),
         name="news_content_object",
     ),
-    path(
-        "contentnewsrelationship/<int:content_type>/<int:content_id>/",
-        views.NewsRelationshipModelViewSet.as_view({"get": "list"}),
-        name="news_relationship_content_object",
-    ),
 ]

wbnews/utils.py

@@ -0,0 +1,57 @@
+import logging
+
+import numpy as np
+import pandas as pd
+from django.utils.html import strip_tags
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+logger = logging.getLogger("news")
+
+
+def _get_similarity_matrix_df(data: dict[int, str]) -> pd.DataFrame:
+    # Convert texts to TF-IDF vectors
+    ids, texts = zip(*data.items(), strict=False)
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform(texts)
+    # Compute pairwise cosine similarity...
+    similarity_matrix = cosine_similarity(tfidf_matrix)
+    # convert the matrix into a proper dataframe
+    return pd.DataFrame(similarity_matrix, index=ids, columns=ids)
+
+
+def detect_near_duplicates(data: dict[int, str], threshold: float = 0.9) -> list[int]:
+    """
+    Detects near-duplicate articles based on TF-IDF & Cosine Similarity.
+
+    Parameters:
+    - data (dict[int, str]): dictionary of new id with their respective content
+    - threshold (float): Similarity threshold (default = 0.9).
+
+    Returns:
+    - List of duplicated ids
+    """
+    if len(data.keys()) < 2:
+        return []
+    logger.info(f"Processing {len(data.keys())} news")
+    # Cleanup step
+    clean_data = {}
+    for _id, text in data.items():
+        clean_data[_id] = strip_tags(text)
+
+    # get similarity matrix
+    df = _get_similarity_matrix_df(data)
+
+    # Replace the lower matrix triangle with NaN
+    df = df.where(np.triu(np.ones(df.shape)).astype(bool))
+    # melt the symmetrical matrix into a key value store
+    df = df.stack().reset_index(name="value")
+    # remove duplicate pair with same id (expected to be 1.0)
+    df = df[df["level_0"] != df["level_1"]]
+    # get duplicates candidates
+    df = df[df["value"] > threshold]
+    # return only one side of the duplicate pair
+    duplicate_ids = df["level_1"].unique().tolist()
+    logger.info(f"{len(duplicate_ids)} duplicated news found")
+
+    return duplicate_ids

wbnews/viewsets/__init__.py

@@ -1,6 +1,6 @@
 from .buttons import NewsButtonConfig
 from .display import NewsDisplayConfig, NewsSourceDisplayConfig, SourceDisplayConfig
-from .endpoints import NewsEndpointConfig, NewsSourceEndpointConfig
+from .endpoints import NewsEndpointConfig, NewsSourceEndpointConfig, NewsRelationshipEndpointConfig
 from .menu import NEWS_MENUITEM, NEWSSOURCE_MENUITEM
 from .titles import NewsSourceModelTitleConfig, NewsTitleConfig, SourceModelTitleConfig
 from .views import (

wbnews/viewsets/buttons.py

@@ -2,6 +2,7 @@ from django.dispatch import receiver
 from django.utils.translation import gettext as _
 from rest_framework.reverse import reverse
 from wbcore.contrib.icons import WBIcon
+from wbcore.enums import RequestType
 from wbcore.metadata.configs import buttons as bt
 from wbcore.metadata.configs.buttons.view_config import ButtonViewConfig
 from wbcore.signals.instance_buttons import add_extra_button
@@ -9,15 +10,33 @@ from wbcore.signals.instance_buttons import add_extra_button
 
 class NewsButtonConfig(ButtonViewConfig):
     def get_custom_list_instance_buttons(self):
-        return {bt.HyperlinkButton(key="open_link", label=_("Open News"), icon=WBIcon.LINK.icon)}
+        buttons = set()
+        buttons.add(bt.HyperlinkButton(key="open_link", label=_("Open News"), icon=WBIcon.LINK.icon))
+        if self.request.user.is_superuser and (pk := self.view.kwargs.get("pk")):
+            buttons.add(
+                bt.ActionButton(
+                    method=RequestType.PATCH,
+                    identifiers=("wbnews:newsrelationship",),
+                    endpoint=reverse("wbnews:news-refreshrelationship", args=[pk]),
+                    action_label=_("Reset relationships"),
+                    label=_("Reset relationships"),
+                    icon=WBIcon.REGENERATE.icon,
+                    title=_("Reset relationships"),
+                )
+            )
+        return buttons
 
     def get_custom_instance_buttons(self):
         return self.get_custom_list_instance_buttons()
 
 
+class NewsRelationshipButtonConfig(ButtonViewConfig):
+    pass
+
+
 @receiver(add_extra_button)
 def add_new_extra_button(sender, instance, request, view, pk=None, **kwargs):
     if instance and pk and view:
         content_type = view.get_content_type()
-        endpoint = reverse("wbnews:news_relationship_content_object", args=[content_type.id, pk], request=request)
+        endpoint = f'{reverse("wbnews:newsrelationship-list", args=[], request=request)}?content_type={content_type.id}&object_id={pk}'
         return bt.WidgetButton(endpoint=endpoint, label="News", icon=WBIcon.NEWSPAPER.icon)

wbnews/viewsets/display.py

@@ -10,6 +10,8 @@ from wbcore.metadata.configs.display.instance_display.shortcuts import (
 from wbcore.metadata.configs.display.instance_display.utils import repeat_field
 from wbcore.metadata.configs.display.view_config import DisplayViewConfig
 
+from wbnews.models import NewsRelationship
+
 
 class SourceDisplayConfig(DisplayViewConfig):
     def get_list_display(self) -> Optional[dp.ListDisplay]:
@@ -63,23 +65,38 @@ class NewsDisplayConfig(DisplayViewConfig):
 
 class NewsRelationshipDisplayConfig(DisplayViewConfig):
     def get_instance_display(self) -> Display:
+        if self.new_mode:
+            return create_simple_display(
+                [
+                    ["news", "news"],
+                    ["important", "sentiment"],
+                    ["analysis", "analysis"],
+                ]
+            )
+
         return create_simple_display(
             [
-                ["datetime", "source"],
-                ["language", "link"],
-                [repeat_field(2, "summary")],
-                [repeat_field(2, "description")],
+                ["news", "news"],
+                ["content_type", "object_id"],
+                [
+                    "important",
+                    "sentiment",
+                ],
+                ["analysis", "analysis"],
+                ["summary", "summary"],
+                ["description", "description"],
             ]
         )
 
     def get_list_display(self) -> Optional[dp.ListDisplay]:
-        POSITIVE = "#96DD99"
-        SLIGHTLY_POSITIVE = "#FFEE8C"
-        SLIGHTLY_NEGATIVE = "#FF964F"
-        NEGATIVE = "#FF6961"
+        fields = (
+            [dp.Field(key="content_object_repr", label=_("Linked Object"))]
+            if self.view.object_id is None and self.view.content_type is None
+            else []
+        )
 
-        return dp.ListDisplay(
-            fields=[
+        fields.extend(
+            [
                 dp.Field(key="datetime", label=_("Datetime")),
                 dp.Field(key="analysis", label=_("Analysis")),
                 dp.Field(key="title", label=_("Title")),
@@ -87,26 +104,22 @@ class NewsRelationshipDisplayConfig(DisplayViewConfig):
                 dp.Field(key="description", label=_("Description")),
                 dp.Field(key="important", label=_("Important")),
                 dp.Field(key="source", label=_("Source")),
-            ],
+            ]
+        )
+        return dp.ListDisplay(
+            fields=fields,
             formatting=[
                 dp.Formatting(
                     column="sentiment",
                     formatting_rules=[
-                        dp.FormattingRule(condition=("==", 4), style={"backgroundColor": POSITIVE}),
-                        dp.FormattingRule(condition=("==", 3), style={"backgroundColor": SLIGHTLY_POSITIVE}),
-                        dp.FormattingRule(condition=("==", 2), style={"backgroundColor": SLIGHTLY_NEGATIVE}),
-                        dp.FormattingRule(condition=("==", 1), style={"backgroundColor": NEGATIVE}),
+                        dp.FormattingRule(condition=("==", s.value), style={"backgroundColor": s.get_color()})
+                        for s in NewsRelationship.SentimentChoices
                     ],
                 )
             ],
             legends=[
                 dp.Legend(
-                    items=[
-                        dp.LegendItem(icon=POSITIVE, label=_("Positive")),
-                        dp.LegendItem(icon=SLIGHTLY_POSITIVE, label=_("Slightly Positive")),
-                        dp.LegendItem(icon=SLIGHTLY_NEGATIVE, label=_("Slightly Negative")),
-                        dp.LegendItem(icon=NEGATIVE, label=_("Negative")),
-                    ]
+                    items=[dp.LegendItem(icon=s.get_color(), label=s.label) for s in NewsRelationship.SentimentChoices]
                 )
             ],
         )

wbnews/viewsets/endpoints.py

@@ -1,18 +1,34 @@
 from rest_framework.reverse import reverse
 from wbcore.metadata.configs.endpoints import EndpointViewConfig
+from wbcore.utils.urls import get_urlencode_endpoint
 
 
 class NewsEndpointConfig(EndpointViewConfig):
     def get_endpoint(self, **kwargs):
         return None
 
-    def get_list_endpoint(self, **kwargs):
-        return reverse("wbnews:news-list", request=self.request)
-
     def get_instance_endpoint(self, **kwargs):
-        return self.get_list_endpoint()
+        return reverse("wbnews:news-list", request=self.request)
 
 
 class NewsSourceEndpointConfig(NewsEndpointConfig):
-    def get_list_endpoint(self, **kwargs):
-        return reverse("wbnews:source-news-list", args=[self.view.kwargs["source_id"]], request=self.request)
+    pass
+
+
+class NewsRelationshipEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return reverse("wbnews:newsrelationship-list", args=[], request=self.request)
+
+    def get_create_endpoint(self, **kwargs):
+        params = {}
+        if ct := self.view.content_type:
+            params["content_type"] = ct.id
+        if object_id := self.view.object_id:
+            params["object_id"] = object_id
+        return get_urlencode_endpoint(self.get_endpoint(**kwargs), params)
+
+    # def get_instance_endpoint(self, **kwargs):
+    #     return reverse("wbnews:news-list", args=[], request=self.request)
+    #
+    # def get_update_endpoint(self, **kwargs):
+    #     return self.get_endpoint(**kwargs)

wbnews/viewsets/menu.py

@@ -7,6 +7,12 @@ NEWS_MENUITEM = MenuItem(
     endpoint="wbnews:news-list",
     permission=ItemPermission(permissions=["wbnews.view_news"]),
 )
+NEWSRELATIONSHIP_MENUITEM = MenuItem(
+    label=_("News Relationships"),
+    endpoint="wbnews:newsrelationship-list",
+    permission=ItemPermission(permissions=["wbnews.view_news"]),
+)
+
 NEWSSOURCE_MENUITEM = MenuItem(
     label=_("Sources"),
     endpoint="wbnews:source-list",

wbnews/viewsets/titles.py

@@ -37,4 +37,8 @@ class NewsRelationshipTitleConfig(TitleViewConfig):
         return _("News Article")
 
     def get_instance_title(self):
-        return self.get_list_title()
+        try:
+            instance = self.view.get_object()
+            return str(instance)
+        except AssertionError:
+            return _("News Article")