wbnews 2.2.1__py2.py3-none-any.whl

wbnews/migrations/0009_newsrelationship_analysis_newsrelationship_sentiment.py

@@ -0,0 +1,94 @@
+# Generated by Django 5.0.9 on 2024-10-24 11:00
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+def migrate_new_source(apps, schema_editor):
+    from wbcore.contrib.io.models import (
+        DataBackend,
+        ImportSource,
+        ParserHandler,
+        Source,
+    )
+
+    parser_handler = ParserHandler.objects.get(parser="wbnews.import_export.parsers.rss.news")
+    io_news_sources = Source.objects.filter(parser_handler=parser_handler)
+    crontab = io_news_sources.last().crontab
+    data_backend = DataBackend.objects.get(backend_class_path="wbnews.import_export.backends.news")
+    default_news_source, created = Source.objects.get_or_create(
+        data_backend=data_backend,
+        defaults={
+            "crontab": crontab,
+            "title": "RSS Feeds -> News",
+        },
+    )
+    default_news_source.parser_handler.set([parser_handler])
+    ImportSource.objects.filter(source__in=io_news_sources).update(source=default_news_source)
+    io_news_sources.exclude(id=default_news_source.id).delete()
+    NewsSource = apps.get_model("wbnews", "NewsSource")
+    NewsSource.objects.filter(title__icontains="Email").update(type="EMAIL")
+    NewsSource.objects.filter(type="RSS").filter(url__isnull=True, identifier__isnull=False).update(
+        url=models.F("identifier")
+    )
+
+    unvalid_rss_feeds = [
+        "https://www.dailytelegraph.com.au/help-rss",
+        "https://techcrunch.com/rssfeeds/",
+        "https://www.bbc.co.uk/news/",
+        "https://www.nytimes.com/section/technology",
+        "http://online.wsj.com/page/2_0006.html",
+        "http://online.wsj.com",
+        "https://www.bbc.co.uk/news/technology",
+        "https://www.bbc.co.uk/news/world",
+        "https://www.bbc.co.uk/news/business",
+        "https://www.techworld.com/news/rss",
+    ]
+    NewsSource.objects.filter(url__in=unvalid_rss_feeds).update(is_active=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("wbnews", "0008_alter_news_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="newsrelationship",
+            name="analysis",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="newsrelationship",
+            name="sentiment",
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="clean_content",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="is_active",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="type",
+            field=models.CharField(choices=[("RSS", "RSS"), ("EMAIL", "EMAIL")], default="RSS", max_length=6),
+        ),
+        migrations.AlterField(
+            model_name="newssource",
+            name="description",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AlterField(
+            model_name="newssource",
+            name="tags",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=16), blank=True, default=list, size=None
+            ),
+        ),
+        migrations.RunPython(migrate_new_source),
+    ]

wbnews/models/__init__.py

@@ -0,0 +1,3 @@
+from .news import News
+from .relationships import NewsRelationship
+from .sources import NewsSource

wbnews/models/news.py

@@ -0,0 +1,116 @@
+from typing import Any
+
+from celery import chord, shared_task
+from celery.canvas import Signature
+from django.conf.global_settings import LANGUAGES
+from django.contrib.postgres.fields import ArrayField
+from django.db import models
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from django.utils.translation import gettext_lazy as _
+from wbcore.contrib.ai.llm.decorators import llm
+from wbcore.contrib.io.mixins import ImportMixin
+from wbcore.models import WBModel
+from wbnews.import_export.handlers.news import NewsImportHandler
+from wbnews.models.llm.cleaned_news import clean_news_config, summarized_news_config
+from wbnews.models.relationships import NewsRelationship
+from wbnews.models.sources import NewsSource
+from wbnews.signals import create_news_relationships
+
+
+@shared_task
+def create_relationship(chain_results: list[list[dict[str, Any]]], news_id: int):
+    objs = []
+    for relationships in chain_results:
+        for relationship in relationships:
+            objs.append(NewsRelationship(news_id=news_id, **relationship))
+    NewsRelationship.objects.bulk_create(
+        objs,
+        ignore_conflicts=True,
+        unique_fields=["content_type", "object_id", "news"],
+    )
+
+
+@llm([clean_news_config, summarized_news_config])
+class News(ImportMixin, WBModel):
+    errors = {
+        "relationship_signal": "using the fetch_new_relationships signal must return a list of tuples, sender: {0} did not."
+    }
+    import_export_handler_class = NewsImportHandler
+
+    datetime = models.DateTimeField(verbose_name=_("Datetime"))
+    title = models.CharField(max_length=500, verbose_name=_("Title"))
+    description = models.TextField(blank=True, verbose_name=_("Description"))
+    summary = models.TextField(blank=True, verbose_name=_("Summary"))
+    language = models.CharField(max_length=16, choices=LANGUAGES, blank=True, null=True, verbose_name=_("Language"))
+    link = models.CharField(max_length=500, blank=True, null=True, verbose_name=_("Link"))
+    tags = ArrayField(models.CharField(max_length=16), default=list)
+    enclosures = ArrayField(models.URLField(), default=list)
+    source = models.ForeignKey(
+        "wbnews.NewsSource", on_delete=models.CASCADE, related_name="news", verbose_name=_("Source")
+    )
+    image_url = models.URLField(blank=True, null=True)
+
+    class Meta:
+        unique_together = ["title", "source", "datetime"]
+
+    def __str__(self) -> str:
+        return f"{self.title} ({self.source.title})"
+
+    def update_and_create_news_relationships(self, synchronous: bool = False):
+        """
+        This methods fires the signal to fetch the possible relationship to be linked to the news
+        """
+        tasks = []
+        for sender, task_signature in create_news_relationships.send(sender=News, instance=self):
+            assert isinstance(task_signature, Signature), self.errors["relationship_signal"].format(sender)
+            tasks.append(task_signature)
+        if tasks:
+            res = chord(tasks, create_relationship.s(self.id))
+            if synchronous:
+                res.apply()
+            else:
+                res.apply_async()
+
+    # TODO: Consider moving this into a get_or_create queryset method on NewsSource?
+    @classmethod
+    def source_dict_to_model(cls, data: dict) -> NewsSource:
+        sources = NewsSource.objects
+        if "id" in data:
+            return sources.get(id=data["id"])
+        if identifier := data.get("identifier"):
+            sources = sources.filter(identifier=identifier)
+        elif url := data.get("url"):
+            sources = sources.filter(url=url)
+        elif title := data.get("title"):
+            sources = sources.filter(title=title)
+        if sources.count() == 1:
+            return sources.first()
+        else:
+            return NewsSource.objects.create(**data)
+
+    @classmethod
+    def get_representation_endpoint(cls) -> str:
+        return "wbnews:news-list"
+
+    @classmethod
+    def get_representation_value_key(cls) -> str:
+        return "id"
+
+    @classmethod
+    def get_representation_label_key(cls) -> str:
+        return "{{title}} ({{datetime}})"
+
+    @classmethod
+    def get_endpoint_basename(cls) -> str:
+        return "wbnews:news"
+
+
+@receiver(post_save, sender="wbnews.News")
+def post_save_create_news_relationships(sender: type, instance: "News", raw: bool, created: bool, **kwargs):
+    """
+    Post save to lazy create relationship between an instrument and a news upon creation
+    """
+
+    if not raw and created:
+        instance.update_and_create_news_relationships()

wbnews/models/relationships.py

@@ -0,0 +1,20 @@
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+
+
+class NewsRelationship(models.Model):
+    news = models.ForeignKey(to="wbnews.News", related_name="relationships", on_delete=models.CASCADE)
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    object_id = models.PositiveIntegerField()
+    content_object = GenericForeignKey("content_type", "object_id")
+
+    sentiment = models.PositiveIntegerField(null=True, blank=True)
+    analysis = models.TextField(null=True, blank=True)
+
+    def __str__(self) -> str:
+        return f"{self.news.title} -> {self.content_object}"
+
+    class Meta:
+        verbose_name = "News Relationship"
+        indexes = [models.Index(fields=["content_type", "object_id"])]

wbnews/models/sources.py

@@ -0,0 +1,43 @@
+from django.contrib.postgres.fields import ArrayField
+from django.db import models
+from wbcore.models import WBModel
+
+
+class NewsSource(WBModel):
+    class Type(models.TextChoices):
+        RSS = "RSS", "RSS"
+        EMAIL = "EMAIL", "EMAIL"
+
+    type = models.CharField(default=Type.RSS, choices=Type.choices, max_length=6)
+    title = models.CharField(max_length=255)
+    identifier = models.CharField(max_length=255, unique=True, blank=True, null=True)
+    tags = ArrayField(models.CharField(max_length=16), default=list, blank=True)
+    image = models.URLField(blank=True, null=True)
+    description = models.TextField(default="", blank=True)
+    author = models.CharField(max_length=255, default="")
+    clean_content = models.BooleanField(default=False)
+    url = models.URLField(
+        blank=True,
+        null=True,
+        unique=True,
+    )
+    is_active = models.BooleanField(default=True)
+
+    def __str__(self):
+        return f"{self.title}"
+
+    @classmethod
+    def get_representation_endpoint(cls) -> str:
+        return "wbnews:sourcerepresentation-list"
+
+    @classmethod
+    def get_representation_value_key(cls) -> str:
+        return "id"
+
+    @classmethod
+    def get_representation_label_key(cls) -> str:
+        return "{{title}}"
+
+    @classmethod
+    def get_endpoint_basename(cls) -> str:
+        return "wbnews:source"

wbnews/serializers.py

@@ -0,0 +1,83 @@
+from django.utils.translation import gettext_lazy as _
+from rest_framework.reverse import reverse
+from wbcore import serializers as wb_serializers
+
+from .models import News, NewsSource
+
+
+class SourceRepresentationSerializer(wb_serializers.RepresentationSerializer):
+    _detail = wb_serializers.HyperlinkField(reverse_name="wbnews:source-detail")
+
+    class Meta:
+        model = NewsSource
+        fields = ("id", "title", "_detail")
+
+
+class SourceModelSerializer(wb_serializers.ModelSerializer):
+    title = wb_serializers.CharField(read_only=True, label=_("Title"))
+    identifier = wb_serializers.CharField(read_only=True, label=_("Identifier"))
+    image = wb_serializers.CharField(read_only=True)
+    description = wb_serializers.CharField(read_only=True, label=_("Description"))
+    author = wb_serializers.CharField(read_only=True, label=_("Author"))
+    updated = wb_serializers.DateTimeField(read_only=True, label=_("Updated"))
+
+    @wb_serializers.register_resource()
+    def news(self, instance, request, user):
+        return {"news": reverse("wbnews:source-news-list", args=[instance.id], request=request)}
+
+    class Meta:
+        model = NewsSource
+        fields = ("id", "title", "identifier", "image", "description", "author", "updated", "_additional_resources")
+
+
+class NewsRepresentationSerializer(wb_serializers.RepresentationSerializer):
+    _detail = wb_serializers.HyperlinkField(reverse_name="wbnews:news-detail")
+
+    class Meta:
+        model = News
+        fields = ("id", "datetime", "title", "_detail")
+
+
+class NewsModelSerializer(wb_serializers.ModelSerializer):
+    _source = SourceRepresentationSerializer(source="source")
+    image_url = wb_serializers.ImageURLField()
+
+    @wb_serializers.register_resource()
+    def open_link(self, instance, request, user):
+        if instance.link:
+            return {"open_link": instance.link}
+        return {}
+
+    # link = wb_serializers.URL()
+    class Meta:
+        model = News
+        fields = (
+            "id",
+            "datetime",
+            "title",
+            "description",
+            "summary",
+            "link",
+            "language",
+            "image_url",
+            "source",
+            "_source",
+            "_additional_resources",
+        )
+
+
+class NewsRelationshipModelSerializer(wb_serializers.ModelSerializer):
+    sentiment = wb_serializers.IntegerField(required=False)
+    analysis = wb_serializers.TextField(required=False)
+
+    class Meta:
+        model = News
+        fields = (
+            "id",
+            "datetime",
+            "sentiment",
+            "analysis",
+            "title",
+            "description",
+            "summary",
+        )

wbnews/signals.py

@@ -0,0 +1,4 @@
+from django.db.models.signals import ModelSignal
+
+# this signal is gather the news relationship to efficiently handle the insertion.
+create_news_relationships = ModelSignal(use_caching=False)

wbnews/tests/conftest.py

@@ -0,0 +1,6 @@
+from pytest_factoryboy import register
+from wbcore.tests.conftest import *
+from wbnews.factories import NewsFactory, NewsSourceFactory
+
+register(NewsSourceFactory)
+register(NewsFactory)

wbnews/tests/test_models.py

@@ -0,0 +1,15 @@
+import pytest
+
+
+@pytest.mark.django_db
+class TestSource:
+    @pytest.mark.parametrize("news_source__title", ["source1"])
+    def test_str(self, news_source):
+        assert str(news_source) == f"{news_source.title}"
+
+
+@pytest.mark.django_db
+class TestNews:
+    @pytest.mark.parametrize("news__title", ["new1"])
+    def test_str(self, news):
+        assert str(news) == f"{news.title} ({news.source.title})"

wbnews/tests/tests.py

@@ -0,0 +1,12 @@
+import pytest
+from wbcore.test import GenerateTest, default_config
+
+config = {}
+for key, value in default_config.items():
+    config[key] = list(filter(lambda x: x.__module__.startswith("wbnews"), value))
+
+
+@pytest.mark.django_db
+@GenerateTest(config)
+class TestProject:
+    pass

wbnews/urls.py

@@ -0,0 +1,29 @@
+from django.urls import include, path
+from wbcore.routers import WBCoreRouter
+from wbnews.viewsets import views
+
+router = WBCoreRouter()
+router.register(r"newsrepresentation", views.NewsRepresentationViewSet, basename="newsrepresentation")
+router.register(r"newssourcerepresentation", views.SourceRepresentationViewSet, basename="sourcerepresentation")
+router.register(r"news", views.NewsModelViewSet, basename="news")
+router.register(r"newssource", views.SourceModelViewSet, basename="source")
+router.register(r"newsrelationship", views.NewsRelationshipModelViewSet, basename="newsrelationship")
+
+
+source_router = WBCoreRouter()
+source_router.register(r"news", views.NewsSourceModelViewSet, basename="source-news")
+
+urlpatterns = [
+    path("", include(router.urls)),
+    path("source/<int:source_id>/", include(source_router.urls)),
+    path(
+        "contentnews/<int:content_type>/<int:content_id>/",
+        views.NewsModelViewSet.as_view({"get": "list"}),
+        name="news_content_object",
+    ),
+    path(
+        "contentnewsrelationship/<int:content_type>/<int:content_id>/",
+        views.NewsRelationshipModelViewSet.as_view({"get": "list"}),
+        name="news_relationship_content_object",
+    ),
+]

wbnews/viewsets/__init__.py

@@ -0,0 +1,12 @@
+from .buttons import NewsButtonConfig
+from .display import NewsDisplayConfig, NewsSourceDisplayConfig, SourceDisplayConfig
+from .endpoints import NewsEndpointConfig, NewsSourceEndpointConfig
+from .menu import NEWS_MENUITEM, NEWSSOURCE_MENUITEM
+from .titles import NewsSourceModelTitleConfig, NewsTitleConfig, SourceModelTitleConfig
+from .views import (
+    NewsModelViewSet,
+    NewsRepresentationViewSet,
+    SourceModelViewSet,
+    SourceRepresentationViewSet,
+    NewsRelationshipModelViewSet,
+)

wbnews/viewsets/buttons.py

@@ -0,0 +1,23 @@
+from django.dispatch import receiver
+from django.utils.translation import gettext as _
+from rest_framework.reverse import reverse
+from wbcore.contrib.icons import WBIcon
+from wbcore.metadata.configs import buttons as bt
+from wbcore.metadata.configs.buttons.view_config import ButtonViewConfig
+from wbcore.signals.instance_buttons import add_extra_button
+
+
+class NewsButtonConfig(ButtonViewConfig):
+    def get_custom_list_instance_buttons(self):
+        return {bt.HyperlinkButton(key="open_link", label=_("Open News"), icon=WBIcon.LINK.icon)}
+
+    def get_custom_instance_buttons(self):
+        return self.get_custom_list_instance_buttons()
+
+
+@receiver(add_extra_button)
+def add_new_extra_button(sender, instance, request, view, pk=None, **kwargs):
+    if instance and pk and view:
+        content_type = view.get_content_type()
+        endpoint = reverse("wbnews:news_relationship_content_object", args=[content_type.id, pk], request=request)
+        return bt.WidgetButton(endpoint=endpoint, label="News", icon=WBIcon.NEWSPAPER.icon)

wbnews/viewsets/display.py

@@ -0,0 +1,133 @@
+from typing import Optional
+
+from django.utils.translation import gettext as _
+from wbcore.metadata.configs import display as dp
+from wbcore.metadata.configs.display.instance_display.shortcuts import (
+    Display,
+    create_simple_display,
+    create_simple_section,
+)
+from wbcore.metadata.configs.display.instance_display.utils import repeat_field
+from wbcore.metadata.configs.display.view_config import DisplayViewConfig
+
+
+class SourceDisplayConfig(DisplayViewConfig):
+    def get_list_display(self) -> Optional[dp.ListDisplay]:
+        return dp.ListDisplay(
+            fields=[
+                dp.Field(key="title", label=_("Title")),
+                dp.Field(key="identifier", label=_("RSS feed")),
+                dp.Field(key="author", label=_("Author")),
+                dp.Field(key="description", label=_("Description")),
+                dp.Field(key="updated", label=_("Last Update")),
+            ]
+        )
+
+    def get_instance_display(self) -> Display:
+        return create_simple_display(
+            [
+                ["title", "identifier"],
+                ["author", "updated"],
+                [repeat_field(2, "description")],
+                [repeat_field(2, "news_section")],
+            ],
+            [create_simple_section("news_section", "News", [["news"]], "news", collapsed=False)],
+        )
+
+
+class NewsDisplayConfig(DisplayViewConfig):
+    def get_instance_display(self) -> Display:
+        return create_simple_display(
+            [
+                ["datetime", "source"],
+                ["language", "link"],
+                [repeat_field(2, "summary")],
+                [repeat_field(2, "description")],
+            ]
+        )
+
+    def get_list_display(self) -> Optional[dp.ListDisplay]:
+        return dp.ListDisplay(
+            fields=[
+                dp.Field(key="datetime", label=_("Datetime")),
+                dp.Field(key="title", label=_("Title")),
+                dp.Field(key="summary", label=_("Summary")),
+                dp.Field(key="description", label=_("Description")),
+                # dp.Field(key="tags", label=_("Edited")),
+                dp.Field(key="source", label=_("Source")),
+                dp.Field(key="language", label=_("Language")),
+                dp.Field(key="image_url", label=_("Image")),
+            ]
+        )
+
+
+class NewsRelationshipDisplayConfig(DisplayViewConfig):
+    def get_instance_display(self) -> Display:
+        return create_simple_display(
+            [
+                ["datetime", "source"],
+                ["language", "link"],
+                [repeat_field(2, "summary")],
+                [repeat_field(2, "description")],
+            ]
+        )
+
+    def get_list_display(self) -> Optional[dp.ListDisplay]:
+        POSITIVE = "#96DD99"
+        SLIGHTLY_POSITIVE = "#FFEE8C"
+        SLIGHTLY_NEGATIVE = "#FF964F"
+        NEGATIVE = "#FF6961"
+
+        return dp.ListDisplay(
+            fields=[
+                dp.Field(key="datetime", label=_("Datetime")),
+                dp.Field(key="analysis", label=_("Analysis")),
+                dp.Field(key="title", label=_("Title")),
+                dp.Field(key="summary", label=_("Summary")),
+                dp.Field(key="description", label=_("Description")),
+            ],
+            formatting=[
+                dp.Formatting(
+                    column="sentiment",
+                    formatting_rules=[
+                        dp.FormattingRule(condition=("==", 4), style={"backgroundColor": POSITIVE}),
+                        dp.FormattingRule(condition=("==", 3), style={"backgroundColor": SLIGHTLY_POSITIVE}),
+                        dp.FormattingRule(condition=("==", 2), style={"backgroundColor": SLIGHTLY_NEGATIVE}),
+                        dp.FormattingRule(condition=("==", 1), style={"backgroundColor": NEGATIVE}),
+                    ],
+                )
+            ],
+            legends=[
+                dp.Legend(
+                    items=[
+                        dp.LegendItem(icon=POSITIVE, label=_("Positive")),
+                        dp.LegendItem(icon=SLIGHTLY_POSITIVE, label=_("Slightly Positive")),
+                        dp.LegendItem(icon=SLIGHTLY_NEGATIVE, label=_("Slightly Negative")),
+                        dp.LegendItem(icon=NEGATIVE, label=_("Negative")),
+                    ]
+                )
+            ],
+        )
+
+
+class NewsSourceDisplayConfig(DisplayViewConfig):
+    def get_instance_display(self) -> Display:
+        return create_simple_display(
+            [
+                [repeat_field(2, "title")],
+                ["datetime", "language", "link"],
+                [repeat_field(2, "description")],
+            ]
+        )
+
+    def get_list_display(self) -> Optional[dp.ListDisplay]:
+        return dp.ListDisplay(
+            fields=[
+                dp.Field(key="datetime", label=_("Datetime")),
+                dp.Field(key="title", label=_("Title")),
+                dp.Field(key="description", label=_("Description")),
+                dp.Field(key="language", label=_("Language")),
+                # dp.Field(key="tags", label="_(Edited")),
+                # dp.Field(key="link", label="_(Link"))
+            ]
+        )

wbnews/viewsets/endpoints.py

@@ -0,0 +1,18 @@
+from rest_framework.reverse import reverse
+from wbcore.metadata.configs.endpoints import EndpointViewConfig
+
+
+class NewsEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return None
+
+    def get_list_endpoint(self, **kwargs):
+        return reverse("wbnews:news-list", request=self.request)
+
+    def get_instance_endpoint(self, **kwargs):
+        return self.get_list_endpoint()
+
+
+class NewsSourceEndpointConfig(NewsEndpointConfig):
+    def get_list_endpoint(self, **kwargs):
+        return reverse("wbnews:source-news-list", args=[self.view.kwargs["source_id"]], request=self.request)

wbnews/viewsets/menu.py

@@ -0,0 +1,23 @@
+from django.utils.translation import gettext_lazy as _
+from wbcore.menus import ItemPermission, MenuItem
+from wbcore.permissions.shortcuts import is_internal_user
+
+NEWS_MENUITEM = MenuItem(
+    label=_("News"),
+    endpoint="wbnews:news-list",
+    permission=ItemPermission(permissions=["wbnews.view_news"]),
+)
+NEWSSOURCE_MENUITEM = MenuItem(
+    label=_("Sources"),
+    endpoint="wbnews:source-list",
+    permission=ItemPermission(
+        method=lambda request: is_internal_user(request.user), permissions=["wbnews.view_newssource"]
+    ),
+    add=MenuItem(
+        label=_("Create Source"),
+        endpoint="wbnews:source-list",
+        permission=ItemPermission(
+            method=lambda request: is_internal_user(request.user), permissions=["wbnews.add_newssource"]
+        ),
+    ),
+)