wbnews 1.45.0__py2.py3-none-any.whl → 1.58.1__py2.py3-none-any.whl

wbnews/locale/en/LC_MESSAGES/django.po

@@ -0,0 +1,159 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 11:37+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: models/news.py:51 viewsets/display.py:54 viewsets/display.py:100
+#: viewsets/display.py:141
+msgid "Datetime"
+msgstr ""
+
+#: models/news.py:52 serializers.py:21 serializers.py:76 viewsets/display.py:20
+#: viewsets/display.py:55 viewsets/display.py:102 viewsets/display.py:142
+msgid "Title"
+msgstr ""
+
+#: models/news.py:54 serializers.py:24 serializers.py:77 viewsets/display.py:23
+#: viewsets/display.py:57 viewsets/display.py:104 viewsets/display.py:143
+msgid "Description"
+msgstr ""
+
+#: models/news.py:55 serializers.py:78 viewsets/display.py:56
+#: viewsets/display.py:103
+msgid "Summary"
+msgstr ""
+
+#: models/news.py:56 viewsets/display.py:60 viewsets/display.py:144
+msgid "Language"
+msgstr ""
+
+#: models/news.py:57
+msgid "Link"
+msgstr ""
+
+#: models/news.py:61 viewsets/display.py:59 viewsets/display.py:106
+msgid "Source"
+msgstr ""
+
+#: models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: serializers.py:22
+msgid "Identifier"
+msgstr ""
+
+#: serializers.py:25 viewsets/display.py:22
+msgid "Author"
+msgstr ""
+
+#: serializers.py:26
+msgid "Updated"
+msgstr ""
+
+#: serializers.py:79
+msgid "Date"
+msgstr ""
+
+#: serializers.py:92 viewsets/menu.py:6 viewsets/titles.py:24
+msgid "News"
+msgstr ""
+
+#: viewsets/buttons.py:14
+msgid "Open News"
+msgstr ""
+
+#: viewsets/buttons.py:21 viewsets/buttons.py:22 viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: viewsets/display.py:21
+msgid "RSS feed"
+msgstr ""
+
+#: viewsets/display.py:24
+msgid "Last Update"
+msgstr ""
+
+#: viewsets/display.py:61
+msgid "Image"
+msgstr ""
+
+#: viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: viewsets/menu.py:17 viewsets/titles.py:9
+msgid "Sources"
+msgstr ""
+
+#: viewsets/menu.py:23
+msgid "Create Source"
+msgstr ""
+
+#: viewsets/titles.py:13
+#, python-brace-format
+msgid "Source: {source}"
+msgstr ""
+
+#: viewsets/titles.py:14
+msgid "News Source"
+msgstr ""
+
+#: viewsets/titles.py:19
+msgid "News Flow"
+msgstr ""
+
+#: viewsets/titles.py:30
+#, python-brace-format
+msgid "News from {source}"
+msgstr ""
+
+#: viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: viewsets/titles.py:37 viewsets/titles.py:44
+msgid "News Article"
+msgstr ""

wbnews/locale/fr/LC_MESSAGES/django.po

@@ -0,0 +1,162 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 11:37+0200\n"
+"PO-Revision-Date: 2025-05-30 09:40+0000\n"
+"Language-Team: French (https://app.transifex.com/stainly/teams/171242/fr/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: fr\n"
+"Plural-Forms: nplurals=3; plural=(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2;\n"
+
+#: models/news.py:51 viewsets/display.py:54 viewsets/display.py:100
+#: viewsets/display.py:141
+msgid "Datetime"
+msgstr ""
+
+#: models/news.py:52 serializers.py:21 serializers.py:76
+#: viewsets/display.py:20 viewsets/display.py:55 viewsets/display.py:102
+#: viewsets/display.py:142
+msgid "Title"
+msgstr ""
+
+#: models/news.py:54 serializers.py:24 serializers.py:77
+#: viewsets/display.py:23 viewsets/display.py:57 viewsets/display.py:104
+#: viewsets/display.py:143
+msgid "Description"
+msgstr ""
+
+#: models/news.py:55 serializers.py:78 viewsets/display.py:56
+#: viewsets/display.py:103
+msgid "Summary"
+msgstr ""
+
+#: models/news.py:56 viewsets/display.py:60 viewsets/display.py:144
+msgid "Language"
+msgstr ""
+
+#: models/news.py:57
+msgid "Link"
+msgstr ""
+
+#: models/news.py:61 viewsets/display.py:59 viewsets/display.py:106
+msgid "Source"
+msgstr ""
+
+#: models/news.py:64
+msgid "Mark as duplicate"
+msgstr ""
+
+#: models/relationships.py:9
+msgid "Positive"
+msgstr ""
+
+#: models/relationships.py:10
+msgid "Slightly Positive"
+msgstr ""
+
+#: models/relationships.py:11
+msgid "Slightly Negative"
+msgstr ""
+
+#: models/relationships.py:12
+msgid "Negative"
+msgstr ""
+
+#: serializers.py:22
+msgid "Identifier"
+msgstr ""
+
+#: serializers.py:25 viewsets/display.py:22
+msgid "Author"
+msgstr ""
+
+#: serializers.py:26
+msgid "Updated"
+msgstr ""
+
+#: serializers.py:79
+msgid "Date"
+msgstr ""
+
+#: serializers.py:92 viewsets/menu.py:6 viewsets/titles.py:24
+msgid "News"
+msgstr ""
+
+#: viewsets/buttons.py:14
+msgid "Open News"
+msgstr ""
+
+#: viewsets/buttons.py:21 viewsets/buttons.py:22 viewsets/buttons.py:24
+msgid "Reset relationships"
+msgstr ""
+
+#: viewsets/display.py:21
+msgid "RSS feed"
+msgstr ""
+
+#: viewsets/display.py:24
+msgid "Last Update"
+msgstr ""
+
+#: viewsets/display.py:61
+msgid "Image"
+msgstr ""
+
+#: viewsets/display.py:93
+msgid "Linked Object"
+msgstr ""
+
+#: viewsets/display.py:101
+msgid "Analysis"
+msgstr ""
+
+#: viewsets/display.py:105
+msgid "Important"
+msgstr ""
+
+#: viewsets/menu.py:11
+msgid "News Relationships"
+msgstr ""
+
+#: viewsets/menu.py:17 viewsets/titles.py:9
+msgid "Sources"
+msgstr ""
+
+#: viewsets/menu.py:23
+msgid "Create Source"
+msgstr ""
+
+#: viewsets/titles.py:13
+#, python-brace-format
+msgid "Source: {source}"
+msgstr ""
+
+#: viewsets/titles.py:14
+msgid "News Source"
+msgstr ""
+
+#: viewsets/titles.py:19
+msgid "News Flow"
+msgstr ""
+
+#: viewsets/titles.py:30
+#, python-brace-format
+msgid "News from {source}"
+msgstr ""
+
+#: viewsets/titles.py:36
+msgid "News Article for {}"
+msgstr ""
+
+#: viewsets/titles.py:37 viewsets/titles.py:44
+msgid "News Article"
+msgstr ""

wbnews/migrations/0011_newsrelationship_content_object_repr.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.12 on 2025-02-13 10:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbnews', '0010_newsrelationship_important'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='newsrelationship',
+            name='content_object_repr',
+            field=models.CharField(default='', max_length=512),
+        ),
+    ]

wbnews/migrations/0012_alter_news_unique_together_news_identifier_and_more.py

@@ -0,0 +1,91 @@
+# Generated by Django 5.0.12 on 2025-02-24 13:45
+from collections import defaultdict
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+from django.db import migrations, models
+from tqdm import tqdm
+
+
+def migrate_identifier( apps, schema_editor):
+    from wbnews.models.news import News as NewsClass
+    News = apps.get_model('wbnews', 'News')
+    objs = []
+    qs = News.objects.all()
+    guid = set()
+    for new in tqdm(qs, total=qs.count()):
+        new.guid = NewsClass.get_default_guid(new.title, new.link)
+        if new.guid not in guid:
+            guid.add(new.guid)
+            objs.append(new)
+    print(len(objs))
+    News.objects.bulk_update(objs, ['guid'], batch_size=10000)
+
+    leftovers = News.objects.filter(guid__isnull=True)
+    print(leftovers.count())
+    leftovers.delete()
+
+    qs = News.objects.all()
+    for new in tqdm(qs, total=qs.count()):
+        try:
+            URLValidator()(new.link)
+        except ValidationError:
+            new.link = None
+            objs.append(new)
+
+    News.objects.bulk_update(objs, ['link'], batch_size=10000)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('io', '0008_importsource_resource_kwargs'),
+        ('wbnews', '0011_newsrelationship_content_object_repr'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='news',
+            unique_together=set(),
+        ),
+        migrations.AddField(
+            model_name='news',
+            name='guid',
+            field=models.CharField(default=None, blank=True, null=True, max_length=1024),
+            preserve_default=False,
+        ),
+         migrations.RunSQL(sql="SET CONSTRAINTS ALL IMMEDIATE;"),
+        migrations.RunPython(migrate_identifier),
+        migrations.RunSQL(sql="SET CONSTRAINTS ALL DEFERRED;"),
+        migrations.AlterField(
+            model_name='news',
+            name='guid',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='news',
+            name='link',
+            field=models.URLField(max_length=1024, blank=True, null=True, verbose_name='Link'),
+        ),
+        migrations.AddField(
+            model_name='news',
+            name='mark_as_duplicate',
+            field=models.BooleanField(default=False, verbose_name='Mark as duplicate'),
+        ),
+        migrations.AlterField(
+            model_name='newsrelationship',
+            name='sentiment',
+            field=models.PositiveIntegerField(blank=True, choices=[(4, 'Positive'), (3, 'Slightly Positive'),
+                                                                   (2, 'Slightly Negative'), (1, 'Negative')],
+                                              null=True),
+        ),
+        migrations.RenameField(
+            model_name='newssource',
+            old_name='url',
+            new_name='endpoint',
+        ),
+        migrations.AlterField(
+            model_name='newssource',
+            name='endpoint',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+    ]

wbnews/migrations/0013_alter_news_datetime.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.12 on 2025-04-03 08:08
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbnews', '0012_alter_news_unique_together_news_identifier_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='datetime',
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='Datetime'),
+        ),
+    ]

wbnews/migrations/0014_newsrelationship_unique_news_relationship.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.14 on 2025-05-06 14:44
+
+from django.db import migrations, models
+from django.db.models import Count
+from tqdm import tqdm
+
+def handle_duplicated_relationship(apps, schema_editor):
+    NewsRelationship = apps.get_model("wbnews", "NewsRelationship")
+    qs = NewsRelationship.objects.values('content_type', 'object_id', 'news').annotate(c=Count('*')).filter(c__gt=1)
+    for row in tqdm(qs, total=qs.count()):
+        for rel in NewsRelationship.objects.filter(content_type=row['content_type'], news=row['news'], object_id=row["object_id"])[1:]:
+            rel.delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('wbnews', '0013_alter_news_datetime'),
+    ]
+
+    operations = [
+        migrations.RunPython(handle_duplicated_relationship),
+        migrations.AddConstraint(
+            model_name='newsrelationship',
+            constraint=models.UniqueConstraint(fields=('content_type', 'object_id', 'news'), name='unique_news_relationship'),
+        ),
+    ]

wbnews/models/llm/cleaned_news.py

@@ -5,27 +5,7 @@ from pydantic import BaseModel, Field
 from wbcore.contrib.ai.llm.config import LLMConfig
 
 if TYPE_CHECKING:
-    from wbnews.models import News
-
-
-def clean_news_prompt(news: "News"):
-    return [
-        SystemMessage(
-            content="I have an HTML email title and body, and I want to extract only the meaningful content in plain text format, removing all metadata, subscription links, and non-essential parts. The output should be HTML- and Markdown-free and should exclude any text related to links, subscription information, or common phrases like 'Unsubscribe' or 'View online'. Only retain the main email content but do not remove any information related to news."
-        ),
-        HumanMessage(
-            content=f"Title: {news.title}\n\nDescription: {news.description}",
-        ),
-    ]
-
-
-def summarize_news_prompt(news: "News"):
-    return [
-        SystemMessage(content="Given this news description, please extract a short summary"),
-        HumanMessage(
-            content=f"Description: {news.description}",
-        ),
-    ]
+    from wbnews.models import News  # noqa
 
 
 class CleanNewsModel(BaseModel):
@@ -46,18 +26,41 @@ class SummarizedNewsModel(BaseModel):
     )
 
 
+def get_clean_news_config_query(instance):
+    return {"description": instance.description, "title": instance.title}
+
+
 clean_news_config = LLMConfig["News"](
     key="clean",
     output_model=CleanNewsModel,
-    prompt=clean_news_prompt,
+    prompt=[
+        SystemMessage(
+            content="I have an HTML email title and body, and I want to extract only the meaningful content in plain text format, removing all metadata, subscription links, and non-essential parts. The output should be HTML- and Markdown-free and should exclude any text related to links, subscription information, or common phrases like 'Unsubscribe' or 'View online'. Only retain the main email content but do not remove any information related to news."
+        ),
+        HumanMessage(
+            content="Title: {title}\n\nDescription: {description}",
+        ),
+    ],
     on_save=True,
     on_condition=lambda n: n.source.clean_content,
+    query=get_clean_news_config_query,
 )
 
+
+def get_summarized_news_config_query(instance):
+    return {"description": instance.description}
+
+
 summarized_news_config = LLMConfig["News"](
     key="summarize",
     output_model=SummarizedNewsModel,
-    prompt=summarize_news_prompt,
+    prompt=[
+        SystemMessage(content="Given this news description, please extract a short summary"),
+        HumanMessage(
+            content="Description: {description}",
+        ),
+    ],
     on_save=True,
     on_condition=lambda n: not n.summary,
+    query=get_summarized_news_config_query,
 )

wbnews/models/news.py

@@ -1,3 +1,4 @@
+from datetime import date
 from typing import Any
 
 from celery import chord, shared_task
@@ -7,7 +8,9 @@ from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.db.models.signals import post_save
 from django.dispatch import receiver
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from slugify import slugify
 from wbcore.contrib.ai.llm.decorators import llm
 from wbcore.contrib.io.mixins import ImportMixin
 from wbcore.models import WBModel
@@ -15,9 +18,10 @@ from wbcore.models import WBModel
 from wbnews.import_export.handlers.news import NewsImportHandler
 from wbnews.models.llm.cleaned_news import clean_news_config, summarized_news_config
 from wbnews.models.relationships import NewsRelationship
-from wbnews.models.sources import NewsSource
 from wbnews.signals import create_news_relationships
 
+from ..utils import detect_near_duplicates
+
 
 @shared_task
 def create_relationship(chain_results: list[list[dict[str, Any]]], news_id: int):
@@ -32,6 +36,11 @@ def create_relationship(chain_results: list[list[dict[str, Any]]], news_id: int)
     )
 
 
+class DefaultObjectManager(models.Manager):
+    def get_queryset(self):
+        return super().get_queryset().filter(mark_as_duplicate=False)
+
+
 @llm([clean_news_config, summarized_news_config])
 class News(ImportMixin, WBModel):
     errors = {
@@ -39,21 +48,29 @@ class News(ImportMixin, WBModel):
     }
     import_export_handler_class = NewsImportHandler
 
-    datetime = models.DateTimeField(verbose_name=_("Datetime"))
+    datetime = models.DateTimeField(verbose_name=_("Datetime"), default=timezone.now)
     title = models.CharField(max_length=500, verbose_name=_("Title"))
+    guid = models.CharField(max_length=1024, unique=True)
     description = models.TextField(blank=True, verbose_name=_("Description"))
     summary = models.TextField(blank=True, verbose_name=_("Summary"))
     language = models.CharField(max_length=16, choices=LANGUAGES, blank=True, null=True, verbose_name=_("Language"))
-    link = models.CharField(max_length=500, blank=True, null=True, verbose_name=_("Link"))
+    link = models.URLField(max_length=1024, blank=True, null=True, verbose_name=_("Link"))
     tags = ArrayField(models.CharField(max_length=16), default=list)
     enclosures = ArrayField(models.URLField(), default=list)
     source = models.ForeignKey(
         "wbnews.NewsSource", on_delete=models.CASCADE, related_name="news", verbose_name=_("Source")
     )
     image_url = models.URLField(blank=True, null=True)
+    mark_as_duplicate = models.BooleanField(default=False, verbose_name=_("Mark as duplicate"))
 
-    class Meta:
-        unique_together = ["title", "source", "datetime"]
+    objects = DefaultObjectManager()
+    all_objects = models.Manager()
+
+    def save(self, *args, **kwargs):
+        self.datetime = min(self.datetime, timezone.now())  # we ensure a news is never in the future
+        if self.guid is None:
+            self.guid = self.get_default_guid(self.title, self.link)
+        super().save(*args, **kwargs)
 
     def __str__(self) -> str:
         return f"{self.title} ({self.source.title})"
@@ -64,7 +81,8 @@ class News(ImportMixin, WBModel):
         """
         tasks = []
         for sender, task_signature in create_news_relationships.send(sender=News, instance=self):
-            assert isinstance(task_signature, Signature), self.errors["relationship_signal"].format(sender)
+            if not isinstance(task_signature, Signature):
+                raise AssertionError(self.errors["relationship_signal"].format(sender))
             tasks.append(task_signature)
         if tasks:
             res = chord(tasks, create_relationship.s(self.id))
@@ -73,22 +91,11 @@ class News(ImportMixin, WBModel):
             else:
                 res.apply_async()
 
-    # TODO: Consider moving this into a get_or_create queryset method on NewsSource?
     @classmethod
-    def source_dict_to_model(cls, data: dict) -> NewsSource:
-        sources = NewsSource.objects
-        if "id" in data:
-            return sources.get(id=data["id"])
-        if identifier := data.get("identifier"):
-            sources = sources.filter(identifier=identifier)
-        elif url := data.get("url"):
-            sources = sources.filter(url=url)
-        elif title := data.get("title"):
-            sources = sources.filter(title=title)
-        if sources.count() == 1:
-            return sources.first()
-        else:
-            return NewsSource.objects.create(**data)
+    def get_default_guid(cls, title: str, link: str | None, max_length: int = 1024) -> str:
+        if link:
+            return link
+        return slugify(title)[0:max_length]
 
     @classmethod
     def get_representation_endpoint(cls) -> str:
@@ -106,6 +113,13 @@ class News(ImportMixin, WBModel):
     def get_endpoint_basename(cls) -> str:
         return "wbnews:news"
 
+    @classmethod
+    def handle_duplicates(cls, start: date, end: date, content_label: str = "description", threshold: float = 0.9):
+        qs = News.objects.filter(datetime__gte=start, datetime__lte=end)
+        data = dict(qs.values_list("id", content_label))
+        duplicate_ids = detect_near_duplicates(data, threshold=threshold)
+        qs.filter(id__in=duplicate_ids).update(mark_as_duplicate=True)
+
 
 @receiver(post_save, sender="wbnews.News")
 def post_save_create_news_relationships(sender: type, instance: "News", raw: bool, created: bool, **kwargs):