wbnews 1.58.3__py2.py3-none-any.whl

wbnews/migrations/0006_alter_news_language.py

@@ -0,0 +1,122 @@
+# Generated by Django 4.2.6 on 2023-10-18 13:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("wbnews", "0001_initial_squashed_0005_alter_news_import_source"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="news",
+            name="language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("af", "Afrikaans"),
+                    ("ar", "Arabic"),
+                    ("ar-dz", "Algerian Arabic"),
+                    ("ast", "Asturian"),
+                    ("az", "Azerbaijani"),
+                    ("bg", "Bulgarian"),
+                    ("be", "Belarusian"),
+                    ("bn", "Bengali"),
+                    ("br", "Breton"),
+                    ("bs", "Bosnian"),
+                    ("ca", "Catalan"),
+                    ("ckb", "Central Kurdish (Sorani)"),
+                    ("cs", "Czech"),
+                    ("cy", "Welsh"),
+                    ("da", "Danish"),
+                    ("de", "German"),
+                    ("dsb", "Lower Sorbian"),
+                    ("el", "Greek"),
+                    ("en", "English"),
+                    ("en-au", "Australian English"),
+                    ("en-gb", "British English"),
+                    ("eo", "Esperanto"),
+                    ("es", "Spanish"),
+                    ("es-ar", "Argentinian Spanish"),
+                    ("es-co", "Colombian Spanish"),
+                    ("es-mx", "Mexican Spanish"),
+                    ("es-ni", "Nicaraguan Spanish"),
+                    ("es-ve", "Venezuelan Spanish"),
+                    ("et", "Estonian"),
+                    ("eu", "Basque"),
+                    ("fa", "Persian"),
+                    ("fi", "Finnish"),
+                    ("fr", "French"),
+                    ("fy", "Frisian"),
+                    ("ga", "Irish"),
+                    ("gd", "Scottish Gaelic"),
+                    ("gl", "Galician"),
+                    ("he", "Hebrew"),
+                    ("hi", "Hindi"),
+                    ("hr", "Croatian"),
+                    ("hsb", "Upper Sorbian"),
+                    ("hu", "Hungarian"),
+                    ("hy", "Armenian"),
+                    ("ia", "Interlingua"),
+                    ("id", "Indonesian"),
+                    ("ig", "Igbo"),
+                    ("io", "Ido"),
+                    ("is", "Icelandic"),
+                    ("it", "Italian"),
+                    ("ja", "Japanese"),
+                    ("ka", "Georgian"),
+                    ("kab", "Kabyle"),
+                    ("kk", "Kazakh"),
+                    ("km", "Khmer"),
+                    ("kn", "Kannada"),
+                    ("ko", "Korean"),
+                    ("ky", "Kyrgyz"),
+                    ("lb", "Luxembourgish"),
+                    ("lt", "Lithuanian"),
+                    ("lv", "Latvian"),
+                    ("mk", "Macedonian"),
+                    ("ml", "Malayalam"),
+                    ("mn", "Mongolian"),
+                    ("mr", "Marathi"),
+                    ("ms", "Malay"),
+                    ("my", "Burmese"),
+                    ("nb", "Norwegian Bokmål"),
+                    ("ne", "Nepali"),
+                    ("nl", "Dutch"),
+                    ("nn", "Norwegian Nynorsk"),
+                    ("os", "Ossetic"),
+                    ("pa", "Punjabi"),
+                    ("pl", "Polish"),
+                    ("pt", "Portuguese"),
+                    ("pt-br", "Brazilian Portuguese"),
+                    ("ro", "Romanian"),
+                    ("ru", "Russian"),
+                    ("sk", "Slovak"),
+                    ("sl", "Slovenian"),
+                    ("sq", "Albanian"),
+                    ("sr", "Serbian"),
+                    ("sr-latn", "Serbian Latin"),
+                    ("sv", "Swedish"),
+                    ("sw", "Swahili"),
+                    ("ta", "Tamil"),
+                    ("te", "Telugu"),
+                    ("tg", "Tajik"),
+                    ("th", "Thai"),
+                    ("tk", "Turkmen"),
+                    ("tr", "Turkish"),
+                    ("tt", "Tatar"),
+                    ("udm", "Udmurt"),
+                    ("uk", "Ukrainian"),
+                    ("ur", "Urdu"),
+                    ("uz", "Uzbek"),
+                    ("vi", "Vietnamese"),
+                    ("zh-hans", "Simplified Chinese"),
+                    ("zh-hant", "Traditional Chinese"),
+                ],
+                max_length=16,
+                null=True,
+                verbose_name="Language",
+            ),
+        ),
+    ]

wbnews/migrations/0007_auto_20240103_0955.py

@@ -0,0 +1,43 @@
+# Generated by Django 4.2.8 on 2024-01-03 08:55
+
+from contextlib import suppress
+
+from django.apps import apps
+from django.contrib.contenttypes.models import ContentType
+from django.db import migrations
+
+
+def migrate_content_type(apps, schema_editor):
+    NewsRelationship = apps.get_model("wbnews", "NewsRelationship")
+
+    for model_name in [
+        "Classification",
+        "Instrument",
+        "ClassificationGroup",
+        "Deal",
+        "Exchange",
+        "InstrumentClassificationRelatedInstrument",
+        "InstrumentClassificationThroughModel",
+        "InstrumentFavoriteGroup",
+        "InstrumentList",
+        "InstrumentListThroughModel",
+        "InstrumentRequest",
+        "RelatedInstrumentThroughModel",
+        "InstrumentPrice",
+    ]:
+        with suppress(ContentType.DoesNotExist):
+            old_ct = ContentType.objects.get(app_label="wbportfolio", model=model_name.lower())
+            new_ct = ContentType.objects.get(app_label="wbfdm", model=model_name.lower())
+            NewsRelationship.objects.filter(content_type_id=old_ct.id).update(content_type_id=new_ct.id)
+
+
+class Migration(migrations.Migration):
+    dependencies = (
+        [
+            ("wbnews", "0006_alter_news_language"),
+            ("wbfdm", "0012_instrumentprice_created_instrumentprice_modified"),
+        ]
+        if apps.is_installed("wbfdm")
+        else [("wbnews", "0006_alter_news_language")]
+    )
+    operations = [migrations.RunPython(migrate_content_type)]

wbnews/migrations/0008_alter_news_language.py

@@ -0,0 +1,123 @@
+# Generated by Django 5.0.3 on 2024-04-11 08:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("wbnews", "0007_auto_20240103_0955"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="news",
+            name="language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("af", "Afrikaans"),
+                    ("ar", "Arabic"),
+                    ("ar-dz", "Algerian Arabic"),
+                    ("ast", "Asturian"),
+                    ("az", "Azerbaijani"),
+                    ("bg", "Bulgarian"),
+                    ("be", "Belarusian"),
+                    ("bn", "Bengali"),
+                    ("br", "Breton"),
+                    ("bs", "Bosnian"),
+                    ("ca", "Catalan"),
+                    ("ckb", "Central Kurdish (Sorani)"),
+                    ("cs", "Czech"),
+                    ("cy", "Welsh"),
+                    ("da", "Danish"),
+                    ("de", "German"),
+                    ("dsb", "Lower Sorbian"),
+                    ("el", "Greek"),
+                    ("en", "English"),
+                    ("en-au", "Australian English"),
+                    ("en-gb", "British English"),
+                    ("eo", "Esperanto"),
+                    ("es", "Spanish"),
+                    ("es-ar", "Argentinian Spanish"),
+                    ("es-co", "Colombian Spanish"),
+                    ("es-mx", "Mexican Spanish"),
+                    ("es-ni", "Nicaraguan Spanish"),
+                    ("es-ve", "Venezuelan Spanish"),
+                    ("et", "Estonian"),
+                    ("eu", "Basque"),
+                    ("fa", "Persian"),
+                    ("fi", "Finnish"),
+                    ("fr", "French"),
+                    ("fy", "Frisian"),
+                    ("ga", "Irish"),
+                    ("gd", "Scottish Gaelic"),
+                    ("gl", "Galician"),
+                    ("he", "Hebrew"),
+                    ("hi", "Hindi"),
+                    ("hr", "Croatian"),
+                    ("hsb", "Upper Sorbian"),
+                    ("hu", "Hungarian"),
+                    ("hy", "Armenian"),
+                    ("ia", "Interlingua"),
+                    ("id", "Indonesian"),
+                    ("ig", "Igbo"),
+                    ("io", "Ido"),
+                    ("is", "Icelandic"),
+                    ("it", "Italian"),
+                    ("ja", "Japanese"),
+                    ("ka", "Georgian"),
+                    ("kab", "Kabyle"),
+                    ("kk", "Kazakh"),
+                    ("km", "Khmer"),
+                    ("kn", "Kannada"),
+                    ("ko", "Korean"),
+                    ("ky", "Kyrgyz"),
+                    ("lb", "Luxembourgish"),
+                    ("lt", "Lithuanian"),
+                    ("lv", "Latvian"),
+                    ("mk", "Macedonian"),
+                    ("ml", "Malayalam"),
+                    ("mn", "Mongolian"),
+                    ("mr", "Marathi"),
+                    ("ms", "Malay"),
+                    ("my", "Burmese"),
+                    ("nb", "Norwegian Bokmål"),
+                    ("ne", "Nepali"),
+                    ("nl", "Dutch"),
+                    ("nn", "Norwegian Nynorsk"),
+                    ("os", "Ossetic"),
+                    ("pa", "Punjabi"),
+                    ("pl", "Polish"),
+                    ("pt", "Portuguese"),
+                    ("pt-br", "Brazilian Portuguese"),
+                    ("ro", "Romanian"),
+                    ("ru", "Russian"),
+                    ("sk", "Slovak"),
+                    ("sl", "Slovenian"),
+                    ("sq", "Albanian"),
+                    ("sr", "Serbian"),
+                    ("sr-latn", "Serbian Latin"),
+                    ("sv", "Swedish"),
+                    ("sw", "Swahili"),
+                    ("ta", "Tamil"),
+                    ("te", "Telugu"),
+                    ("tg", "Tajik"),
+                    ("th", "Thai"),
+                    ("tk", "Turkmen"),
+                    ("tr", "Turkish"),
+                    ("tt", "Tatar"),
+                    ("udm", "Udmurt"),
+                    ("ug", "Uyghur"),
+                    ("uk", "Ukrainian"),
+                    ("ur", "Urdu"),
+                    ("uz", "Uzbek"),
+                    ("vi", "Vietnamese"),
+                    ("zh-hans", "Simplified Chinese"),
+                    ("zh-hant", "Traditional Chinese"),
+                ],
+                max_length=16,
+                null=True,
+                verbose_name="Language",
+            ),
+        ),
+    ]

wbnews/migrations/0009_newsrelationship_analysis_newsrelationship_sentiment.py

@@ -0,0 +1,94 @@
+# Generated by Django 5.0.9 on 2024-10-24 11:00
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+def migrate_new_source(apps, schema_editor):
+    from wbcore.contrib.io.models import (
+        DataBackend,
+        ImportSource,
+        ParserHandler,
+        Source,
+    )
+
+    parser_handler = ParserHandler.objects.get(parser="wbnews.import_export.parsers.rss.news")
+    io_news_sources = Source.objects.filter(parser_handler=parser_handler)
+    crontab = io_news_sources.last().crontab
+    data_backend = DataBackend.objects.get(backend_class_path="wbnews.import_export.backends.news")
+    default_news_source, created = Source.objects.get_or_create(
+        data_backend=data_backend,
+        defaults={
+            "crontab": crontab,
+            "title": "RSS Feeds -> News",
+        },
+    )
+    default_news_source.parser_handler.set([parser_handler])
+    ImportSource.objects.filter(source__in=io_news_sources).update(source=default_news_source)
+    io_news_sources.exclude(id=default_news_source.id).delete()
+    NewsSource = apps.get_model("wbnews", "NewsSource")
+    NewsSource.objects.filter(title__icontains="Email").update(type="EMAIL")
+    NewsSource.objects.filter(type="RSS").filter(url__isnull=True, identifier__isnull=False).update(
+        url=models.F("identifier")
+    )
+
+    unvalid_rss_feeds = [
+        "https://www.dailytelegraph.com.au/help-rss",
+        "https://techcrunch.com/rssfeeds/",
+        "https://www.bbc.co.uk/news/",
+        "https://www.nytimes.com/section/technology",
+        "http://online.wsj.com/page/2_0006.html",
+        "http://online.wsj.com",
+        "https://www.bbc.co.uk/news/technology",
+        "https://www.bbc.co.uk/news/world",
+        "https://www.bbc.co.uk/news/business",
+        "https://www.techworld.com/news/rss",
+    ]
+    NewsSource.objects.filter(url__in=unvalid_rss_feeds).update(is_active=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("wbnews", "0008_alter_news_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="newsrelationship",
+            name="analysis",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="newsrelationship",
+            name="sentiment",
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="clean_content",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="is_active",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="newssource",
+            name="type",
+            field=models.CharField(choices=[("RSS", "RSS"), ("EMAIL", "EMAIL")], default="RSS", max_length=6),
+        ),
+        migrations.AlterField(
+            model_name="newssource",
+            name="description",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AlterField(
+            model_name="newssource",
+            name="tags",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=16), blank=True, default=list, size=None
+            ),
+        ),
+        migrations.RunPython(migrate_new_source),
+    ]

wbnews/migrations/0010_newsrelationship_important.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.9 on 2024-12-19 09:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("wbnews", "0009_newsrelationship_analysis_newsrelationship_sentiment"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="newsrelationship",
+            name="important",
+            field=models.BooleanField(blank=True, null=True),
+        ),
+    ]

wbnews/migrations/0011_newsrelationship_content_object_repr.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.12 on 2025-02-13 10:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbnews', '0010_newsrelationship_important'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='newsrelationship',
+            name='content_object_repr',
+            field=models.CharField(default='', max_length=512),
+        ),
+    ]

wbnews/migrations/0012_alter_news_unique_together_news_identifier_and_more.py

@@ -0,0 +1,91 @@
+# Generated by Django 5.0.12 on 2025-02-24 13:45
+from collections import defaultdict
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+from django.db import migrations, models
+from tqdm import tqdm
+
+
+def migrate_identifier( apps, schema_editor):
+    from wbnews.models.news import News as NewsClass
+    News = apps.get_model('wbnews', 'News')
+    objs = []
+    qs = News.objects.all()
+    guid = set()
+    for new in tqdm(qs, total=qs.count()):
+        new.guid = NewsClass.get_default_guid(new.title, new.link)
+        if new.guid not in guid:
+            guid.add(new.guid)
+            objs.append(new)
+    print(len(objs))
+    News.objects.bulk_update(objs, ['guid'], batch_size=10000)
+
+    leftovers = News.objects.filter(guid__isnull=True)
+    print(leftovers.count())
+    leftovers.delete()
+
+    qs = News.objects.all()
+    for new in tqdm(qs, total=qs.count()):
+        try:
+            URLValidator()(new.link)
+        except ValidationError:
+            new.link = None
+            objs.append(new)
+
+    News.objects.bulk_update(objs, ['link'], batch_size=10000)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('io', '0008_importsource_resource_kwargs'),
+        ('wbnews', '0011_newsrelationship_content_object_repr'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='news',
+            unique_together=set(),
+        ),
+        migrations.AddField(
+            model_name='news',
+            name='guid',
+            field=models.CharField(default=None, blank=True, null=True, max_length=1024),
+            preserve_default=False,
+        ),
+         migrations.RunSQL(sql="SET CONSTRAINTS ALL IMMEDIATE;"),
+        migrations.RunPython(migrate_identifier),
+        migrations.RunSQL(sql="SET CONSTRAINTS ALL DEFERRED;"),
+        migrations.AlterField(
+            model_name='news',
+            name='guid',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='news',
+            name='link',
+            field=models.URLField(max_length=1024, blank=True, null=True, verbose_name='Link'),
+        ),
+        migrations.AddField(
+            model_name='news',
+            name='mark_as_duplicate',
+            field=models.BooleanField(default=False, verbose_name='Mark as duplicate'),
+        ),
+        migrations.AlterField(
+            model_name='newsrelationship',
+            name='sentiment',
+            field=models.PositiveIntegerField(blank=True, choices=[(4, 'Positive'), (3, 'Slightly Positive'),
+                                                                   (2, 'Slightly Negative'), (1, 'Negative')],
+                                              null=True),
+        ),
+        migrations.RenameField(
+            model_name='newssource',
+            old_name='url',
+            new_name='endpoint',
+        ),
+        migrations.AlterField(
+            model_name='newssource',
+            name='endpoint',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+    ]

wbnews/migrations/0013_alter_news_datetime.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.12 on 2025-04-03 08:08
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbnews', '0012_alter_news_unique_together_news_identifier_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='datetime',
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='Datetime'),
+        ),
+    ]

wbnews/migrations/0014_newsrelationship_unique_news_relationship.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.14 on 2025-05-06 14:44
+
+from django.db import migrations, models
+from django.db.models import Count
+from tqdm import tqdm
+
+def handle_duplicated_relationship(apps, schema_editor):
+    NewsRelationship = apps.get_model("wbnews", "NewsRelationship")
+    qs = NewsRelationship.objects.values('content_type', 'object_id', 'news').annotate(c=Count('*')).filter(c__gt=1)
+    for row in tqdm(qs, total=qs.count()):
+        for rel in NewsRelationship.objects.filter(content_type=row['content_type'], news=row['news'], object_id=row["object_id"])[1:]:
+            rel.delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('wbnews', '0013_alter_news_datetime'),
+    ]
+
+    operations = [
+        migrations.RunPython(handle_duplicated_relationship),
+        migrations.AddConstraint(
+            model_name='newsrelationship',
+            constraint=models.UniqueConstraint(fields=('content_type', 'object_id', 'news'), name='unique_news_relationship'),
+        ),
+    ]

wbnews/models/__init__.py

@@ -0,0 +1,3 @@
+from .news import News
+from .relationships import NewsRelationship
+from .sources import NewsSource

wbnews/models/llm/cleaned_news.py

@@ -0,0 +1,66 @@
+from typing import TYPE_CHECKING
+
+from langchain_core.messages import HumanMessage, SystemMessage
+from pydantic import BaseModel, Field
+from wbcore.contrib.ai.llm.config import LLMConfig
+
+if TYPE_CHECKING:
+    from wbnews.models import News  # noqa
+
+
+class CleanNewsModel(BaseModel):
+    title: str = Field(
+        ...,
+        description="The news title cleaned from the email subject without any formatting only meaningful information",
+    )
+    description: str = Field(
+        ...,
+        description="The news description cleaned from the email body without any formatting and retaining only meaningful information",
+    )
+
+
+class SummarizedNewsModel(BaseModel):
+    summary: str = Field(
+        ...,
+        description="A summary of the news description in English.",
+    )
+
+
+def get_clean_news_config_query(instance):
+    return {"description": instance.description, "title": instance.title}
+
+
+clean_news_config = LLMConfig["News"](
+    key="clean",
+    output_model=CleanNewsModel,
+    prompt=[
+        SystemMessage(
+            content="I have an HTML email title and body, and I want to extract only the meaningful content in plain text format, removing all metadata, subscription links, and non-essential parts. The output should be HTML- and Markdown-free and should exclude any text related to links, subscription information, or common phrases like 'Unsubscribe' or 'View online'. Only retain the main email content but do not remove any information related to news."
+        ),
+        HumanMessage(
+            content="Title: {title}\n\nDescription: {description}",
+        ),
+    ],
+    on_save=True,
+    on_condition=lambda n: n.source.clean_content,
+    query=get_clean_news_config_query,
+)
+
+
+def get_summarized_news_config_query(instance):
+    return {"description": instance.description}
+
+
+summarized_news_config = LLMConfig["News"](
+    key="summarize",
+    output_model=SummarizedNewsModel,
+    prompt=[
+        SystemMessage(content="Given this news description, please extract a short summary"),
+        HumanMessage(
+            content="Description: {description}",
+        ),
+    ],
+    on_save=True,
+    on_condition=lambda n: not n.summary,
+    query=get_summarized_news_config_query,
+)