wbnews 2.2.1__py2.py3-none-any.whl

wbnews/.coveragerc

@@ -0,0 +1,23 @@
+[report]
+exclude_lines =
+    print()
+    def api_endpoints_root
+    def get_or_create_model_sql
+    def profile_check
+    if hasattr
+    raise Exception
+    raise Http404
+    except:
+
+
+[run]
+omit = */migrations/*
+    manage.py
+    */tests/*
+    */wbnews_config/*
+    */apps.py
+    */docs/*
+    */dynamic_preferences_registry.py
+    */permissions.py
+    */preferences/*
+    */.venv/*

wbnews/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

wbnews/admin.py

@@ -0,0 +1,27 @@
+from django.contrib import admin
+
+from .models import News, NewsRelationship, NewsSource
+
+
+@admin.register(NewsRelationship)
+class NewsRelationshipAdmin(admin.ModelAdmin):
+    list_display = ["news", "content_object"]
+    autocomplete_fields = ["news"]
+
+
+@admin.register(News)
+class NewsAdmin(admin.ModelAdmin):
+    search_fields = ("title", "description")
+    raw_id_fields = ["import_source"]
+    autocomplete_fields = [
+        "source",
+    ]
+    list_display = ["title", "language", "tags", "source", "datetime"]
+
+    list_filter = ("source",)
+
+
+@admin.register(NewsSource)
+class NewsSourceAdmin(admin.ModelAdmin):
+    search_fields = ("type", "title", "identifier", "description", "author", "url")
+    list_filter = ("type",)

wbnews/apps.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+from django.utils.module_loading import autodiscover_modules
+
+
+class NewConfig(AppConfig):
+    name = "wbnews"
+
+    def ready(self) -> None:
+        autodiscover_modules("news")

wbnews/factories.py

@@ -0,0 +1,33 @@
+import factory
+from django.conf.global_settings import LANGUAGES
+from django.utils import timezone
+from faker import Factory
+from wbnews.models import News, NewsSource
+
+langs = [n for (n, v) in LANGUAGES]
+faker = Factory.create()
+
+
+class NewsSourceFactory(factory.django.DjangoModelFactory):
+    title = factory.Sequence(lambda n: f"source_{n}")
+    identifier = factory.Sequence(lambda n: f"http://myurl_{n}.com")
+    image = faker.url()
+    description = factory.Faker("sentence", nb_words=32)
+    author = faker.name()
+    url = factory.Faker("url")
+
+    class Meta:
+        model = NewsSource
+
+
+class NewsFactory(factory.django.DjangoModelFactory):
+    datetime = factory.LazyFunction(timezone.now)
+    title = factory.Sequence(lambda n: f"news_{n}")
+    description = factory.Faker("sentence", nb_words=32)
+    summary = factory.Faker("sentence", nb_words=32)
+    language = factory.Iterator(langs)
+    link = faker.url()
+    source = factory.SubFactory(NewsSourceFactory)
+
+    class Meta:
+        model = News

wbnews/import_export/backends/__init__.py

@@ -0,0 +1 @@
+from .news import DataBackend

wbnews/import_export/backends/news.py

@@ -0,0 +1,35 @@
+import json
+from datetime import datetime
+from io import BytesIO
+from typing import Generator
+
+import feedparser
+from django.db.models import QuerySet
+from slugify import slugify
+from wbcore.contrib.io.backends.abstract import AbstractDataBackend
+from wbcore.contrib.io.backends.utils import register
+from wbnews.models import NewsSource
+
+
+@register("News RSS Backend", save_data_in_import_source=True)
+class DataBackend(AbstractDataBackend):
+    def is_object_valid(self, obj: "NewsSource") -> bool:
+        return obj.type == NewsSource.Type.RSS and obj.is_active and obj.url
+
+    def get_default_queryset(self) -> QuerySet["NewsSource"]:
+        return NewsSource.objects.filter(type=NewsSource.Type.RSS, is_active=True, url__isnull=False)
+
+    def get_files(
+        self, execution_time: datetime, queryset=None, **kwargs
+    ) -> Generator[tuple[str, BytesIO], None, None] | None:
+        if queryset is not None:
+            for source in queryset:
+                data = feedparser.parse(source.url)
+                if not data.get("bozo_exception"):
+                    data["news_source"] = source.id
+                    content_file = BytesIO()
+                    content_file.write(json.dumps(data).encode())
+                    file_name = (
+                        f"{slugify(source.title, separator='_')}_rss_file_{datetime.timestamp(execution_time)}.json"
+                    )
+                    yield file_name, content_file

wbnews/import_export/handlers/__init__.py

@@ -0,0 +1 @@
+from .news import NewsImportHandler

wbnews/import_export/handlers/news.py

@@ -0,0 +1,25 @@
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+import pytz
+from django.db import models
+from django.utils import timezone
+from wbcore.contrib.io.imports import ImportExportHandler
+
+
+class NewsImportHandler(ImportExportHandler):
+    MODEL_APP_LABEL = "wbnews.News"
+
+    def _deserialize(self, data: Dict[str, Any]):
+        data["source"] = self.model.source_dict_to_model(data["source"])
+        if parsed_datetime := data.get("datetime", None):
+            data["datetime"] = pytz.utc.localize(datetime.strptime(parsed_datetime, "%Y-%m-%dT%H:%M:%S"))
+        else:
+            data["datetime"] = timezone.now()
+
+    def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
+        return self.model.objects.filter(source=data["source"], datetime=data["datetime"], title=data["title"]).first()
+
+    def _create_instance(self, data: Dict[str, Any], **kwargs) -> models.Model:
+        self.import_source.log += "\nCreate News."
+        return self.model.objects.create(**data, import_source=self.import_source)

wbnews/import_export/parsers/emails/news.py

@@ -0,0 +1,48 @@
+import re
+from contextlib import suppress
+from datetime import datetime
+
+from django.conf.global_settings import LANGUAGES
+from langdetect import detect, lang_detect_exception
+from wbcore.utils.importlib import import_from_dotted_path
+
+from .utils import EmlContentParser
+
+languages_dict = dict(LANGUAGES)
+
+
+def clean_string_with_paragraphs(string):
+    return re.sub(r"  +", " ", re.sub(r"(?<!\.)\\n", " ", string.strip()))
+
+
+def parse(import_source):
+    parser = EmlContentParser(
+        import_source.file.read(), encoding=import_source.source.import_parameters.get("email_encoding", "latin-1")
+    )
+    email_date = parser.date if parser.date else datetime.now()
+
+    # Source
+    html = parser.html
+
+    # If source define a custom html parser, we import it and convert the returned html
+    if html_parser_path := import_source.source.import_parameters.get("html_parser", None):
+        with suppress(ModuleNotFoundError):
+            html_parser = import_from_dotted_path(html_parser_path)
+            html = html_parser(html)
+
+    data = {
+        "datetime": email_date.strftime("%Y-%m-%dT%H:%M:%S"),
+        "title": parser.subject.replace(f"[{import_source.source.uuid}]", ""),
+        "description": html,
+        "source": parser.source,
+    }
+
+    # Language
+    try:
+        language = detect(data["description"])
+        if language in languages_dict:
+            data["language"] = language
+    except lang_detect_exception.LangDetectException:
+        pass
+
+    return {"data": [data]}

wbnews/import_export/parsers/emails/utils.py

@@ -0,0 +1,61 @@
+from email import message, parser
+from email.utils import parseaddr, parsedate_to_datetime
+
+
+class EmlContentParser:
+    def __init__(self, email: bytes, encoding: str = "latin-1"):
+        self.message = parser.BytesParser().parsebytes(email)
+        self.encoding = encoding
+
+    @property
+    def date(self):
+        if date_str := self.message.get("date"):
+            return parsedate_to_datetime(date_str)
+
+    @property
+    def subject(self) -> str:
+        return self.message.get("subject", "")
+
+    @property
+    def html(self):
+        html = self.get_html(self.message)
+        return html.decode(self.encoding) if html else None
+
+    def get_html(cls, parsed: message.Message) -> bytes | None:
+        if parsed.is_multipart():
+            for item in parsed.get_payload():  # type:message.Message
+                if html := cls.get_html(item):
+                    return html
+        elif parsed.get_content_type() == "text/html":
+            return parsed.get_payload(decode=True)
+        return None
+
+    @property
+    def text(self):
+        text = self.get_text(self.message)
+        return text.decode(self.encoding) if text else None
+
+    @classmethod
+    def get_text(cls, parsed: message.Message) -> bytes | None:
+        if parsed.is_multipart():
+            for item in parsed.get_payload():
+                if text := cls.get_text(item):
+                    return text
+        elif parsed.get_content_type() == "text/plain":
+            return parsed.get_payload(decode=True)
+        return None
+
+    @property
+    def source(self) -> dict[str, any]:
+        name, email = parseaddr(self.message["From"])
+        if not name:
+            name = "Generic"
+        if not email:
+            email = "generic"
+        source = {
+            "title": f"{name} Research Email",
+            "identifier": "research-email-" + email.lower(),
+            "author": name,
+            "url": email,
+        }
+        return source

wbnews/import_export/parsers/rss/news.py

@@ -0,0 +1,63 @@
+import json
+from datetime import datetime
+from time import mktime
+
+from django.conf.global_settings import LANGUAGES
+from langdetect import detect, lang_detect_exception
+
+languages_dict = dict(LANGUAGES)
+
+
+def _get_source(d):
+    source = {}
+    if source_id := d.get("news_source"):
+        source["id"] = source_id
+    else:
+        if "title" in d["feed"]:
+            source["title"] = d["feed"]["title"]
+        if "author" in d["feed"]:
+            source["author"] = d["feed"]["author"]
+        if "image" in d["feed"]:
+            source["image"] = d["feed"]["image"]["href"]
+        if "href" in d["feed"]:
+            source["identifier"] = d["feed"]["href"]
+        if "link" in d["feed"]:
+            source["url"] = d["feed"]["link"]
+    return source
+
+
+def parse(import_source):
+    content = json.load(import_source.file)
+    data = []
+    source = _get_source(content)
+
+    for entry in content["entries"]:
+        if summary := entry.get("summary", None):
+            description = entry.get("description", summary)
+            res = {
+                "description": description,
+                "summary": summary,
+                "source": source,
+                "title": entry.get("title", ""),
+                "link": entry.get("link", None),
+            }
+            try:
+                language = detect(entry["summary"])
+                if language in languages_dict:
+                    res["language"] = language
+            except lang_detect_exception.LangDetectException:
+                pass
+            if published_parsed := entry.get("published_parsed", None):
+                updated = datetime.fromtimestamp(mktime(tuple(published_parsed)))
+                res["datetime"] = updated.strftime("%Y-%m-%dT%H:%M:%S")
+            if enclosures := entry.get("enclosures", None):
+                res["enclosures"] = [e.get("href", "") for e in enclosures]
+            if (
+                (media_content := entry.get("media_content", []))
+                and isinstance(media_content, list)
+                and len(media_content) > 0
+                and (image_url := media_content[0].get("url", None))
+            ):
+                res["image_url"] = image_url
+            data.append(res)
+    return {"data": data}