redturtle.rssservice 2.2.2__py3-none-any.whl

redturtle/rssservice/rss_mixer.py

@@ -0,0 +1,378 @@
+# -*- coding: utf-8 -*-
+from DateTime import DateTime
+from DateTime.interfaces import SyntaxError
+from os import environ
+from plone.dexterity.utils import iterSchemata
+from plone.restapi.serializer.converters import json_compatible
+from plone.restapi.serializer.utils import uid_to_url
+from plone.restapi.services import Service
+from redturtle.rssservice import _
+from redturtle.rssservice.interfaces import IRSSMixerFeed
+from requests.exceptions import RequestException
+from requests.exceptions import Timeout
+from time import time
+from zExceptions import BadRequest
+from zExceptions import NotFound
+from zope.i18n import translate
+from zope.interface import implementer
+from zope.schema import getFields
+from App.config import getConfiguration
+import feedparser
+import json
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+# Accept these bozo_exceptions encountered by feedparser when parsing
+# the feed:
+ACCEPTED_FEEDPARSER_EXCEPTIONS = (feedparser.CharacterEncodingOverride,)
+
+# store the feeds here (which means in RAM)
+FEED_DATA = {}  # url: ({date, title, url, itemlist})
+
+REQUESTS_TIMEOUT = int(environ.get("RSS_SERVICE_TIMEOUT", "5")) or 5
+REQUESTS_USER_AGENT = environ.get("RSS_USER_AGENT")
+RSSMIXER_HTTP_PROXY = environ.get("RSSMIXER_PROXY", "")
+
+DEBUGMODE = getConfiguration().debug_mode
+
+
+class RSSMixerService(Service):
+    """ """
+
+    def reply(self):
+        feed_config = self.get_feed_config()
+        limit = feed_config.get("limit", 20)
+        feeds = feed_config.get("feeds", [])
+        if not feeds:
+            raise BadRequest(
+                translate(
+                    _(
+                        "missing_feeds_parameter",
+                        default="Missing required parameter: feeds",
+                    ),
+                    context=self.request,
+                )
+            )
+        return self._getFeeds(feeds=feeds, limit=limit)
+
+    def get_feed_config(self):
+        """ """
+        query = self.request.form
+        if DEBUGMODE and query.get("rss_debug_uri"):
+            return {"feeds": [{"url": query.get("rss_debug_uri")}]}
+
+        block_id = query.get("block", "")
+        if not block_id:
+            raise BadRequest(
+                translate(
+                    _(
+                        "missing_block_id",
+                        default="Missing required parameter: block",
+                    ),
+                    context=self.request,
+                )
+            )
+
+        block_data = self.get_block_data(block_id=block_id)
+        if not block_data:
+            raise NotFound(
+                translate(
+                    _(
+                        "block_not_found",
+                        default='Block with id "{}" not found in this context.'.format(
+                            block_id
+                        ),
+                    ),
+                    context=self.request,
+                )
+            )
+        if block_data.get("@type", "") != "rssBlock":
+            raise BadRequest(
+                translate(
+                    _(
+                        "wrong_block_type",
+                        default='Block with id "{}" is not an RSS block.'.format(
+                            block_id
+                        ),
+                    ),
+                    context=self.request,
+                )
+            )
+        return block_data
+
+    def get_block_data(self, block_id):
+        blocks = getattr(self.context, "blocks", {})
+        if not blocks:
+            return {}
+        if not isinstance(blocks, dict):
+            # plone < 6 support
+            blocks = json.loads(blocks)
+        rss_block = blocks.get(block_id, {})
+        if not rss_block:
+            # maybe is in some Block field
+            for schema in iterSchemata(self.context):
+                for name, field in getFields(schema).items():
+                    value = field.get(self.context)
+                    if not value:
+                        continue
+                    if not isinstance(value, dict):
+                        continue
+                    rss_block = value.get("blocks", {}).get(block_id, {})
+        return rss_block
+
+    def _getFeeds(self, feeds, limit=20):
+        """Return all feeds"""
+        data = []
+        for feed_data in feeds:
+            url = feed_data.get("url", "")
+            source = feed_data.get("source", "")
+            feed = FEED_DATA.get(url, None)
+            if feed is None:
+                # create it
+                feed = FEED_DATA[url] = RSSMixerFeed(
+                    url=url,
+                    source=source,
+                    timeout=100,
+                )
+            # if it's new, populate it, else try to see if it need to be updated
+            else:
+                # check if we need to update the source
+                if feed.source != source:
+                    feed.source = source
+            feed.update()
+            data.append(feed)
+        return self._sortedFeeds(feeds=data, limit=limit)
+
+    def _sortedFeeds(self, feeds, limit):
+        """Sort feed items by date"""
+
+        itemsWithDate = []
+        itemsWithoutDate = []
+        for feed in feeds:
+            for item in feed.items:
+                if "date" in item:
+                    itemsWithDate.append(item)
+                else:
+                    itemsWithoutDate.append(item)
+        sortedItems = sorted(itemsWithDate, key=lambda d: d["date"], reverse=True)
+        total = sortedItems + itemsWithoutDate
+
+        # fix date format
+        return total[:limit]
+
+
+@implementer(IRSSMixerFeed)
+class RSSMixerFeed(object):
+    """An RSS feed."""
+
+    FAILURE_DELAY = 10
+
+    def __init__(self, url, source, timeout):
+        self.url = url
+        self.timeout = timeout
+        self.source = source
+        self._items = []
+        self._title = ""
+        self._siteurl = ""
+        self._loaded = False  # is the feed loaded
+        self._failed = False  # does it fail at the last update?
+        self._last_update_time_in_minutes = 0  # when was the feed updated?
+        self._last_update_time = None  # time as DateTime or Nonw
+
+    @property
+    def last_update_time_in_minutes(self):
+        """Return the time the last update was done in minutes."""
+        return self._last_update_time_in_minutes
+
+    @property
+    def last_update_time(self):
+        """Return the time the last update was done in minutes."""
+        return self._last_update_time
+
+    @property
+    def update_failed(self):
+        return self._failed
+
+    @property
+    def ok(self):
+        return not self._failed and self._loaded
+
+    @property
+    def loaded(self):
+        """Return whether this feed is loaded or not."""
+        return self._loaded
+
+    @property
+    def needs_update(self):
+        """Check if this feed needs updating."""
+        now = time() / 6
+        return (self.last_update_time_in_minutes + self.timeout) < now
+
+    def update(self):
+        """Update this feed."""
+        now = time() / 60  # time in minutes
+        # check for failure and retry
+        if self.update_failed:
+            if (self.last_update_time_in_minutes + self.FAILURE_DELAY) < now:
+                return self._retrieveFeed()
+            else:
+                return False
+
+        # check for regular update
+        if self.needs_update:
+            return self._retrieveFeed()
+
+        return self.ok
+
+    def _getFeedFromUrl(self, url):
+        """
+        Use urllib to retrieve an rss feed.
+        In this way, we can manage timeouts.
+        """
+        url = uid_to_url(url)
+        headers = {}
+        if REQUESTS_USER_AGENT:
+            headers["User-Agent"] = REQUESTS_USER_AGENT
+        try:
+            if RSSMIXER_HTTP_PROXY:
+                url = f"{RSSMIXER_HTTP_PROXY}/{url}"
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=REQUESTS_TIMEOUT,
+            )
+        except (Timeout, RequestException) as e:
+            logger.warning("exception %s during %s request", e, url)
+            return None
+        if response.status_code != 200:
+            message = response.text or response.reason
+            logger.error(
+                "Unable to retrieve feed from {url}: {message}".format(
+                    url=url, message=message
+                )
+            )
+            return None
+        return feedparser.parse(response.content)
+
+    def _retrieveFeed(self):
+        """Do the actual work and try to retrieve the feed."""
+        url = self.url
+        if not url:
+            self._loaded = True
+            self._failed = True  # no url set means failed
+            # no url set, although that actually should not really happen
+            return False
+        self._last_update_time_in_minutes = time() / 60
+        self._last_update_time = DateTime()
+        parsed_feed = self._getFeedFromUrl(url)
+        if not parsed_feed:
+            self._loaded = True  # we tried at least but have a failed load
+            self._failed = True
+            return False
+        if parsed_feed.bozo == 1 and not isinstance(
+            parsed_feed.get("bozo_exception"),
+            ACCEPTED_FEEDPARSER_EXCEPTIONS,
+        ):
+            self._loaded = True  # we tried at least but have a failed load
+            self._failed = True
+            return False
+        self._title = parsed_feed.feed.title
+        self._siteurl = parsed_feed.feed.link
+        self._items = []
+
+        for item in parsed_feed["items"]:
+            itemdict = {
+                "title": item.title,
+                "url": item.get("link", ""),
+                "contentSnippet": item.get("description", ""),
+                "source": getattr(self, "source", ""),
+            }
+
+            date = self.get_item_date(item=item)
+            if date:
+                itemdict["date"] = date
+
+            image = self.get_item_image(item=item)
+            if image:
+                # format needed in blocks to keep compatibility
+                itemdict["enclosure"] = image
+
+            categories = self.get_item_categories(item=item)
+            if categories:
+                itemdict["categories"] = categories
+
+            self._items.append(itemdict)
+        self._loaded = True
+        self._failed = False
+        return True
+
+    def get_item_categories(self, item):
+        categories = []
+        if getattr(item, "tags", None):
+            for tag in item["tags"]:
+                term = getattr(tag, "term", None)
+                if term:
+                    categories.append(term)
+        return categories
+
+    def get_item_date(self, item):
+        if getattr(item, "updated", None):
+            try:
+                return json_compatible(DateTime(item.updated))
+            except SyntaxError:
+                return json_compatible(item.updated)
+        elif getattr(item, "published", None):
+            try:
+                return json_compatible(DateTime(item.published))
+            except SyntaxError:
+                return json_compatible(item.published)
+
+        return ""
+
+    def get_item_image(self, item):
+        image = ""
+        if item.get("media_thumbnail", []):
+            image = item["media_thumbnail"][0].get("url", "")
+        elif item.get("media_content", []):
+            images = [
+                x.get("url", "")
+                for x in item.media_content
+                if x.get("medium", "") == "image"
+            ]
+            if images:
+                image = images[0]
+        elif item.get("links", []):
+            images = [
+                x.get("url", "")
+                for x in item.links
+                if x.get("rel", "") == "enclosure" and "image" in x.get("type", "")
+            ]
+            if images:
+                image = images[0]
+        if not image:
+            return {}
+        return {"url": image}
+
+    @property
+    def items(self):
+        return self._items
+
+    # convenience methods for displaying
+
+    @property
+    def feed_link(self):
+        """Return rss url of feed for tile."""
+        return self.url.replace("http://", "feed://")
+
+    @property
+    def title(self):
+        """Return title of feed for tile."""
+        return self._title
+
+    @property
+    def siteurl(self):
+        """Return the link to the site the RSS feed points to."""
+        return self._siteurl

redturtle/rssservice/testing.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE
+from plone.app.robotframework.testing import REMOTE_LIBRARY_BUNDLE_FIXTURE
+from plone.app.testing import applyProfile
+from plone.app.testing import FunctionalTesting
+from plone.app.testing import IntegrationTesting
+from plone.app.testing import PloneSandboxLayer
+from plone.restapi.testing import PloneRestApiDXLayer
+from plone.testing import z2
+
+import plone.restapi
+import redturtle.rssservice
+
+
+class RedTurtleRSSServiceLayer(PloneSandboxLayer):
+    defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,)
+
+    def setUpZope(self, app, configurationContext):
+        # Load any other ZCML that is required for your tests.
+        # The z3c.autoinclude feature is disabled in the Plone fixture base
+        # layer.
+        self.loadZCML(package=plone.restapi)
+        self.loadZCML(package=redturtle.rssservice)
+
+    def setUpPloneSite(self, portal):
+        applyProfile(portal, "plone.restapi:default")
+
+
+REDTURTLE_RSSSERVICE_FIXTURE = RedTurtleRSSServiceLayer()
+
+
+REDTURTLE_RSSSERVICE_INTEGRATION_TESTING = IntegrationTesting(
+    bases=(REDTURTLE_RSSSERVICE_FIXTURE,),
+    name="RedTurtleRSSServiceLayer:IntegrationTesting",
+)
+
+
+REDTURTLE_RSSSERVICE_FUNCTIONAL_TESTING = FunctionalTesting(
+    bases=(REDTURTLE_RSSSERVICE_FIXTURE,),
+    name="RedTurtleRSSServiceLayer:FunctionalTesting",
+)
+
+
+REDTURTLE_RSSSERVICE_ACCEPTANCE_TESTING = FunctionalTesting(
+    bases=(
+        REDTURTLE_RSSSERVICE_FIXTURE,
+        REMOTE_LIBRARY_BUNDLE_FIXTURE,
+        z2.ZSERVER_FIXTURE,
+    ),
+    name="RedTurtleRSSServiceLayer:AcceptanceTesting",
+)
+
+
+class RedTurtleRSSServiceRestApiLayer(PloneRestApiDXLayer):
+    defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,)
+
+    def setUpZope(self, app, configurationContext):
+        super(RedTurtleRSSServiceRestApiLayer, self).setUpZope(
+            app, configurationContext
+        )
+
+        self.loadZCML(package=plone.restapi)
+        self.loadZCML(package=redturtle.rssservice)
+
+    def setUpPloneSite(self, portal):
+        applyProfile(portal, "plone.restapi:default")
+
+
+REDTURTLE_RSSSERVICE_API_FIXTURE = RedTurtleRSSServiceRestApiLayer()
+REDTURTLE_RSSSERVICE_API_INTEGRATION_TESTING = IntegrationTesting(
+    bases=(REDTURTLE_RSSSERVICE_API_FIXTURE,),
+    name="RedTurtleRSSServiceRestApiLayer:Integration",
+)
+
+REDTURTLE_RSSSERVICE_API_FUNCTIONAL_TESTING = FunctionalTesting(
+    bases=(REDTURTLE_RSSSERVICE_API_FIXTURE, z2.ZSERVER_FIXTURE),
+    name="RedTurtleRSSServiceRestApiLayer:Functional",
+)

redturtle/rssservice/tests/test_proxycache.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import importlib
+import os
+import shutil
+import tempfile
+import unittest
+from unittest import mock
+
+main = importlib.import_module("redturtle.rssservice.proxycacheserver.main")
+
+
+class ProxyCacheServerTest(unittest.TestCase):
+    def setUp(self):
+        self.test_dir = tempfile.mkdtemp()
+        main.ACTIVE_REFRESH_THREADS.clear()
+        main.LAST_ACCESS_TIMES.clear()
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir, ignore_errors=True)
+        main.ACTIVE_REFRESH_THREADS.clear()
+        main.LAST_ACCESS_TIMES.clear()
+
+    def test_cache_path(self):
+        url = "https://example.com/rss.xml"
+        path = main.cache_path(url, self.test_dir)
+        self.assertTrue(path.startswith(self.test_dir))
+        self.assertTrue(path.endswith(".json"))
+
+    def test_safe_atomic_write_and_load_json(self):
+        file_path = os.path.join(self.test_dir, "test.json")
+        data = {"url": "https://example.com/rss", "status_code": 200, "body": "OK"}
+        main.safe_atomic_write_json(file_path, data)
+
+        loaded = main.load_json(file_path)
+        self.assertEqual(loaded, data)
+        # Ensure no temporary file left
+        files = os.listdir(self.test_dir)
+        self.assertEqual(files, ["test.json"])
+
+    def test_is_valid_url(self):
+        self.assertTrue(main.is_valid_url("https://example.com/feed"))
+        self.assertTrue(main.is_valid_url("http://news.google.com/rss"))
+        self.assertFalse(main.is_valid_url("ftp://example.com"))
+        self.assertFalse(main.is_valid_url("http://localhost:8080"))
+        self.assertFalse(main.is_valid_url("http://127.0.0.1/admin"))
+        self.assertFalse(main.is_valid_url("http://169.254.169.254/latest/meta-data/"))
+
+    def test_ensure_refresh_thread_deduplication(self):
+        url = "https://example.com/feed"
+        with mock.patch("threading.Thread") as mock_thread_cls:
+            mock_thread_instance = mock.MagicMock()
+            mock_thread_cls.return_value = mock_thread_instance
+
+            main.ensure_refresh_thread(url, self.test_dir, ttl=60)
+            self.assertIn(url, main.ACTIVE_REFRESH_THREADS)
+            self.assertEqual(mock_thread_cls.call_count, 1)
+
+            # Second call for the same URL should be deduplicated
+            main.ensure_refresh_thread(url, self.test_dir, ttl=60)
+            self.assertEqual(mock_thread_cls.call_count, 1)
+
+    @mock.patch("requests.get")
+    def test_fetch_and_cache_success(self, mock_requests_get):
+        mock_response = mock.MagicMock()
+        mock_response.status_code = 200
+        mock_response.headers = {"Content-Type": "application/rss+xml"}
+        mock_response.text = "<rss><channel><title>Test Feed</title></channel></rss>"
+        mock_requests_get.return_value = mock_response
+
+        url = "https://example.com/rss"
+        result = main.fetch_and_cache(url, self.test_dir)
+
+        self.assertEqual(result["status_code"], 200)
+        self.assertEqual(result["body"], mock_response.text)
+
+        # Check cached file
+        cache_file = main.cache_path(url, self.test_dir)
+        self.assertTrue(os.path.exists(cache_file))
+        loaded = main.load_json(cache_file)
+        self.assertEqual(loaded["status_code"], 200)
+
+    @mock.patch("requests.get")
+    def test_fetch_and_cache_connection_error_not_persisted(self, mock_requests_get):
+        mock_requests_get.side_effect = Exception("Connection refused")
+
+        url = "https://example.com/failing_rss"
+        result = main.fetch_and_cache(url, self.test_dir)
+
+        self.assertEqual(result["status_code"], 502)
+        cache_file = main.cache_path(url, self.test_dir)
+        self.assertFalse(os.path.exists(cache_file))