webtoolkit 0.1.72__tar.gz → 0.1.74__tar.gz

{webtoolkit-0.1.72 → webtoolkit-0.1.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.1.72
+Version: 0.1.74
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny

{webtoolkit-0.1.72 → webtoolkit-0.1.74}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.1.72"
+version = "0.1.74"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.1.72 → webtoolkit-0.1.74}/webtoolkit/handlers/handlerchannelyoutube.py

@@ -209,7 +209,7 @@ class YouTubeChannelHandler(DefaultCompoundChannelHandler):
             if self.code:
                 feed = self.code2feed(self.code)
                 url = self.build_http_url(feed)
-                self.channel_sources_urls[url.url] = url
+                self.all_urls_to_fetch[url.url] = url
                 responses.append(url.get_response())
 
         return responses

{webtoolkit-0.1.72 → webtoolkit-0.1.74}/webtoolkit/handlers/handlervideoyoutube.py

@@ -5,6 +5,7 @@ from ..utils.dateutils import DateUtils
 from ..urllocation import UrlLocation
 from ..pages import ReturnDislikeJson
 from ..webtools import WebLogger
+from ..statuses import HTTP_STATUS_NOT_FOUND
 from .defaulturlhandler import DefaultCompoundChannelHandler
 
 
@@ -186,3 +187,15 @@ class YouTubeVideoHandler(DefaultCompoundChannelHandler):
         locale of platform, not contents
         """
         return None
+
+    def get_responses_implementation(self, urls_to_fetch):
+        """
+         - YouTube returns 200 for unavailable videos.
+         - we check if there is schema information about video. If none, it is error
+        """
+        responses = super().get_responses_implementation(urls_to_fetch)
+        for response in responses:
+            if response.is_html():
+                if response.get_text().find("schema.org") == -1:
+                    response.status_code = HTTP_STATUS_NOT_FOUND
+        return responses