tcd2 3.2.2.post1__py3-none-any.whl → 3.2.2.post2__py3-none-any.whl

tcd/__init__.py

@@ -11,7 +11,7 @@ from .logger import Logger, Log
 from .settings import Settings
 
 __name__: str = 'tcd'
-__version__: str = '3.2.2.post1'
+__version__: str = '3.2.2.post2'
 __all__: List[Callable] = [Arguments, Settings, Downloader, Logger, Log]
 
 

tcd/comments.py

@@ -0,0 +1,13 @@
+from typing import Iterable
+
+from twitch.helix import Video
+from twitch.v5 import Comment
+
+from .graphql_comments import GraphQLComments
+
+
+def get_comments(video: Video) -> Iterable[Comment]:
+    """
+    Return an iterator of comments for a video using Twitch GraphQL.
+    """
+    return GraphQLComments(video.id)

tcd/downloader.py

@@ -9,11 +9,12 @@ import dateutil
 from twitch import Helix
 from twitch.helix import Video
 
-from .arguments import Arguments
-from .formatter import Formatter
-from .logger import Logger, Log
-from .pipe import Pipe
-from .settings import Settings
+from .arguments import Arguments
+from .comments import get_comments
+from .formatter import Formatter
+from .logger import Logger, Log
+from .pipe import Pipe
+from .settings import Settings
 
 
 class Downloader:
@@ -97,7 +98,7 @@ class Downloader:
                 'comments': []
             }
 
-            for comment in video.comments:
+            for comment in get_comments(video):
 
                 # Skip unspecified users if a list is provided.
                 if Arguments().users and comment.commenter.name.lower() not in Arguments().users:

tcd/formats/custom.py

@@ -1,10 +1,10 @@
-from typing import Generator, Tuple
-
-from twitch.helix import Video
-from twitch.v5 import Comments, Comment
-
-from tcd.formats.format import Format
-from tcd.pipe import Pipe
+from typing import Generator, Tuple, Iterable
+
+from twitch.helix import Video
+from twitch.v5 import Comment
+
+from tcd.formats.format import Format
+from tcd.pipe import Pipe
 
 
 class Custom(Format):
@@ -12,19 +12,19 @@ class Custom(Format):
     def __init__(self, video: Video, format_name: str):
         super().__init__(video, format_name)
 
-    def use(self) -> Tuple[Generator[Tuple[str, Comment], None, None], str]:
-        """
-        Use this format
-        :return: tuple(formatted comment, comment), output format
-        """
-        # Format comments
-        comments = self.comment_generator(self.video.comments)
+    def use(self) -> Tuple[Generator[Tuple[str, Comment], None, None], str]:
+        """
+        Use this format
+        :return: tuple(formatted comment, comment), output format
+        """
+        # Format comments
+        comments = self.comment_generator(self.comments())
 
         # Format output
         output: str = Pipe(self.format_dictionary['output']).output(self.video.data)
 
         return comments, output
 
-    def comment_generator(self, comments: Comments) -> Generator[Tuple[str, Comment], None, None]:
-        for comment in comments:
-            yield Pipe(self.format_dictionary['comments']).comment(comment.data), comment
+    def comment_generator(self, comments: Iterable[Comment]) -> Generator[Tuple[str, Comment], None, None]:
+        for comment in comments:
+            yield Pipe(self.format_dictionary['comments']).comment(comment.data), comment

tcd/formats/format.py

@@ -1,11 +1,21 @@
-from twitch.helix import Video
+from typing import Iterable
+
+from twitch.helix import Video
+from twitch.v5 import Comment
+
+from tcd.comments import get_comments
+from tcd.settings import Settings
+
+
+class Format:
 
-from tcd.settings import Settings
-
-
-class Format:
-
-    def __init__(self, video: Video, format_name: str):
-        self.video: Video = video
-        self.format_name: str = format_name
-        self.format_dictionary: dict = Settings().config['formats'][format_name]
+    def __init__(self, video: Video, format_name: str):
+        self.video: Video = video
+        self.format_name: str = format_name
+        self.format_dictionary: dict = Settings().config['formats'][format_name]
+
+    def comments(self) -> Iterable[Comment]:
+        """
+        Return an iterator for video comments using GraphQL.
+        """
+        return get_comments(self.video)

tcd/formats/srt.py

@@ -1,12 +1,12 @@
-import datetime
-from typing import Tuple, Generator
-
-from twitch.helix import Video
-from twitch.v5 import Comment, Comments
-
-from tcd.formats.format import Format
-from tcd.pipe import Pipe
-from tcd.safedict import SafeDict
+import datetime
+from typing import Tuple, Generator, Iterable
+
+from twitch.helix import Video
+from twitch.v5 import Comment
+
+from tcd.formats.format import Format
+from tcd.pipe import Pipe
+from tcd.safedict import SafeDict
 
 
 class SRT(Format):
@@ -18,12 +18,12 @@ class SRT(Format):
         """
         super().__init__(video, format_name='srt')
 
-    def use(self) -> Tuple[Generator[Tuple[str, Comment], None, None], str]:
-        """
-        Use SRT format
-        :return: Comment generator and output string
-        """
-        return self.subtitles(self.video.comments), Pipe(self.format_dictionary['output']).output(self.video.data)
+    def use(self) -> Tuple[Generator[Tuple[str, Comment], None, None], str]:
+        """
+        Use SRT format
+        :return: Comment generator and output string
+        """
+        return self.subtitles(self.comments()), Pipe(self.format_dictionary['output']).output(self.video.data)
 
     @staticmethod
     def format_timestamp(time: datetime.timedelta) -> str:
@@ -45,12 +45,12 @@ class SRT(Format):
 
         return f'{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d},{milliseconds:03d}'
 
-    def subtitles(self, comments: Comments) -> Generator[Tuple[str, Comment], None, None]:
-        """
-        Subtitle generator
-        :param comments: Comments to turn into subtitles
-        :return: Generator with subtitles and subtitle data
-        """
+    def subtitles(self, comments: Iterable[Comment]) -> Generator[Tuple[str, Comment], None, None]:
+        """
+        Subtitle generator
+        :param comments: Comments to turn into subtitles
+        :return: Generator with subtitles and subtitle data
+        """
         for index, comment in enumerate(comments):
             # Stat and stop timestamps. Add a millisecond for timedelta to include millisecond digits
             start = datetime.timedelta(seconds=comment.content_offset_seconds)

tcd/formats/ssa.py

@@ -1,9 +1,9 @@
-import datetime
-from itertools import chain
-from typing import Tuple, Generator, List, Optional
+import datetime
+from itertools import chain
+from typing import Tuple, Generator, List, Optional, Iterable
 
-from twitch.helix import Video
-from twitch.v5 import Comment, Comments
+from twitch.helix import Video
+from twitch.v5 import Comment
 
 from tcd.formats.format import Format
 from tcd.pipe import Pipe
@@ -32,8 +32,8 @@ class SSA(Format):
         Line generator
         :return:
         """
-        for line in chain(self.prefix(), self.dialogues(self.video.comments)):
-            yield line
+        for line in chain(self.prefix(), self.dialogues(self.comments())):
+            yield line
 
     @staticmethod
     def format_timestamp(time: datetime.timedelta) -> str:
@@ -55,7 +55,7 @@ class SSA(Format):
 
         return f'{int(hours):01d}:{int(minutes):02d}:{int(seconds):02d}.{centiseconds:02d}'
 
-    def dialogues(self, comments: Comments) -> Generator[Tuple[str, Comments], None, None]:
+    def dialogues(self, comments: Iterable[Comment]) -> Generator[Tuple[str, Comment], None, None]:
         """
         Format comments as SSA dialogues
         :param comments: Comment to format

tcd/graphql_comments.py

@@ -0,0 +1,211 @@
+"""
+GraphQL-based comments module for Twitch VODs.
+This replaces the deprecated V5 API with Twitch's GraphQL API.
+"""
+from typing import Union, Generator, Dict, Any, List
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+import twitch.v5 as v5
+
+
+class GraphQLComments:
+    """
+    Fetches VOD comments using Twitch's GraphQL API.
+    """
+
+    GRAPHQL_ENDPOINT = 'https://gql.twitch.tv/gql'
+    # Public client ID used by Twitch web
+    CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
+    # SHA256 hash for the VideoCommentsByOffsetOrCursor persisted query
+    QUERY_HASH = 'b70a3591ff0f4e0313d126c6a1502d79a1c02baebb288227c582044aa76adf6a'
+
+    def __init__(self, video_id: Union[str, int], num_threads: int = 4):
+        self._video_id: str = str(video_id)
+        self._num_threads: int = num_threads
+        self._session = requests.Session()
+        self._session_lock = threading.Lock()
+
+    def _graphql_request(self, variables: Dict[str, Any]) -> Dict[str, Any]:
+        query = [{
+            "operationName": "VideoCommentsByOffsetOrCursor",
+            "variables": variables,
+            "extensions": {
+                "persistedQuery": {
+                    "version": 1,
+                    "sha256Hash": self.QUERY_HASH
+                }
+            }
+        }]
+
+        headers = {
+            'Client-Id': self.CLIENT_ID,
+            'Content-Type': 'application/json'
+        }
+
+        with self._session_lock:
+            response = self._session.post(self.GRAPHQL_ENDPOINT, json=query, headers=headers)
+        response.raise_for_status()
+        return response.json()
+
+    def _fetch_chunk(self, start_offset: float) -> List[Dict[str, Any]]:
+        comments: List[Dict[str, Any]] = []
+        variables = {
+            "videoID": self._video_id,
+            "contentOffsetSeconds": start_offset
+        }
+
+        try:
+            response_data = self._graphql_request(variables)
+            if not response_data:
+                return comments
+
+            data = response_data[0].get('data', {})
+            video_data = data.get('video')
+            if not video_data:
+                return comments
+
+            comments_data = video_data.get('comments')
+            if not comments_data:
+                return comments
+
+            edges = comments_data.get('edges', [])
+            for edge in edges:
+                comments.append(self._convert_graphql_comment_to_v5_format(edge))
+        except Exception as exc:
+            import sys
+            print(f"Error fetching chunk at offset {start_offset}: {exc}", file=sys.stderr)
+
+        return comments
+
+    def _convert_graphql_comment_to_v5_format(self, comment_node: Dict[str, Any]) -> Dict[str, Any]:
+        comment = comment_node.get('node', {})
+        commenter = comment.get('commenter', {})
+        message = comment.get('message', {})
+
+        fragments = []
+        emoticons = []
+        full_message_text = []
+
+        for fragment in message.get('fragments', []):
+            fragment_text = fragment.get('text', '')
+            full_message_text.append(fragment_text)
+            fragment_data = {'text': fragment_text}
+
+            emote = fragment.get('emote')
+            if emote:
+                emoticon_data = {
+                    '_id': emote.get('emoteID'),
+                    'begin': len(''.join([f['text'] for f in fragments])),
+                    'end': None,
+                    'emoticon_id': emote.get('emoteID'),
+                    'emoticon_set_id': None
+                }
+                emoticon_data['end'] = emoticon_data['begin'] + len(fragment_data['text'])
+                emoticons.append(emoticon_data)
+                fragment_data['emoticon'] = emoticon_data
+
+            fragments.append(fragment_data)
+
+        message_body = ''.join(full_message_text)
+
+        user_badges = []
+        for badge in message.get('userBadges', []):
+            user_badges.append({
+                '_id': badge.get('setID'),
+                'version': badge.get('version')
+            })
+
+        v5_comment = {
+            '_id': comment.get('id'),
+            'created_at': comment.get('createdAt'),
+            'updated_at': comment.get('updatedAt', comment.get('createdAt')),
+            'channel_id': None,
+            'content_type': 'video',
+            'content_id': self._video_id,
+            'content_offset_seconds': float(comment.get('contentOffsetSeconds', 0)),
+            'commenter': {
+                'display_name': commenter.get('displayName'),
+                '_id': commenter.get('id'),
+                'name': commenter.get('login'),
+                'type': None,
+                'bio': None,
+                'created_at': None,
+                'updated_at': None,
+                'logo': None
+            } if commenter else None,
+            'source': 'chat',
+            'state': 'published',
+            'message': {
+                'body': message_body,
+                'emoticons': emoticons,
+                'fragments': fragments,
+                'is_action': False,
+                'user_badges': user_badges,
+                'user_color': message.get('userColor')
+            },
+            'more_replies': False
+        }
+
+        return v5_comment
+
+    def __iter__(self) -> Generator['v5.Comment', None, None]:
+        initial_chunk = self._fetch_chunk(0.0)
+        if not initial_chunk:
+            return
+
+        chunk_interval = 3600
+        all_comments: List[Dict[str, Any]] = []
+        current_offset = 0.0
+        max_offset = 0.0
+
+        while True:
+            chunk_offsets = []
+            for i in range(self._num_threads):
+                offset = current_offset + (i * chunk_interval)
+                chunk_offsets.append(offset)
+
+            chunks_data = []
+            with ThreadPoolExecutor(max_workers=self._num_threads) as executor:
+                future_to_offset = {
+                    executor.submit(self._fetch_chunk, offset): offset
+                    for offset in chunk_offsets
+                }
+
+                for future in as_completed(future_to_offset):
+                    offset = future_to_offset[future]
+                    try:
+                        chunk_comments = future.result()
+                        if chunk_comments:
+                            chunks_data.append((offset, chunk_comments))
+                    except Exception as exc:
+                        import sys
+                        print(f"Error downloading chunk at {offset}: {exc}", file=sys.stderr)
+
+            chunks_data.sort(key=lambda x: x[0])
+
+            got_new_data = False
+            for _, chunk_comments in chunks_data:
+                for comment_data in chunk_comments:
+                    comment_offset = float(comment_data.get('content_offset_seconds', 0))
+                    if comment_offset <= max_offset:
+                        continue
+
+                    all_comments.append(comment_data)
+                    max_offset = max(max_offset, comment_offset)
+                    got_new_data = True
+
+            if not got_new_data:
+                break
+
+            current_offset = max_offset
+
+        all_comments.sort(key=lambda x: x.get('content_offset_seconds', 0))
+
+        from twitch.api import API
+        minimal_api = API(base_url='https://api.twitch.tv/v5/',
+                         client_id=self.CLIENT_ID)
+
+        for comment_data in all_comments:
+            yield v5.Comment(api=minimal_api, data=comment_data)

{tcd2-3.2.2.post1.dist-info → tcd2-3.2.2.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tcd2
-Version: 3.2.2.post1
+Version: 3.2.2.post2
 Summary: Twitch Chat Downloader (unofficial fork)
 Author: imeloben23
 Maintainer: imeloben23

tcd2-3.2.2.post2.dist-info/RECORD

@@ -0,0 +1,24 @@
+tcd/__init__.py,sha256=UR76UQ1ofdIZtdYsWegE5FcU9e2NYeTd9HvUHEBsXR4,4944
+tcd/__main__.py,sha256=r-uXw3zgKZ6lSdY61Q36HFSMTjzc6shN_W6F2x7gkDc,88
+tcd/arguments.py,sha256=Vl9PnEPelrksAV8BNpYlHPqm0JE51qgR7_9yEZpaLmU,3283
+tcd/comments.py,sha256=UCBJMs1nfedH3hCAn24r9L0Zi35VEMDuFG9tvOtIXA0,314
+tcd/downloader.py,sha256=_RHe0EZy2USklXFj-oIbIpEDmBR6SuIdVFThAQlfv2U,9558
+tcd/formatter.py,sha256=FilDWcys7nD-5b96eTYxMBA3TeHujOrnv8Is-nYn8GY,903
+tcd/graphql_comments.py,sha256=vlMDyrzGH5tr2Xp5x95dq9oVWda1XgY88TfyZTK_JAQ,7476
+tcd/logger.py,sha256=nnIrv0_-1ACy-32oel0PsfhMxc0s9H7Qu80SeyAN8rU,3362
+tcd/pipe.py,sha256=U3VuSES3r_wG0zsq0eY_S2QJ4RoumKrYqpv8mI4_FLA,10567
+tcd/safedict.py,sha256=_2EM-9LGA3JrSvQb7Z75aNeBDFQBYbsBa3TREaSNvOU,263
+tcd/settings.py,sha256=_yxe1eC7jltc5dRRCZkx7ImyG5D_tWnnJnRCC58zsBY,4091
+tcd/settings.reference.json,sha256=zQS5Bb7ZlaZSOMbLBG00IZvKNRXkQM6rQ_4vLUlnmGY,3230
+tcd/singleton.py,sha256=PGSVYql6sCbrTsZVCZOTUufndCBfqRy0-6OQCaXIbII,371
+tcd/formats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tcd/formats/custom.py,sha256=Jr2tVZgBJo_nPxY7-PhJLIDJeuZ0H_zMCY5phMqtgmY,964
+tcd/formats/format.py,sha256=LBgBKwQdcozvsgwfxFRoACVsZDvyi4C2IJK-cIkurEM,567
+tcd/formats/srt.py,sha256=RQL17u0nvV9pIMzbS2miFDvVDLfSCSjrpYLJUiSbXRs,2643
+tcd/formats/ssa.py,sha256=uiHfJ50vJsewJYYztsdJIvLimoCLUUrVNYnMuwtrXNg,7064
+tcd2-3.2.2.post2.dist-info/licenses/LICENSE,sha256=Ch-ZtxzUuhHcxKI4Yq4L_en60uZKBBpMamPeImjzpQM,1093
+tcd2-3.2.2.post2.dist-info/METADATA,sha256=AxKDvI-nuUriIj7mOPl6BdF2kcv0jG9MtGmmMiH-704,1622
+tcd2-3.2.2.post2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tcd2-3.2.2.post2.dist-info/entry_points.txt,sha256=47ljm618xZuZ07uL5o9HYnJ8ZN8m0kFptoMIjzYy8cI,33
+tcd2-3.2.2.post2.dist-info/top_level.txt,sha256=HF0kJO8nLMAOZIROY5suwglVvTWTXhkycR2TjicOJSA,4
+tcd2-3.2.2.post2.dist-info/RECORD,,

tcd2-3.2.2.post1.dist-info/RECORD

@@ -1,22 +0,0 @@
-tcd/__init__.py,sha256=56y8gP6QGKvOpl2Tl6UB_OW8ewlq4y9PEtwoiNmzvfI,4944
-tcd/__main__.py,sha256=r-uXw3zgKZ6lSdY61Q36HFSMTjzc6shN_W6F2x7gkDc,88
-tcd/arguments.py,sha256=Vl9PnEPelrksAV8BNpYlHPqm0JE51qgR7_9yEZpaLmU,3283
-tcd/downloader.py,sha256=-EU4Nh_nvDARXG3Lc5pbDI1Dm4RHUS4XXtIStizzf8k,9524
-tcd/formatter.py,sha256=FilDWcys7nD-5b96eTYxMBA3TeHujOrnv8Is-nYn8GY,903
-tcd/logger.py,sha256=nnIrv0_-1ACy-32oel0PsfhMxc0s9H7Qu80SeyAN8rU,3362
-tcd/pipe.py,sha256=U3VuSES3r_wG0zsq0eY_S2QJ4RoumKrYqpv8mI4_FLA,10567
-tcd/safedict.py,sha256=_2EM-9LGA3JrSvQb7Z75aNeBDFQBYbsBa3TREaSNvOU,263
-tcd/settings.py,sha256=_yxe1eC7jltc5dRRCZkx7ImyG5D_tWnnJnRCC58zsBY,4091
-tcd/settings.reference.json,sha256=zQS5Bb7ZlaZSOMbLBG00IZvKNRXkQM6rQ_4vLUlnmGY,3230
-tcd/singleton.py,sha256=PGSVYql6sCbrTsZVCZOTUufndCBfqRy0-6OQCaXIbII,371
-tcd/formats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tcd/formats/custom.py,sha256=_s8vY1CuYPLSwbV4KMRph0QHTekNNmERPh5DgyE9MaI,976
-tcd/formats/format.py,sha256=BzUJluE-SUVQ5XGayVEsnOD14TKirb5YGo9nvS-md04,309
-tcd/formats/srt.py,sha256=W7sRVq4opVNhFvPy94w1lBK_QSJ1qFhreh-0devRPOE,2659
-tcd/formats/ssa.py,sha256=9uB_5IcUxjECs55iuTfKgGAzGDHMASIzl5HkRSeG-e8,7068
-tcd2-3.2.2.post1.dist-info/licenses/LICENSE,sha256=Ch-ZtxzUuhHcxKI4Yq4L_en60uZKBBpMamPeImjzpQM,1093
-tcd2-3.2.2.post1.dist-info/METADATA,sha256=R2KQ2VN8xOhxS-fQHaM9ZKCArKAspEh6L-6xF_Yb6fk,1622
-tcd2-3.2.2.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tcd2-3.2.2.post1.dist-info/entry_points.txt,sha256=47ljm618xZuZ07uL5o9HYnJ8ZN8m0kFptoMIjzYy8cI,33
-tcd2-3.2.2.post1.dist-info/top_level.txt,sha256=HF0kJO8nLMAOZIROY5suwglVvTWTXhkycR2TjicOJSA,4
-tcd2-3.2.2.post1.dist-info/RECORD,,