tcd2 3.2.2.post1__tar.gz

tcd2-3.2.2.post1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Petter Kraabøl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tcd2-3.2.2.post1/MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE
+include README.md
+recursive-include tcd *.json

tcd2-3.2.2.post1/PKG-INFO

@@ -0,0 +1,57 @@
+Metadata-Version: 2.4
+Name: tcd2
+Version: 3.2.2.post1
+Summary: Twitch Chat Downloader (unofficial fork)
+Author: imeloben23
+Maintainer: imeloben23
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/PetterKraabol/Twitch-Chat-Downloader
+Project-URL: Original Project, https://github.com/PetterKraabol/Twitch-Chat-Downloader
+Keywords: Twitch
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests==2.27.1
+Requires-Dist: twitch-python==0.0.20
+Requires-Dist: pytz==2022.1
+Requires-Dist: python-dateutil==2.8.2
+Dynamic: license-file
+
+# tcd2
+
+Unofficial fork of **Twitch Chat Downloader** (the `tcd` package) by Petter Kraabøl.
+This distribution keeps the CLI name `tcd` but publishes under a new name (`tcd2`) on PyPI.
+
+## Install
+
+```bash
+pip install tcd2
+```
+
+## Usage
+
+```bash
+tcd
+```
+
+```bash
+# Download chat from VODs by video id
+tcd --video 789654123,987456321 --format irc --output ~/Downloads
+```
+
+```bash
+# Download chat from the first 10 VODs from multiple streamers
+tcd --channel sodapoppin,nymn,lirik --first=10
+```
+
+## Credits
+
+Original project: https://github.com/PetterKraabol/Twitch-Chat-Downloader

tcd2-3.2.2.post1/README.md

@@ -0,0 +1,30 @@
+# tcd2
+
+Unofficial fork of **Twitch Chat Downloader** (the `tcd` package) by Petter Kraabøl.
+This distribution keeps the CLI name `tcd` but publishes under a new name (`tcd2`) on PyPI.
+
+## Install
+
+```bash
+pip install tcd2
+```
+
+## Usage
+
+```bash
+tcd
+```
+
+```bash
+# Download chat from VODs by video id
+tcd --video 789654123,987456321 --format irc --output ~/Downloads
+```
+
+```bash
+# Download chat from the first 10 VODs from multiple streamers
+tcd --channel sodapoppin,nymn,lirik --first=10
+```
+
+## Credits
+
+Original project: https://github.com/PetterKraabol/Twitch-Chat-Downloader

tcd2-3.2.2.post1/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tcd2"
+version = "3.2.2.post1"
+description = "Twitch Chat Downloader (unofficial fork)"
+readme = "README.md"
+requires-python = ">=3.8"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+  { name = "imeloben23" }
+]
+maintainers = [
+  { name = "imeloben23" }
+]
+keywords = ["Twitch"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: End Users/Desktop",
+  "Natural Language :: English",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12"
+]
+dependencies = [
+  "requests==2.27.1",
+  "twitch-python==0.0.20",
+  "pytz==2022.1",
+  "python-dateutil==2.8.2"
+]
+
+[project.urls]
+Homepage = "https://github.com/PetterKraabol/Twitch-Chat-Downloader"
+"Original Project" = "https://github.com/PetterKraabol/Twitch-Chat-Downloader"
+
+[project.scripts]
+tcd = "tcd:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["tcd*"]
+
+[tool.setuptools.package-data]
+tcd = ["settings.reference.json"]

tcd2-3.2.2.post1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tcd2-3.2.2.post1/tcd/__init__.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+from pathlib import Path
+from typing import List, Callable
+
+import requests
+
+from .arguments import Arguments
+from .downloader import Downloader
+from .logger import Logger, Log
+from .settings import Settings
+
+__name__: str = 'tcd'
+__version__: str = '3.2.2.post1'
+__all__: List[Callable] = [Arguments, Settings, Downloader, Logger, Log]
+
+
+def main():
+    # Arguments
+    parser = argparse.ArgumentParser(description=f'Twitch Chat Downloader {__version__}')
+    parser.add_argument('-v', f'--{Arguments.Name.VIDEO}', type=str, help='Video IDs separated by commas')
+    parser.add_argument('-c', f'--{Arguments.Name.CHANNEL}', type=str, help='Channel names separated by commas')
+    parser.add_argument('-u', f'--{Arguments.Name.USER}', type=str, help='Messages from users, separated by commas')
+    parser.add_argument(f'--{Arguments.Name.FIRST}', type=int, default=None, help='Download chat from the last n VODs (default: all)')
+    parser.add_argument(f'--{Arguments.Name.CLIENT_ID.replace("_", "-")}', type=str, help='Twitch client ID')
+    parser.add_argument(f'--{Arguments.Name.CLIENT_SECRET.replace("_", "-")}', type=str, help='Twitch client secret')
+    parser.add_argument(f'--{Arguments.Name.VERBOSE}', action='store_true', help='Verbose output')
+    parser.add_argument('-q', f'--{Arguments.Name.QUIET}', action='store_true')
+    parser.add_argument('-o', f'--{Arguments.Name.OUTPUT}', type=str, help='Output directory', default='./')
+    parser.add_argument('-f', f'--{Arguments.Name.FORMAT}', type=str, help='Message format', default='default')
+    parser.add_argument(f'--{Arguments.Name.TIMEZONE}', type=str, help='Timezone name')
+    parser.add_argument(f'--includes', type=str, help='Messages must include specified text')
+    parser.add_argument(f'--{Arguments.Name.INIT}', action='store_true', help='Script setup')
+    parser.add_argument(f'--{Arguments.Name.VERSION}', action='store_true', help='Settings version')
+    parser.add_argument(f'--{Arguments.Name.FORMATS}', action='store_true', help='List available formats')
+    parser.add_argument(f'--{Arguments.Name.PREVIEW}', action='store_true', help='Preview output')
+    parser.add_argument(f'--{Arguments.Name.SETTINGS}', action='store_true', help='Print settings file location')
+    parser.add_argument(f'--{Arguments.Name.SETTINGS_FILE.replace("_", "-")}', type=str,
+                        default=str(Path.home()) + '/.config/tcd/settings.json',
+                        help='Use a custom settings file')
+    parser.add_argument(f'--{Arguments.Name.DEBUG}', action='store_true', help='Print debug messages')
+    parser.add_argument(f'--{Arguments.Name.LOG}', action='store_true', help='Save log file')
+
+    Arguments(parser.parse_args().__dict__)
+    Settings(Arguments().settings_file,
+             reference_filepath=f'{os.path.dirname(os.path.abspath(__file__))}/settings.reference.json')
+
+    # Print version number
+    if Arguments().print_version:
+        Logger().log(f'Twitch Chat Downloader {__version__}', retain=False)
+        return
+
+    # Print settings file location
+    if Arguments().settings:
+        Logger().log(str(Settings().filepath))
+        return
+
+    # Client ID
+    Settings().config['client_id'] = Arguments().client_id or Settings().config.get('client_id', None) or input(
+        'Twitch client ID: ').strip()
+    Settings().config['client_secret'] = Arguments().client_secret or Settings().config.get('client_secret', None) or input(
+        'Twitch client secret: ').strip()
+    Settings().save()
+
+    Arguments().oauth_token = requests.post(f"https://id.twitch.tv/oauth2/token"
+                                            f"?client_id={Settings().config['client_id']}"
+                                            f"&client_secret={Settings().config['client_secret']}"
+                                            f"&grant_type=client_credentials").json()['access_token']
+
+    # List formats
+    if Arguments().print_formats:
+        for format_name in [f for f in Settings().config['formats'] if f not in ['all']]:
+            format_dictionary = Settings().config['formats'][format_name]
+            Logger().log(f'[{format_name}]', retain=False)
+
+            if 'comments' in format_dictionary:
+                print('comment: {}'.format(Settings().config['formats'][format_name]['comments']['format']))
+
+            if 'output' in format_dictionary:
+                print('output: {}'.format(Settings().config['formats'][format_name]['output']['format']))
+
+            Logger().log('\n', retain=False)
+        return
+
+    # Downloader
+    if Arguments().video_ids or Arguments().channels:
+
+        if Arguments().video_ids:
+            Downloader().videos(Arguments().video_ids)
+
+        if Arguments().channels:
+            Downloader().channels(Arguments().channels)
+
+        return
+
+    parser.print_help()

tcd2-3.2.2.post1/tcd/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from . import main
+
+if __name__ == "__main__":
+    main()

tcd2-3.2.2.post1/tcd/arguments.py

@@ -0,0 +1,84 @@
+from typing import Optional, Dict, Union, List
+
+from .singleton import Singleton
+
+
+class Arguments(metaclass=Singleton):
+    """
+    Arguments singleton
+    """
+
+    class Name:
+        SETTINGS_FILE: str = 'settings_file'
+        SETTINGS: str = 'settings'
+        INIT: str = 'init'
+        VERBOSE: str = 'verbose'
+        QUIET: str = 'quiet'
+        PREVIEW: str = 'preview'
+        FORMATS: str = 'formats'
+        VERSION: str = 'version'
+        OUTPUT: str = 'output'
+        CLIENT_ID: str = 'client_id'
+        CLIENT_SECRET: str = 'client_secret'
+        CHANNEL: str = 'channel'
+        USER: str = 'user'
+        INCLUDES: str = 'includes'
+        FIRST: str = 'first'
+        VIDEO: str = 'video'
+        FORMAT: str = 'format'
+        TIMEZONE: str = 'timezone'
+        DEBUG: str = 'debug'
+        LOG: str = 'log'
+
+    def __init__(self, arguments: Optional[Dict[str, Union[str, bool, int]]] = None):
+        """
+        Initialize arguments
+        :param arguments: Arguments from cli (Optional to call singleton instance without parameters)
+        """
+
+        if arguments is None:
+            print('Error: arguments were not provided')
+            exit()
+
+        # Required arguments and booleans
+        self.settings_file: str = arguments[Arguments.Name.SETTINGS_FILE]
+        self.settings: str = arguments[Arguments.Name.SETTINGS]
+        self.init: bool = arguments[Arguments.Name.INIT]
+        self.verbose: bool = arguments[Arguments.Name.VERBOSE]
+        self.debug: bool = arguments[Arguments.Name.DEBUG]
+        self.quiet: bool = arguments[Arguments.Name.QUIET]
+        self.preview: bool = arguments[Arguments.Name.PREVIEW]
+        self.print_formats: bool = arguments[Arguments.Name.FORMATS]
+        self.print_version: bool = arguments[Arguments.Name.VERSION]
+        self.output: str = arguments[Arguments.Name.OUTPUT]
+        self.log: bool = arguments[Arguments.Name.LOG]
+
+        # Optional or prompted arguments
+        self.client_id: Optional[str] = arguments[Arguments.Name.CLIENT_ID]
+        self.client_secret: Optional[str] = arguments[Arguments.Name.CLIENT_SECRET]
+        self.oauth_token: Optional[str] = None
+        self.first: Optional[int] = arguments[Arguments.Name.FIRST]
+        self.timezone: Optional[str] = arguments[Arguments.Name.TIMEZONE]
+        self.includes: Optional[str] = arguments[Arguments.Name.INCLUDES]
+
+        # Arguments that require some formatting
+        self.video_ids: List[int] = []
+        self.formats: List[str] = []
+        self.channels: List[str] = []
+        self.users: List[str] = []
+
+        # Videos
+        if arguments[Arguments.Name.VIDEO]:
+            self.video_ids = [int(video_id) for video_id in arguments[Arguments.Name.VIDEO].lower().split(',')]
+
+        # Formats
+        if arguments[Arguments.Name.FORMAT]:
+            self.formats: Optional[List[str]] = arguments[Arguments.Name.FORMAT].lower().split(',')
+
+        # Channels
+        if arguments[Arguments.Name.CHANNEL]:
+            self.channels = arguments[Arguments.Name.CHANNEL].lower().split(',')
+
+        # Users
+        if arguments[Arguments.Name.USER]:
+            self.users = arguments[Arguments.Name.USER].lower().split(',')

tcd2-3.2.2.post1/tcd/downloader.py

@@ -0,0 +1,229 @@
+import datetime
+import json
+import os
+import re
+import sys
+from typing import List
+
+import dateutil
+from twitch import Helix
+from twitch.helix import Video
+
+from .arguments import Arguments
+from .formatter import Formatter
+from .logger import Logger, Log
+from .pipe import Pipe
+from .settings import Settings
+
+
+class Downloader:
+
+    def __init__(self):
+        self.helix_api = Helix(client_id=Settings().config['client_id'], bearer_token=Arguments().oauth_token, use_cache=True)
+
+        self.formats: List[str] = []
+        self.whitelist: List[str] = []
+        self.blacklist: List[str] = []
+
+        # Populate format list according to whitelist and blacklist
+        if 'all' in Arguments().formats and 'all' in Settings().config['formats']:
+            self.blacklist = Settings().config['formats']['all']['whitelist'] or []
+            self.whitelist = Settings().config['formats']['all']['blacklist'] or []
+
+            # Append formats to list if they can be used
+            self.formats = [format_name for format_name in Settings().config['formats'].keys() if
+                            self._can_use_format(format_name)]
+
+        else:
+            self.formats = [format_name for format_name in Arguments().formats if self._can_use_format(format_name)]
+
+    def _can_use_format(self, format_name: str) -> bool:
+        """
+        Check if format name should be used based on whitelist and blacklist
+        :param format_name: Name of format
+        :return: If format should be used
+        """
+
+        # Lowercase format name
+        format_name = format_name.lower()
+
+        # Reserved format names
+        if format_name in ['all']:
+            return False
+
+        # Format does not exist
+        if format_name not in Settings().config['formats'].keys():
+            return False
+
+        # Whitelisted formats
+        if self.whitelist and format_name not in self.whitelist:
+            return False
+
+        # Blacklisted formats
+        if self.blacklist and format_name in self.blacklist:
+            return False
+
+        return True
+
+    def video(self, video: Video) -> None:
+        """
+        Download chat from video
+        :param video: Video object
+        :return: None
+        """
+
+        # Parse video duration
+        regex = re.compile(r'((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
+        parts = regex.match(video.duration).groupdict()
+
+        time_params = {}
+        for name, param in parts.items():
+            if param:
+                time_params[name] = int(param)
+
+        video_duration = datetime.timedelta(**time_params)
+
+        formatter = Formatter(video)
+
+        # Special case for JSON
+        # Build JSON object before writing it
+        if 'json' in self.formats:
+            Logger().log('Downloading JSON data', Log.VERBOSE)
+            output: str = Pipe(Settings().config['formats']['json']['output']).output(video.data)
+            os.makedirs(os.path.dirname(output), exist_ok=True)
+
+            data: dict = {
+                'video': video.data,
+                'comments': []
+            }
+
+            for comment in video.comments:
+
+                # Skip unspecified users if a list is provided.
+                if Arguments().users and comment.commenter.name.lower() not in Arguments().users:
+                    continue
+
+                # If specified, only include messages that include a specified string
+                if Arguments().includes and Arguments().includes not in comment.message.body.lower():
+                    continue
+
+                # Add comment to dictionary
+                data['comments'].append(comment.data)
+
+                # Ignore comments that were posted after the VOD finished
+                if Settings().config['formats']['json'].get('comments', {}).get('ignore_new_comments', False):
+                    comment_date = dateutil.parser.parse(comment.created_at)
+                    vod_finish_date = dateutil.parser.parse(video.created_at) + video_duration
+
+                    if comment_date > vod_finish_date:
+                        continue
+
+                if Logger().should_print_type(Log.PROGRESS):
+                    self.draw_progress(current=comment.content_offset_seconds,
+                                       end=video_duration.seconds,
+                                       description='json')
+
+            with open(output, 'w', encoding='utf-8') as file:
+                json.dump(data, file, indent=4, sort_keys=True)
+
+            Logger().log(f'[json] {output}')
+
+        # For each format (ignore json this time)
+        for format_name in [x for x in self.formats if x not in ['json']]:
+            Logger().log(f'Formatting chat using: {format_name}', Log.VERBOSE)
+
+            # Get (formatted_comment, comment), output
+            comment_tuple, output = formatter.use(format_name)
+
+            # Create output directory and write to file
+            os.makedirs(os.path.dirname(output), exist_ok=True)
+            with open(output, '+w', encoding='utf-8') as file:
+                # Write Twitch URL as header at the beginning of the file
+                twitch_url = f"https://www.twitch.tv/videos/{video.id}?filter=archives&sort=time"
+                file.write(f"{twitch_url}\n\n")
+
+                # For every comment in video
+                for formatted_comment, comment in comment_tuple:
+
+                    # Skip unspecified users if a list is provided.
+                    if Arguments().users and comment.commenter.name.lower() not in Arguments().users:
+                        continue
+
+                    # If specified, only include messages that include a specified string
+                    if Arguments().includes and Arguments().includes.lower() not in comment.message.body.lower():
+                        continue
+
+                    # Ignore comments that were posted after the VOD finished
+                    if Settings().config['formats'][format_name].get('comments', {}).get('ignore_new_comments', False):
+                        comment_date = dateutil.parser.parse(comment.created_at)
+                        vod_finish_date = dateutil.parser.parse(video.created_at) + video_duration
+
+                        if comment_date > vod_finish_date:
+                            continue
+
+                    # Draw progress
+                    if comment and Logger().should_print_type(Log.PROGRESS):
+                        self.draw_progress(current=comment.content_offset_seconds,
+                                           end=video_duration.seconds,
+                                           description=format_name)
+
+                    # Display preview
+                    Logger().log(formatted_comment, Log.PREVIEW)
+
+                    # Write comment to file
+                    file.write('{}\n'.format(formatted_comment))
+
+            Logger().log('[{}] {}'.format(format_name, output))
+
+    def videos(self, video_ids: List[int]) -> None:
+        """
+        Download multiple video ids
+        :param video_ids: List of video ids
+        :return: None
+        """
+        for video in self.helix_api.videos(video_ids):
+            Logger().log(format('\n{}'.format(video.title)), Log.REGULAR)
+            self.video(video)
+
+    def channels(self, channels: List[str]) -> None:
+        """
+        Download videos from multiple channels
+        :param channels: List of channel names
+        :return: None
+        """
+        Logger().log(f'Starting download with first={Arguments().first}', Log.DEBUG)
+        for channel, videos in self.helix_api.users(channels).videos(first=Arguments().first):
+            Logger().log(format('\n{}'.format(channel.display_name)), Log.REGULAR)
+            Logger().log(f'Videos object type: {type(videos)}', Log.DEBUG)
+            video_count = 0
+            try:
+                for video in videos:
+                    video_count += 1
+                    Logger().log(f'Processing video #{video_count}: {video.id}', Log.DEBUG)
+                    Logger().log(format('\n{}'.format(video.title)), Log.REGULAR)
+                    try:
+                        self.video(video)
+                        Logger().log(f'Successfully processed video {video.id}', Log.DEBUG)
+                    except Exception as e:
+                        Logger().log(f'Error processing video {video.id}: {e}', Log.ERROR)
+                        import traceback
+                        traceback.print_exc()
+                        # Continue to next video instead of stopping
+                        continue
+            except Exception as e:
+                Logger().log(f'Error iterating videos: {e}', Log.ERROR)
+                import traceback
+                traceback.print_exc()
+            Logger().log(f'Processed {video_count} videos for channel {channel.display_name}', Log.VERBOSE)
+
+    @staticmethod
+    def draw_progress(current: float, end: float, description: str = 'Downloading') -> None:
+        """
+        Draw download progress
+        :param current: Current chat position (seconds)
+        :param end: End position (seconds)
+        :param description: Progress description
+        :return:
+        """
+        sys.stdout.write('[{}] {}%\r'.format(description, '%.2f' % min(current * 10 / end * 10, 100.00)))
+        sys.stdout.flush()

tcd2-3.2.2.post1/tcd/formats/custom.py

@@ -0,0 +1,30 @@
+from typing import Generator, Tuple
+
+from twitch.helix import Video
+from twitch.v5 import Comments, Comment
+
+from tcd.formats.format import Format
+from tcd.pipe import Pipe
+
+
+class Custom(Format):
+
+    def __init__(self, video: Video, format_name: str):
+        super().__init__(video, format_name)
+
+    def use(self) -> Tuple[Generator[Tuple[str, Comment], None, None], str]:
+        """
+        Use this format
+        :return: tuple(formatted comment, comment), output format
+        """
+        # Format comments
+        comments = self.comment_generator(self.video.comments)
+
+        # Format output
+        output: str = Pipe(self.format_dictionary['output']).output(self.video.data)
+
+        return comments, output
+
+    def comment_generator(self, comments: Comments) -> Generator[Tuple[str, Comment], None, None]:
+        for comment in comments:
+            yield Pipe(self.format_dictionary['comments']).comment(comment.data), comment

tcd2-3.2.2.post1/tcd/formats/format.py

@@ -0,0 +1,11 @@
+from twitch.helix import Video
+
+from tcd.settings import Settings
+
+
+class Format:
+
+    def __init__(self, video: Video, format_name: str):
+        self.video: Video = video
+        self.format_name: str = format_name
+        self.format_dictionary: dict = Settings().config['formats'][format_name]