ytdl-sub 2023.4.20.post2__py3-none-any.whl → 2026.1.24__py3-none-any.whl

ytdl_sub/__init__.py

@@ -1 +1 @@
-__pypi_version__ = "2023.04.20.post2";__local_version__ = "2023.04.20+d5d9978"
+__pypi_version__ = "2026.01.24";__local_version__ = "2026.01.24+60cdbad"

ytdl_sub/cli/entrypoint.py

@@ -0,0 +1,281 @@
+import gc
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from yt_dlp.utils import sanitize_filename
+
+from ytdl_sub.cli.output_summary import output_summary
+from ytdl_sub.cli.output_transaction_log import _maybe_validate_transaction_log_file
+from ytdl_sub.cli.output_transaction_log import output_transaction_log
+from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
+from ytdl_sub.cli.parsers.dl import DownloadArgsParser
+from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME
+from ytdl_sub.cli.parsers.main import parser
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
+from ytdl_sub.utils.exceptions import ValidationException
+from ytdl_sub.utils.file_handler import FileHandler
+from ytdl_sub.utils.file_lock import working_directory_lock
+from ytdl_sub.utils.logger import Logger
+
+logger = Logger.get()
+
+# View is a command to run a simple dry-run on a URL using the `_view` preset.
+# Use ytdl-sub dl arguments to use the preset
+_VIEW_EXTRA_ARGS_FORMATTER = "--preset _view --overrides.url {}"
+
+
+def _log_time() -> str:
+    return datetime.now().strftime("%Y-%m-%d-%H%M%S")
+
+
+def _maybe_write_subscription_log_file(
+    config: ConfigFile,
+    subscription: Subscription,
+    dry_run: bool,
+    exception: Optional[Exception] = None,
+) -> None:
+    success: bool = exception is None
+
+    # If dry-run, do nothing
+    if dry_run:
+        return
+
+    # If persisting logs is disabled, do nothing
+    if not config.config_options.persist_logs:
+        return
+
+    # If persisting successful logs is disabled, do nothing
+    if success and not config.config_options.persist_logs.keep_successful_logs:
+        return
+
+    log_subscription_name = sanitize_filename(subscription.name).lower().replace(" ", "_")
+    log_success = "success" if success else "error"
+
+    log_filename = f"{_log_time()}.{log_subscription_name}.{log_success}.log"
+    persist_log_path = Path(config.config_options.persist_logs.logs_directory) / log_filename
+
+    if not success:
+        Logger.log_exception(exception=exception, log_filepath=persist_log_path)
+
+    os.makedirs(os.path.dirname(persist_log_path), exist_ok=True)
+    FileHandler.copy(Logger.debug_log_filename(), persist_log_path)
+
+
+def _download_subscriptions_from_yaml_files(
+    config: ConfigFile,
+    subscription_paths: List[str],
+    subscription_matches: List[str],
+    subscription_override_dict: Dict,
+    update_with_info_json: bool,
+    dry_run: bool,
+) -> List[Subscription]:
+    """
+    Downloads all subscriptions from one or many subscription yaml files.
+
+    Parameters
+    ----------
+    config
+        Configuration file
+    subscription_paths
+        Path to subscription files to download
+    subscription_matches
+        Optional list of substrings to match subscription names to (only run if matched)
+    update_with_info_json
+        Whether to actually download or update using existing info json
+    dry_run
+        Whether to dry run or not
+
+    Returns
+    -------
+    List of subscriptions processed
+
+    Raises
+    ------
+    Exception
+        Any exception during download
+    """
+    subscriptions: List[Subscription] = []
+
+    # Load all the subscriptions first to perform all validation before downloading
+    for path in subscription_paths:
+        subscriptions += Subscription.from_file_path(
+            config=config,
+            subscription_path=path,
+            subscription_matches=subscription_matches,
+            subscription_override_dict=subscription_override_dict,
+        )
+
+    for subscription in subscriptions:
+        with subscription.exception_handling():
+            logger.info(
+                "Beginning subscription %s for %s",
+                ("dry run" if dry_run else "download"),
+                subscription.name,
+            )
+            logger.debug("Subscription full yaml:\n%s", subscription.as_yaml())
+
+            if update_with_info_json:
+                subscription.update_with_info_json(dry_run=dry_run)
+            else:
+                subscription.download(dry_run=dry_run)
+
+        _maybe_write_subscription_log_file(
+            config=config,
+            subscription=subscription,
+            dry_run=dry_run,
+            exception=subscription.exception,
+        )
+
+        Logger.cleanup(has_error=False)
+        gc.collect()  # Garbage collect after each subscription download
+
+    return subscriptions
+
+
+def _download_subscription_from_cli(
+    config: ConfigFile, dry_run: bool, extra_args: List[str]
+) -> Subscription:
+    """
+    Downloads a one-off subscription using the CLI
+
+    Parameters
+    ----------
+    config
+        Configuration file
+    dry_run
+        Whether this is a dry-run
+    extra_args
+        Extra arguments from argparse that contain dynamic subscription options
+
+    Returns
+    -------
+    Subscription and its download transaction log
+    """
+    dl_args_parser = DownloadArgsParser(
+        extra_arguments=extra_args, config_options=config.config_options
+    )
+    subscription_args_dict = dl_args_parser.to_subscription_dict()
+    subscription_name = dl_args_parser.get_dl_subscription_name()
+
+    subscription = Subscription.from_dict(
+        config=config, preset_name=subscription_name, preset_dict=subscription_args_dict
+    )
+
+    logger.info("Beginning CLI %s", ("dry run" if dry_run else "download"))
+    subscription.download(dry_run=dry_run)
+
+    return subscription
+
+
+def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Subscription:
+    """
+    `ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset,
+    inject the URL argument, and dry-run.
+    """
+    preset = "_view_split_chapters" if split_chapters else "_view"
+    subscription = Subscription.from_dict(
+        config=config,
+        preset_name="ytdl-sub-view",
+        preset_dict={"preset": preset, "overrides": {"url": url}},
+    )
+
+    logger.info(
+        "Viewing source variables for URL '%s'%s",
+        url,
+        " with split chapters" if split_chapters else "",
+    )
+    subscription.download(dry_run=True)
+
+    return subscription
+
+
+def main() -> List[Subscription]:
+    """
+    Entrypoint for ytdl-sub, without the error handling
+    """
+    # If no args are provided, print help and exit
+    if len(sys.argv) < 2:
+        parser.print_help()
+        return []
+
+    args, extra_args = parser.parse_known_args()
+
+    if args.subparser == "cli-to-sub":
+        print_cli_to_sub(args=extra_args)
+        return []
+
+    # Load the config
+    if args.config:
+        config = ConfigFile.from_file_path(args.config)
+    elif os.path.isfile(DEFAULT_CONFIG_FILE_NAME):
+        config = ConfigFile.from_file_path(DEFAULT_CONFIG_FILE_NAME)
+    else:
+        logger.info("No config specified, using defaults.")
+        config = ConfigFile.default()
+
+    subscriptions: List[Subscription] = []
+
+    # If transaction log file is specified, make sure we can open it
+    _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
+
+    with working_directory_lock(config=config):
+        if args.subparser == "sub":
+            if (
+                args.update_with_info_json
+                and not config.config_options.experimental.enable_update_with_info_json
+            ):
+                raise ExperimentalFeatureNotEnabled(
+                    "--update-with-info-json requires setting"
+                    " configuration.experimental.enable_update_with_info_json to True. This"
+                    " feature is ",
+                    "still being tested and has the ability to destroy files. Ensure you have a ",
+                    "full backup before usage. You have been warned!",
+                )
+
+            subscription_override_dict = {}
+            if args.dl_override:
+                subscription_override_dict = DownloadArgsParser.from_dl_override(
+                    override=args.dl_override, config=config
+                ).to_subscription_dict()
+
+            logger.info("Validating subscriptions...")
+            subscriptions = _download_subscriptions_from_yaml_files(
+                config=config,
+                subscription_paths=args.subscription_paths,
+                subscription_matches=args.match,
+                subscription_override_dict=subscription_override_dict,
+                update_with_info_json=args.update_with_info_json,
+                dry_run=args.dry_run,
+            )
+
+        # One-off download
+        elif args.subparser == "dl":
+            logger.info("Validating presets...")
+            subscriptions.append(
+                _download_subscription_from_cli(
+                    config=config, dry_run=args.dry_run, extra_args=extra_args
+                )
+            )
+        elif args.subparser == "view":
+            subscriptions.append(
+                _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
+            )
+        else:
+            raise ValidationException("Must provide one of the commands: sub, dl, view, cli-to-sub")
+
+    if not args.suppress_transaction_log:
+        output_transaction_log(
+            subscriptions=subscriptions,
+            transaction_log_file_path=args.transaction_log,
+        )
+
+    output_summary(subscriptions, suppress_colors=args.suppress_colors)
+
+    return subscriptions

ytdl_sub/cli/output_summary.py

@@ -0,0 +1,117 @@
+from typing import List
+
+from colorama import Fore
+
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.logger import Logger
+
+logger = Logger.get()
+
+
+def _green(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.GREEN + value + Fore.RESET
+
+
+def _red(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.RED + value + Fore.RESET
+
+
+def _no_color(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.RESET + value + Fore.RESET
+
+
+def _str_int(value: int) -> str:
+    if value > 0:
+        return f"+{value}"
+    return str(value)
+
+
+def _color_int(value: int, suppress_colors: bool = False) -> str:
+    str_int = _str_int(value)
+    if value > 0:
+        return _green(str_int, suppress_colors)
+    if value < 0:
+        return _red(str_int, suppress_colors)
+    return _no_color(str_int, suppress_colors)
+
+
+def output_summary(subscriptions: List[Subscription], suppress_colors: bool) -> None:
+    """
+    Parameters
+    ----------
+    subscriptions
+        Processed subscriptions
+    suppress_colors
+        Whether to have color or not
+
+    Returns
+    -------
+    Output summary to print
+    """
+    # many locals for proper output printing
+    # pylint: disable=too-many-locals
+    if len(subscriptions) == 0:
+        logger.info("No subscriptions ran")
+        return
+
+    summary: List[str] = []
+
+    # Initialize totals to 0
+    total_subs: int = len(subscriptions)
+    total_subs_str = f"Total: {total_subs}"
+    total_added: int = sum(sub.num_entries_added for sub in subscriptions)
+    total_modified: int = sum(sub.num_entries_modified for sub in subscriptions)
+    total_removed: int = sum(sub.num_entries_removed for sub in subscriptions)
+    total_entries: int = sum(sub.num_entries for sub in subscriptions)
+    total_errors: int = sum(sub.exception is not None for sub in subscriptions)
+
+    # Initialize widths to 0
+    width_sub_name: int = max(len(sub.name) for sub in subscriptions) + 4  # aesthetics
+    width_num_entries_added: int = len(_color_int(total_added, suppress_colors))
+    width_num_entries_modified: int = len(_color_int(total_modified, suppress_colors))
+    width_num_entries_removed: int = len(_color_int(total_removed, suppress_colors))
+    width_num_entries: int = len(str(total_entries)) + 4  # aesthetics
+
+    # Build the summary
+    for subscription in subscriptions:
+        num_entries_added = _color_int(subscription.num_entries_added, suppress_colors)
+        num_entries_modified = _color_int(subscription.num_entries_modified, suppress_colors)
+        num_entries_removed = _color_int(subscription.num_entries_removed * -1, suppress_colors)
+        num_entries = str(subscription.num_entries)
+        status = (
+            _red(subscription.exception.__class__.__name__, suppress_colors)
+            if subscription.exception
+            else _green("✔", suppress_colors)
+        )
+
+        summary.append(
+            f"{subscription.name:<{width_sub_name}} "
+            f"{num_entries_added:>{width_num_entries_added}} "
+            f"{num_entries_modified:>{width_num_entries_modified}} "
+            f"{num_entries_removed:>{width_num_entries_removed}} "
+            f"{num_entries:>{width_num_entries}} "
+            f"{status}"
+        )
+
+    total_errors_str = (
+        _green("Success", suppress_colors)
+        if total_errors == 0
+        else _red(f"Error{'s' if total_errors > 1 else ''}", suppress_colors)
+    )
+
+    summary.append(
+        f"{total_subs_str:<{width_sub_name}} "
+        f"{_color_int(total_added, suppress_colors):>{width_num_entries_added}} "
+        f"{_color_int(total_modified, suppress_colors):>{width_num_entries_modified}} "
+        f"{_color_int(total_removed * -1, suppress_colors):>{width_num_entries_removed}} "
+        f"{total_entries:>{width_num_entries}} "
+        f"{total_errors_str}"
+    )
+
+    if total_errors > 0:
+        summary.append("")
+        summary.append(f"See `{Logger.error_log_filename()}` for details on errors.")
+        summary.append("Consider making a GitHub issue including the uploaded log file.")
+
+    # Hack to always show download summary, even if logs are set to quiet
+    logger.warning("Download Summary:\n%s", "\n".join(summary))

ytdl_sub/cli/output_transaction_log.py

@@ -0,0 +1,54 @@
+from typing import List
+from typing import Optional
+
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import ValidationException
+from ytdl_sub.utils.logger import Logger
+
+logger = Logger.get()
+
+
+def _maybe_validate_transaction_log_file(transaction_log_file_path: Optional[str]) -> None:
+    if transaction_log_file_path:
+        try:
+            with open(transaction_log_file_path, "w", encoding="utf-8"):
+                pass
+        except Exception as exc:
+            raise ValidationException(
+                f"Transaction log file '{transaction_log_file_path}' cannot be written to. "
+                f"Reason: {str(exc)}"
+            ) from exc
+
+
+def output_transaction_log(
+    subscriptions: List[Subscription],
+    transaction_log_file_path: Optional[str],
+) -> None:
+    """
+    Maybe print and/or write transaction logs to a file
+
+    Parameters
+    ----------
+    subscriptions
+        Processed subscriptions
+    transaction_log_file_path
+        Optional file path to write to
+    """
+    transaction_log_file_contents = ""
+    for subscription in subscriptions:
+        if subscription.transaction_log.is_empty:
+            transaction_log_contents = f"\nNo files changed for {subscription.name}"
+        else:
+            transaction_log_contents = (
+                f"Transaction log for {subscription.name}:\n"
+                f"{subscription.transaction_log.to_output_message(subscription.output_directory)}"
+            )
+
+        if transaction_log_file_path:
+            transaction_log_file_contents += transaction_log_contents
+        else:
+            logger.info(transaction_log_contents)
+
+    if transaction_log_file_contents:
+        with open(transaction_log_file_path, "w", encoding="utf-8") as transaction_log_file:
+            transaction_log_file.write(transaction_log_file_contents)

ytdl_sub/cli/parsers/cli_to_sub.py

@@ -0,0 +1,64 @@
+from typing import List
+
+import yt_dlp
+import yt_dlp.options
+
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.yaml import dump_yaml
+
+logger = Logger.get()
+
+# pylint: disable=missing-function-docstring
+
+##############################################################
+# --- BEGIN ----
+# Copy of https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+
+    patched_parser = create_parser()
+    patched_parser.defaults.update(
+        {
+            "ignoreerrors": False,
+            "retries": 0,
+            "fragment_retries": 0,
+            "extract_flat": False,
+            "concat_playlist": "never",
+            "update_self": False,
+        }
+    )
+    yt_dlp.options.create_parser = lambda: patched_parser
+    try:
+        return yt_dlp.parse_options(opts)
+    finally:
+        yt_dlp.options.create_parser = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+    opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if "postprocessors" in diff:
+        diff["postprocessors"] = [
+            pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]
+        ]
+    return diff
+
+
+# --- END ----
+##############################################################
+
+
+def print_cli_to_sub(args: List[str]) -> None:
+    api_args = cli_to_api(args)
+    if not api_args:
+        logger.info("Does not resolve to any yt-dlp args")
+        return
+
+    print(dump_yaml({"ytdl_options": api_args}))

ytdl_sub/cli/{download_args_parser.py → parsers/dl.py}

@@ -1,13 +1,15 @@
 import hashlib
 import re
 import shlex
+from typing import Any
 from typing import Dict
 from typing import List
 from typing import Tuple
 
 from mergedeep import mergedeep
 
-from ytdl_sub.cli.main_args_parser import MainArguments
+from ytdl_sub.cli.parsers.main import MainArguments
+from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.config_validator import ConfigOptions
 from ytdl_sub.utils.exceptions import InvalidDlArguments
 
@@ -116,7 +118,7 @@ class DownloadArgsParser:
         return largest_consecutive + 1
 
     @classmethod
-    def _argument_name_and_value_to_dict(cls, arg_name: str, arg_value: str) -> Dict:
+    def _argument_name_and_value_to_dict(cls, arg_name: str, arg_value: Any) -> Dict:
         """
         :param arg_name: Argument name in the form of 'key1.key2.key3'
         :param arg_value: Argument value
@@ -134,11 +136,15 @@ class DownloadArgsParser:
 
         next_dict[arg_name_split[-1]] = arg_value
 
-        # TODO: handle ints/floats
-        if arg_value == "True":
-            next_dict[arg_name_split[-1]] = True
-        elif arg_value == "False":
-            next_dict[arg_name_split[-1]] = False
+        if isinstance(arg_value, str):
+            if arg_value == "True":
+                next_dict[arg_name_split[-1]] = True
+            elif arg_value == "False":
+                next_dict[arg_name_split[-1]] = False
+            elif arg_value.isdigit():
+                next_dict[arg_name_split[-1]] = int(arg_value)
+            elif arg_value.replace(".", "", 1).isdigit():
+                next_dict[arg_name_split[-1]] = float(arg_value)
 
         return argument_dict
 
@@ -236,9 +242,19 @@ class DownloadArgsParser:
 
         return subscription_dict
 
-    def get_args_hash(self) -> str:
+    def get_dl_subscription_name(self) -> str:
         """
-        :return: Hash of the arguments provided
+        Returns a deterministic name based on input args
         """
-        hash_string = str(sorted(self._unknown_arguments))
-        return hashlib.sha256(hash_string.encode()).hexdigest()[-8:]
+        to_hash = str(sorted(self._unknown_arguments))
+        hash_str = hashlib.sha256(to_hash.encode()).hexdigest()[-8:]
+        return f"cli-dl-{hash_str}"
+
+    @classmethod
+    def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser":
+        """
+        Create a DownloadArgsParser from a sub --override argument value
+        """
+        return DownloadArgsParser(
+            extra_arguments=override.split(), config_options=config.config_options
+        )