YTSync 0.0.0a0__py3-none-any.whl

yt2jf/__init__.py

@@ -0,0 +1 @@
+from .main import start  # noqa: F401

yt2jf/bot.py

@@ -0,0 +1,436 @@
+# noinspection PyUnresolvedReferences
+"""Module for TelegramAPI.
+
+>>> Bot
+
+"""
+
+import logging
+import secrets
+import sys
+import time
+from datetime import datetime
+from enum import StrEnum
+from typing import Dict, List
+
+import requests
+from yt_dlp.utils import DownloadError
+
+from yt2jf.config import env
+from yt2jf.exceptions import BotInUse, BotTokenInvalid, BotWebhookConflict
+from yt2jf.settings import Audio, Chat, Document, PhotoFragment, Text, Video, Voice
+from yt2jf.word_match import word_match
+from yt2jf.youtube import queue_download
+
+BASE_URL = f"https://api.telegram.org/bot{env.bot_token}"
+LOGGER = logging.getLogger("uvicorn.default")
+
+
+class RequestMethods(StrEnum):
+    """Allowed request methods.
+
+    >>> RequestMethods
+
+    """
+
+    GET = "GET"
+    POST = "POST"
+
+
+def intro() -> str:
+    """Returns a welcome message as a string.
+
+    Returns:
+        str:
+    """
+    return (
+        "\nTo start, send a link to YT music playlist in the following format:\n\n"
+        "- /id: <playlist id>\n- /url: <playlist url>\n"
+    )
+
+
+def _make_request(
+    url: str,
+    payload: dict,
+    files: dict = None,
+    method: RequestMethods = RequestMethods.POST,
+) -> requests.Response:
+    """Makes a post request with a ``connect timeout`` of 5 seconds and ``read timeout`` of 60.
+
+    Args:
+        url: URL to submit the request.
+        payload: Payload received, to extract information from.
+        files: Take filename as an optional argument.
+
+    Returns:
+        Response:
+        Response class.
+    """
+    if method == RequestMethods.GET:
+        response = requests.get(url=url, data=payload, files=files, timeout=(2, 3))
+    elif method == RequestMethods.POST:
+        response = requests.post(url=url, data=payload, files=files, timeout=(2, 3))
+    else:
+        raise ValueError("Invalid request method received: '%s'", method)
+    if not response.ok:
+        LOGGER.debug(payload)
+        LOGGER.debug(files)
+        LOGGER.warning("Called by: '%s'", sys._getframe(1).f_code.co_name)  # noqa
+        LOGGER.error(response.json())
+    return response
+
+
+def reply_to(
+    chat: Chat,
+    response: str,
+    parse_mode: str | None = "markdown",
+    retry: bool = False,
+) -> requests.Response:
+    """Generates a payload to reply to a message received.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        response: Message to be sent to the user.
+        parse_mode: Parse mode. Defaults to ``markdown``
+        retry: Retry reply in case reply failed because of parsing.
+
+    Returns:
+        Response:
+        Response class.
+    """
+    result = _make_request(
+        url=BASE_URL + "/sendMessage",
+        payload={
+            "chat_id": chat.id,
+            "reply_to_message_id": chat.message_id,
+            "text": response,
+            "parse_mode": parse_mode,
+        },
+    )
+    # Retry with response as plain text
+    if result.status_code == 400 and parse_mode and not retry:
+        LOGGER.warning("Retrying response as plain text with no parsing")
+        reply_to(chat, response, None, True)
+    return result
+
+
+def send_message(
+    chat_id: int,
+    response: str,
+    parse_mode: str | None = "markdown",
+    retry: bool = False,
+) -> requests.Response:
+    """Generates a payload to reply to a message received.
+
+    Args:
+        chat_id: Chat ID.
+        response: Message to be sent to the user.
+        parse_mode: Parse mode. Defaults to ``markdown``
+        retry: Retry reply in case reply failed because of parsing.
+
+    Returns:
+        Response:
+        Response class.
+    """
+    result = _make_request(
+        url=BASE_URL + "/sendMessage",
+        payload={"chat_id": chat_id, "text": response, "parse_mode": parse_mode},
+    )
+    # Retry with response as plain text
+    if result.status_code == 400 and parse_mode and not retry:
+        LOGGER.warning("Retrying response as plain text with no parsing")
+        send_message(chat_id=chat_id, response=response, parse_mode=None, retry=True)
+    return result
+
+
+def poll_for_messages(offset: int) -> None | int:
+    """Polls ``api.telegram.org`` for new messages.
+
+    Args:
+        offset: Offset in messages to poll.
+
+    Raises:
+        BotInUse:
+            - When a new polling is initiated using the same token.
+        ConnectionError:
+            - If unable to connect to the endpoint.
+
+    See Also:
+        Swaps ``offset`` value during every iteration to avoid reprocessing messages.
+    """
+    response = _make_request(
+        url=BASE_URL + "/getUpdates",
+        payload={"offset": offset, "timeout": 60},
+        method=RequestMethods.GET,
+    )
+    if response.ok:
+        results = response.json().get("result", [])
+        if not results:
+            return None
+
+        last_update_id = offset
+        for result in results:
+            if payload := result.get("message"):
+                process_request(payload)
+            else:
+                LOGGER.error("Received empty payload!!")
+            last_update_id = result["update_id"]
+
+        return last_update_id + 1
+
+    # Handle errors
+    error_data = response.json()
+    err_desc = error_data.get("description", "")
+
+    if response.status_code == 409:
+        if "webhook" in err_desc.lower():
+            raise BotWebhookConflict(err_desc)
+        raise BotInUse(err_desc)
+    if response.status_code == 401:
+        raise BotTokenInvalid(error_data)
+    raise ConnectionError(error_data)
+
+
+def process_request(payload: Dict[str, int | dict]) -> None:
+    """Processes the request via Telegram messages.
+
+    Args:
+        payload: Payload as received.
+    """
+    LOGGER.debug(payload)
+    chat = Chat(**{**payload, **payload["chat"], **payload["from"]})
+    if not authenticate(chat):
+        LOGGER.warning(payload)
+        return
+    if not verify_timeout(chat):
+        LOGGER.warning(payload)
+        return
+    if payload.get("text"):
+        chat.message_type = "text"
+        process_text(chat, Text(**payload))
+    elif payload.get("voice"):
+        chat.message_type = "voice"
+        process_voice(chat, Voice(**payload["voice"]))
+    elif payload.get("document"):
+        chat.message_type = "document"
+        process_document(chat, Document(**payload["document"]))
+    elif payload.get("video"):
+        chat.message_type = "video"
+        process_video(chat, Video(**payload["video"]))
+    elif payload.get("audio"):
+        chat.message_type = "audio"
+        process_audio(chat, Audio(**payload["audio"]))
+    elif payload.get("photo"):
+        # Matches for compressed images
+        chat.message_type = "photo"
+        process_photo(chat, [PhotoFragment(**d) for d in payload["photo"]])
+    else:
+        reply_to(chat, "Payload type is not allowed.")
+
+
+def username_is_valid(username: str) -> bool:
+    """Compares username and returns True if username is allowed."""
+    for user in env.bot_users:
+        if secrets.compare_digest(user, username):
+            return True
+    return False
+
+
+def authenticate(chat: Chat) -> bool:
+    """Authenticates the user with ``userId`` and ``userName``.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+
+    Returns:
+        bool:
+        Returns a boolean to indicate whether the user is authenticated.
+    """
+    if chat.is_bot:
+        LOGGER.error("Bot request from %s", chat.username)
+        send_message(
+            chat_id=chat.id,
+            response=f"Sorry {chat.first_name}! I can't process requests from bots.",
+        )
+        return False
+    if chat.id not in env.bot_chat_ids or not username_is_valid(username=chat.username):
+        LOGGER.error("Unauthorized chatID [%d] or userName [%s]", chat.id, chat.username)
+        send_message(chat_id=chat.id, response=f"401 Unauthorized user: ({chat.username})")
+        return False
+    return True
+
+
+def verify_timeout(chat: Chat) -> bool:
+    """Verifies whether the message was received in the past 60 seconds.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+
+    Returns:
+        bool:
+        True or False flag to indicate if the request timed out.
+    """
+    if int(time.time()) - chat.date < 60:
+        return True
+    request_time = time.strftime("%m-%d-%Y %H:%M:%S", time.localtime(chat.date))
+    LOGGER.warning("Request timed out [%s] for %s", request_time, chat.username)
+    reply_to(
+        chat,
+        f"Request timed out\nRequested: {request_time}\n"
+        f"Processed: {time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(time.time()))}",
+    )
+    return False
+
+
+def process_photo(chat: Chat, data_class: List[PhotoFragment]) -> None:
+    """Processes a photo input.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Voice object.
+    """
+    LOGGER.info(data_class)
+    reply_to(
+        chat,
+        "Image fragments are not supported. If you're sending a compressed image, "
+        "please try sending it without compression.",
+    )
+
+
+def process_audio(chat: Chat, data_class: Audio) -> None:
+    """Processes an audio input.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Voice object.
+    """
+    process_document(chat, data_class)
+
+
+def process_video(chat: Chat, data_class: Video) -> None:
+    """Processes a video input.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Voice object.
+    """
+    process_document(chat, data_class)
+
+
+def process_voice(chat: Chat, data_class: Voice) -> None:
+    """Processes the audio file in payload received after checking for authentication.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Voice object.
+    """
+    reply_to(chat, "Audio inputs are not supported at the moment. Please try text input.")
+
+
+def process_document(chat: Chat, data_class: Document | Audio | Video) -> None:
+    """Processes the document in payload received after checking for authentication.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Document object.
+    """
+    reply_to(chat, "Document inputs are not supported at the moment. Please try text input.")
+
+
+def process_text(chat: Chat, data_class: Text) -> None:
+    """Processes the text in payload received after checking for authentication.
+
+    Args:
+        chat: Required section of the payload as Chat object.
+        data_class: Required section of the payload as Text object.
+    """
+    if data_class.text:
+        data_class.text = data_class.text.strip()
+    else:
+        send_message(chat_id=chat.id, response="Un-processable payload")
+        return
+    data_class.text = data_class.text.replace("override", "").replace("OVERRIDE", "")
+    text_lower = data_class.text.lower().lstrip("/")
+    if word_match(
+        phrase=text_lower,
+        match_list=(
+            "hey",
+            "hola",
+            "what's up",
+            "ssup",
+            "whats up",
+            "hello",
+            "hi",
+            "howdy",
+            "hey",
+            "chao",
+            "hiya",
+            "aloha",
+        ),
+        strict=True,
+    ):
+        reply_to(
+            chat,
+            intro(),
+        )
+        return
+    if text_lower == "start":
+        send_message(chat.id, intro())
+        return
+    if text_lower == "help":
+        send_message(
+            chat_id=chat.id,
+            response="Use '/id' or '/url' followed by the identifier.",
+        )
+        return
+    if text_lower == "test":
+        reply_to(chat, f"Test message received at - {datetime.now().strftime('%c')}")
+        return
+    executor(data_class.text, chat)
+
+
+def executor(command: str, chat: Chat) -> None:
+    """Executes the command via offline communicator.
+
+    Args:
+        command: Command to be executed.
+        chat: Required section of the payload as Chat object.
+    """
+    LOGGER.info("Request: %s", command)
+    # TODO: Replace with reply_to
+    kwargs = dict(chat_id=chat.id, callback=send_message)
+    if command.startswith("/id"):
+        if playlist_id := command.replace("/id", "").strip():
+            kwargs["playlist_id"] = playlist_id
+        else:
+            reply_to(chat, "Invalid entry, a playlist id is required followed by /id")
+            return
+    elif command.startswith("/url"):
+        if playlist_url := command.replace("/url", "").strip():
+            kwargs["playlist_url"] = playlist_url
+        else:
+            reply_to(chat, "Invalid entry, a playlist url is required followed by /url")
+            return
+    else:
+        process_response(
+            f"Invalid command received: {command}\n\nEither use '/id' or '/url' followed by the identifier.",
+            chat,
+        )
+        return
+    try:
+        name = queue_download(**kwargs)
+        response = f"Download queued for {name!r}"
+    except (ValueError, AssertionError, DownloadError) as error:
+        response = error.__str__()
+    LOGGER.info("Response: %s", response)
+    process_response(response, chat)
+
+
+def process_response(response: str, chat: Chat) -> None:
+    """Processes the response via Telegram API.
+
+    Args:
+        response: Response from yt2jf.
+        chat: Required section of the payload as Chat object.
+    """
+    send_message(chat.id, response, None)

yt2jf/config.py

@@ -0,0 +1,80 @@
+import math
+import os
+import pathlib
+import socket
+import warnings
+from ipaddress import IPv4Address
+from multiprocessing import current_process
+from typing import Any, Dict, List
+
+from pydantic import (
+    DirectoryPath,
+    Field,
+    FilePath,
+    HttpUrl,
+    NewPath,
+    PositiveFloat,
+    PositiveInt,
+)
+
+from yt2jf.pydantic_config import PydanticEnvConfig
+
+SECRETS_PATH = os.environ.get("SECRETS_PATH") or os.environ.get("secrets_path") or ".env"
+LOGICAL_CORES = os.cpu_count() or 2
+PHYSICAL_CORES = math.ceil(LOGICAL_CORES / 2)
+
+
+class EnvConfig(PydanticEnvConfig):
+    """Configuration values for the project.
+
+    >>> EnvConfig
+
+    """
+
+    host: str = socket.gethostbyname("localhost")
+    port: PositiveInt = 4483
+    log_config: FilePath | Dict[str, Any] | None = None
+
+    # Applies to both rsync and telegram polling
+    max_retries: PositiveInt | PositiveFloat = Field(10, le=30, ge=1)
+    backoff_factor: PositiveInt | PositiveFloat = Field(3, le=10, ge=1)
+
+    # Concurrency
+    max_listeners: PositiveInt = Field(PHYSICAL_CORES, le=LOGICAL_CORES, ge=1)
+    max_transfers: PositiveInt = Field(LOGICAL_CORES, le=LOGICAL_CORES * 2, ge=1)
+
+    # Data
+    data_dir: NewPath | DirectoryPath = pathlib.Path("data")
+
+    # Telegram config
+    bot_token: str
+    bot_chat_ids: List[int]
+    bot_users: List[str]
+    poll_interval: PositiveInt | PositiveFloat = Field(2, le=5, ge=1)
+
+    # Remote config
+    remote_host: str | None = None
+    remote_user: str | None = None
+    remote_path: str | None = None
+    delete_after_sync: bool = True
+
+    # Telegram Webhook specific
+    bot_webhook: HttpUrl | None = None
+    bot_webhook_ip: IPv4Address | None = None
+    bot_endpoint: str = Field("/telegram-webhook", pattern=r"^\/")
+    bot_secret: str | None = Field(None, pattern="^[A-Za-z0-9_-]{1,256}$")
+    bot_certificate: FilePath | None = None
+
+    class Config:
+        """Environment variables configuration."""
+
+        vault_table = "yt2jf"
+        env_file = SECRETS_PATH
+        extra = "ignore"
+
+
+env = EnvConfig()
+
+if not all((env.remote_host, env.remote_path, env.remote_user)) and current_process().name == "MainProcess":
+    warnings.warn("No remote connections have been setup, all downloaded media will be stored locally.")
+env.data_dir.mkdir(exist_ok=True)

yt2jf/exceptions.py

@@ -0,0 +1,40 @@
+import requests
+
+EgressErrors = (
+    ConnectionError,
+    TimeoutError,
+    requests.RequestException,
+    requests.Timeout,
+)
+
+
+class BotError(Exception):
+    """Custom base exception for Telegram Bot.
+
+    >>> BotError
+
+    """
+
+
+class BotWebhookConflict(BotError):
+    """Error for conflict with webhook and getUpdates API call.
+
+    >>> BotWebhookConflict
+
+    """
+
+
+class BotInUse(BotError):
+    """Error indicate bot token is being used else where.
+
+    >>> BotInUse
+
+    """
+
+
+class BotTokenInvalid(BotError):
+    """Error indicate bot token is invalid.
+
+    >>> BotTokenInvalid
+
+    """

yt2jf/main.py

@@ -0,0 +1,79 @@
+import asyncio
+import logging
+import pathlib
+from contextlib import asynccontextmanager
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.routing import APIRoute
+
+from yt2jf.config import env
+from yt2jf.poll import run_polling, shutdown_event
+from yt2jf.routes import (
+    ACTIVE_TASKS,
+    api_delete_webhook,
+    api_get_webhook,
+    api_set_webhook,
+    telegram_webhook,
+)
+from yt2jf.version import __version__
+
+LOGGER = logging.getLogger("uvicorn.default")
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    """Simple startup function to add anything that has to be triggered when Jarvis API starts up."""
+    # noinspection HttpUrlsUsage
+    LOGGER.info("Hosting at http://%s:%s", env.host, env.port)
+    bg_task = None
+    if not env.bot_webhook:
+        LOGGER.info("Polling for incoming messages...")
+        bg_task = asyncio.create_task(run_polling())
+        ACTIVE_TASKS["poll"] = bg_task
+    yield
+    if bg_task:
+        bg_task.cancel()
+    shutdown_event()
+    LOGGER.info("Shutting down API server.")
+
+
+routes = [
+    APIRoute(
+        endpoint=telegram_webhook,
+        methods=["POST"],
+        path=env.bot_endpoint,  # No enum
+        include_in_schema=False,
+    ),
+    APIRoute(
+        endpoint=api_get_webhook,
+        methods=["GET"],
+        path="/get-webhook",
+    ),
+    APIRoute(
+        endpoint=api_set_webhook,
+        methods=["POST"],
+        path="/set-webhook",
+    ),
+    APIRoute(
+        endpoint=api_delete_webhook,
+        methods=["DELETE"],
+        path="/delete-webhook",
+    ),
+]
+
+app = FastAPI(title="YT2JF", version=__version__, lifespan=lifespan, routes=routes)
+
+
+def start():
+    """Start the Jarvis API server using Uvicorn."""
+    module_name = pathlib.Path(__file__)
+    kwargs = dict(
+        host=env.host,
+        port=env.port,
+        app=f"{module_name.parent.stem}.main:app",
+        workers=1,
+    )
+    if env.log_config:
+        kwargs["log_config"] = env.log_config
+    uvicorn.run(**kwargs)

yt2jf/poll.py

@@ -0,0 +1,59 @@
+import asyncio
+import logging
+
+import requests.exceptions
+
+from yt2jf.bot import poll_for_messages
+from yt2jf.config import env
+from yt2jf.exceptions import BotInUse, BotTokenInvalid, BotWebhookConflict, EgressErrors
+from yt2jf.youtube import controllers, process_pool
+
+LOGGER = logging.getLogger("uvicorn.default")
+
+
+def shutdown_event():
+    """Shuts down all the threads and gracefully terminates the processes."""
+    process_pool.shutdown(wait=True)
+    for controller in controllers:
+        LOGGER.info("Shutting down controller for: %s", controller.name)
+        try:
+            result = controller.future.result()
+        except Exception as exc:
+            LOGGER.error("Controller failed for %s: %s", controller.name, exc)
+        else:
+            LOGGER.info("Controller completed for %s: %s", controller.name, result)
+
+
+async def run_polling():
+    """Starts up all the threads and gracefully terminates the processes."""
+    offset = 0
+    failed_connections = 0
+    while True:
+        try:
+            await asyncio.sleep(env.poll_interval)
+            if offset_id := poll_for_messages(offset):
+                offset = offset_id
+        except EgressErrors as error:
+            if isinstance(error, requests.exceptions.ReadTimeout):
+                continue
+            LOGGER.error(error)
+            failed_connections += 1
+            if failed_connections > env.max_retries:
+                LOGGER.critical("ATTENTION::Couldn't recover from connection error. Restarting current process.")
+                delay = failed_connections * env.backoff_factor
+                LOGGER.info("Restarting in %d seconds.", delay)
+                await asyncio.sleep(delay)  # Simple backoff wait
+        except (
+            asyncio.CancelledError,
+            BotWebhookConflict,
+            BotInUse,
+            BotTokenInvalid,
+            KeyboardInterrupt,
+            Exception,
+        ) as error:
+            if isinstance(error, asyncio.CancelledError):
+                LOGGER.info("Shutting down all threads and gracefully terminated.")
+            else:
+                LOGGER.error(error)
+            shutdown_event()
+            break