plexflow 0.0.64__py3-none-any.whl

plexflow/core/storage/object/redis_storage.py

@@ -0,0 +1,169 @@
+import json
+import pickle
+import redis
+
+class RedisObjectStore:
+    """A class used to represent a Redis Object Store.
+
+    Attributes:
+        client (Redis): A Redis client instance.
+        default_ttl (int): The default TTL (in seconds) for stored objects.
+
+    """
+
+    def __init__(self, host='localhost', port=6379, db=0, default_ttl=3600, password=None):
+        """Constructs all the necessary attributes for the RedisObjectStore object.
+
+        Args:
+            host (str, optional): The host of the Redis server. Defaults to 'localhost'.
+            port (int, optional): The port of the Redis server. Defaults to 6379.
+            db (int, optional): The database index of the Redis server. Defaults to 0.
+            default_ttl (int, optional): The default TTL (in seconds) for stored objects. Defaults to 3600.
+
+        """
+
+        self.client = redis.Redis(host=host, port=port, db=db, password=password)
+        self.default_ttl = default_ttl
+
+    def _serialize(self, obj, use_json=False):
+        """Serializes an object.
+
+        Args:
+            obj (object): The object to serialize.
+            use_json (bool, optional): Whether to use JSON for serialization. If False, pickle is used.
+
+        Returns:
+            str: The serialized object.
+
+        Raises:
+            Exception: If an error occurs during serialization.
+
+        """
+
+        try:
+            return json.dumps(obj) if use_json else pickle.dumps(obj)
+        except (TypeError, pickle.PicklingError) as e:
+            raise Exception("Failed to serialize object") from e
+
+    def retrieve_keys(self, pattern):
+        """Retrieves all keys matching a given pattern from the Redis store.
+
+        Args:
+            pattern (str): The pattern to match.
+
+        Returns:
+            list: A list of keys matching the pattern.
+
+        Raises:
+            Exception: If an error occurs during retrieving the keys.
+
+        """
+
+        try:
+            keys = self.client.keys(str(pattern))
+            return keys
+        except redis.RedisError as e:
+            raise Exception("Failed to retrieve keys from Redis") from e
+
+    def retrieve_values(self, pattern, use_json=False):
+        """Retrieves all values matching a given pattern from the Redis store and deserializes them.
+
+        Args:
+            pattern (str): The pattern to match.
+            use_json (bool, optional): Whether to use JSON for deserialization. If False, pickle is used.
+
+        Returns:
+            list: A list of deserialized values matching the pattern.
+
+        Raises:
+            Exception: If an error occurs during retrieving or deserializing the values.
+
+        """
+
+        try:
+            keys = self.client.keys(str(pattern))
+            serialized_values = self.client.mget(keys)
+            values = []
+            for serialized_value in serialized_values:
+                if serialized_value is not None:
+                    value = json.loads(serialized_value) if use_json else pickle.loads(serialized_value)
+                    values.append(value)
+                else:
+                    values.append(None)
+            return values
+        except redis.RedisError as e:
+            raise Exception("Failed to retrieve values from Redis") from e
+
+    def _store(self, key, serialized_object, ttl=None):
+        """Stores a serialized object in the Redis store.
+
+        Args:
+            key (str): The key under which the object is stored.
+            serialized_object (str): The serialized object to store.
+            ttl (int, optional): The TTL (in seconds) for the stored object. If None, the object is stored permanently.
+
+        Raises:
+            Exception: If an error occurs during storing the object.
+
+        """
+
+        try:
+            self.client.set(key, serialized_object, ex=ttl)
+        except redis.RedisError as e:
+            raise Exception("Failed to store object in Redis") from e
+
+    def store(self, key, obj, use_json=False):
+        """Stores a serialized version of an object in the Redis store permanently.
+
+        Args:
+            key (str): The key under which the object is stored.
+            obj (object): The object to store.
+            use_json (bool, optional): Whether to use JSON for serialization. If False, pickle is used.
+
+        """
+
+        serialized_object = self._serialize(obj, use_json)
+        self._store(key, serialized_object)
+
+    def store_temporarily(self, key, obj, ttl=None, use_json=False):
+        """Stores a serialized version of an object in the Redis store temporarily.
+
+        Args:
+            key (str): The key under which the object is stored.
+            obj (object): The object to store.
+            ttl (int, optional): The TTL (in seconds) for the stored object. If None, the default TTL is used.
+            use_json (bool, optional): Whether to use JSON for serialization. If False, pickle is used.
+
+        """
+
+        serialized_object = self._serialize(obj, use_json)
+        ttl = self.default_ttl if ttl is None else ttl
+        self._store(key, serialized_object, ttl)
+
+    def retrieve(self, key, use_json=False):
+        """Retrieves an object from the Redis store and deserializes it.
+
+        Args:
+            key (str): The key of the object to retrieve.
+            use_json (bool, optional): Whether to use JSON for deserialization. If False, pickle is used.
+
+        Returns:
+            object: The retrieved and deserialized object. If the object does not exist, returns None.
+
+        Raises:
+            Exception: If an error occurs during retrieving or deserializing the object.
+
+        """
+
+        try:
+            serialized_object = self.client.get(key)
+        except redis.RedisError as e:
+            raise Exception("Failed to retrieve object from Redis") from e
+
+        if serialized_object is None:
+            return None
+
+        try:
+            return json.loads(serialized_object) if use_json else pickle.loads(serialized_object)
+        except (json.JSONDecodeError, pickle.UnpicklingError) as e:
+            raise Exception("Failed to deserialize object") from e

plexflow/core/subtitles/providers/auto_subtitles.py

@@ -0,0 +1,48 @@
+from plexflow.core.subtitles.providers.oss.search import get_subtitles, OSSSubtitle
+from typing import List, Iterator, Any
+from plexflow.utils.hooks.redis import UniversalRedisHook
+from plexflow.core.subtitles.providers.oss.download import download_subtitle    
+from tqdm import tqdm
+from plexflow.core.subtitles.providers.oss.unlimited_oss import OpenSubtitlesDownloadQuotaReachedException
+from plexflow.utils.retry.utils import execute_until_success
+import time
+from pathlib import Path
+from plexflow.logging.log_setup import logger
+
+class AutoSubtitles:
+    """
+    A class that represents an auto subtitles provider.
+
+    Attributes:
+        imdb_id (str): The IMDb ID of the movie or TV show.
+        languages (List[str]): A list of languages for which subtitles are requested.
+        kwargs: Additional keyword arguments.
+
+    """
+    def __init__(self, imdb_id: str, languages: List[str] = (), **kwargs: Any) -> None:
+        self.imdb_id = imdb_id
+        self.languages = languages
+        self.redis_hook = kwargs.pop("redis_hook", UniversalRedisHook(redis_conn_id='redis', config_folder='config'))
+        self.download = kwargs.pop("download", False)
+        self.download_folder = Path(kwargs.pop("download_folder", Path(".")))
+        self.kwargs = kwargs
+    
+    def __iter__(self) -> Iterator[OSSSubtitle]:
+        subtitles = get_subtitles(self.imdb_id, self.languages, self.redis_hook, ignore_blacklist=True, **self.kwargs)
+        if len(subtitles) == 0:
+            logger.warning(f"No subtitles found for IMDb ID: {self.imdb_id}")
+            return
+        
+        for subtitle in tqdm(subtitles, total=len(subtitles)):
+            if self.download:
+                subtitle_path, skipped = execute_until_success(download_subtitle, delay_type='constant', delay=3, max_retries=10, subtitle=subtitle, redis_hook=self.redis_hook, retry_exceptions=[OpenSubtitlesDownloadQuotaReachedException], save_dir=self.download_folder)
+                if not skipped:
+                    logger.debug(f"Subtitle downloaded: {subtitle_path}")
+                    # time.sleep(1.2)
+            yield subtitle            
+    
+    def __next__(self) -> Any:
+        try:
+            return next(self.__iter__())
+        except StopIteration:
+            raise

plexflow/core/subtitles/providers/oss/datatypes.py

@@ -0,0 +1,104 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+from dataclasses_json import dataclass_json, Undefined
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class Uploader:
+    uploader_id: Optional[int] = None
+    name: Optional[str] = None
+    rank: Optional[str] = None
+    _catchall: dict = field(default_factory=dict)
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class FeatureDetails:
+    feature_id: Optional[int] = None
+    feature_type: Optional[str] = None
+    year: Optional[int] = None
+    title: Optional[str] = None
+    movie_name: Optional[str] = None
+    imdb_id: Optional[int] = None
+    tmdb_id: Optional[int] = None
+    _catchall: dict = field(default_factory=dict)
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class RelatedLinks:
+    label: Optional[str] = None
+    url: Optional[str] = None
+    img_url: Optional[str] = None
+    _catchall: dict = field(default_factory=dict)
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class Files:
+    file_id: Optional[int] = None
+    cd_number: Optional[int] = None
+    file_name: Optional[str] = None
+    _catchall: dict = field(default_factory=dict)
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class Attributes:
+    subtitle_id: Optional[str] = None
+    language: Optional[str] = None
+    download_count: Optional[int] = None
+    new_download_count: Optional[int] = None
+    hearing_impaired: Optional[bool] = None
+    hd: Optional[bool] = None
+    fps: Optional[float] = None
+    votes: Optional[int] = None
+    ratings: Optional[float] = None
+    from_trusted: Optional[bool] = None
+    foreign_parts_only: Optional[bool] = None
+    upload_date: Optional[str] = None
+    ai_translated: Optional[bool] = None
+    nb_cd: Optional[int] = None
+    machine_translated: Optional[bool] = None
+    release: Optional[str] = None
+    comments: Optional[str] = None
+    legacy_subtitle_id: Optional[int] = None
+    legacy_uploader_id: Optional[int] = None
+    uploader: Optional[Uploader] = None
+    feature_details: Optional[FeatureDetails] = None
+    url: Optional[str] = None
+    related_links: Optional[List[RelatedLinks]] = None
+    files: Optional[List[Files]] = None
+    _catchall: dict = field(default_factory=dict)
+
+    def __post_init__(self):
+        if isinstance(self.uploader, dict):
+            self.uploader = Uploader(**self.uploader)
+        if isinstance(self.feature_details, dict):
+            self.feature_details = FeatureDetails(**self.feature_details)
+        if isinstance(self.related_links, list):
+            self.related_links = [RelatedLinks(**rl) if isinstance(rl, dict) else rl for rl in self.related_links]
+        if isinstance(self.files, list):
+            self.files = [Files(**f) if isinstance(f, dict) else f for f in self.files]
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class Data:
+    id: Optional[str] = None
+    type: Optional[str] = None
+    attributes: Optional[Attributes] = None
+    _catchall: dict = field(default_factory=dict)
+
+    def __post_init__(self):
+        if isinstance(self.attributes, dict):
+            self.attributes = Attributes(**self.attributes)
+
+@dataclass_json(undefined=Undefined.EXCLUDE)
+@dataclass
+class SubtitleData:
+    total_pages: Optional[int] = None
+    total_count: Optional[int] = None
+    per_page: Optional[int] = None
+    page: Optional[int] = None
+    data: Optional[List[Data]] = None
+    _catchall: dict = field(default_factory=dict)
+
+    def __post_init__(self):
+        if isinstance(self.data, list):
+            self.data = [Data(**d) if isinstance(d, dict) else d for d in self.data]

plexflow/core/subtitles/providers/oss/download.py

@@ -0,0 +1,48 @@
+from typing import List, Any
+from contextlib import contextmanager, ExitStack
+from plexflow.core.subtitles.providers.oss.unlimited_oss import OpenSubtitlesManager, Subtitle, OpenSubtitlesDownloadQuotaReachedException
+from plexflow.core.subtitles.providers.oss.oss_subtitle import OSSSubtitle
+from plexflow.utils.retry.utils import execute_until_success
+from plexflow.utils.hooks.redis import UniversalRedisHook
+from pathlib import Path
+from plexflow.logging.log_setup import logger
+
+def download_subtitle(subtitle: OSSSubtitle, redis_hook: UniversalRedisHook = None, save_dir: Path = Path('.'), skip_exists: bool = True) -> None:
+    """
+    Downloads and saves the subtitle file using the OpenSubtitlesManager.
+
+    Args:
+        subtitle (OSSSubtitle): The subtitle object containing the file ID.
+
+    Returns:
+        None
+    """
+    folder = save_dir / str(subtitle.imdb_code) / subtitle.subtitle.language / str(subtitle.subtitle.id)
+    filepath = folder / (str(subtitle.subtitle.file_id) + ".srt")
+    metapath = folder / "metadata.json"
+    
+    if skip_exists and filepath.exists():
+        logger.debug(f"Subtitle already exists: {filepath}")
+        return None, True
+    else:
+        with OpenSubtitlesManager.from_yaml(
+            yaml_file='config/credentials.yaml',
+            redis_hook=redis_hook, 
+        ) as manager:
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            metapath.write_text(subtitle.subtitle.to_json())
+            return manager.download_and_save(subtitle.subtitle.file_id, filename=str(filepath)), False
+
+def download_subtitles(subtitles: List[OSSSubtitle], **kwargs) -> None:
+    """
+    Downloads subtitles for a list of OSSSubtitle objects.
+
+    Args:
+        subtitles (List[OSSSubtitle]): A list of OSSSubtitle objects representing the subtitles to be downloaded.
+
+    Returns:
+        None
+    """
+    redis_hook = kwargs.pop("redis_hook", UniversalRedisHook(redis_conn_id='redis', config_folder='config'))
+    for subtitle in subtitles:
+        execute_until_success(download_subtitle, delay_type='constant', delay=3, max_retries=10, subtitle=subtitle, redis_hook=redis_hook, retry_exceptions=[OpenSubtitlesDownloadQuotaReachedException])

plexflow/core/subtitles/providers/oss/old.py

@@ -0,0 +1,144 @@
+import json
+import logging
+import requests
+from slugify import slugify
+import time
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+
+class OpenSubtitles:
+    """A class to interact with the OpenSubtitles API."""
+
+    def __init__(self, credentials_path: str, **kwargs):
+        """Initialize the OpenSubtitles object with user credentials."""
+        self._users = self._parse_credentials(credentials_path)
+        if not self._users:
+            raise RuntimeError("No user credentials specified")
+        self._user_blacklist = set()
+        self._update_user_context(**kwargs)
+
+    def _parse_credentials(self, path: str):
+        """Parse the credentials from the given file path."""
+        with open(path) as fd:
+            credentials = json.load(fd)
+            return [
+                {
+                    **{
+                        "username": item["login"]["username"],
+                        "password": item["login"]["password"]
+                    },
+                    **{
+                        slugify(field["name"].lower(), separator="_", lowercase=True): field["value"]
+                        for field in item["fields"]
+                    }
+                }
+                for item in credentials.get("items", [])
+            ]
+
+    def _get_user_context(self, **kwargs):
+        """Get the user context that is not in the blacklist."""
+        ctx = next((ctx for ctx in self._users if ctx["username"] not in self._user_blacklist), None)
+        if not ctx:
+            raise RuntimeError("No user context available")
+        return ctx
+
+    def _update_user_context(self, **kwargs):
+        """Update the current user context."""
+        self._current_ctx = self._get_user_context(**kwargs)
+        payload = {
+            "username": self._current_ctx["username"],
+            "password": self._current_ctx["password"],
+        }
+        r = requests.post(
+            url="https://api.opensubtitles.com/api/v1/login",
+            headers={
+                'Content-Type': "application/json",
+                'Accept': "application/json",
+                'Api-Key': self._current_ctx["api_key"],
+                'User-Agent': self._current_ctx["user_agent"],
+            },
+            data=json.dumps(payload)
+        )
+        if r.ok:
+            response = r.json()
+            self._current_ctx["token"] = response.get("token")
+        else:
+            logging.error(r.text)
+            raise RuntimeError("Failed to update user context")
+
+    def get_subtitles(self, **kwargs):
+        """Get subtitles from the OpenSubtitles API."""
+        r = requests.get(
+            url=f"https://api.opensubtitles.com/api/v1/subtitles",
+            headers={
+                "Api-Key": self._current_ctx["api_key"],
+                "User-Agent": self._current_ctx["user_agent"],
+            },
+            params=kwargs,
+        )
+        if r.ok:
+            return r.json()
+        else:
+            raise RuntimeError(f"Failed to get subtitles [status={r.status_code}]")
+
+    def get_download_link(self, **kwargs):
+        """Get the download link for the given file ID."""
+        file_id = kwargs.get("file_id")
+        if not isinstance(file_id, int):
+            raise RuntimeError("file_id must be of type int")
+        r = requests.post(
+            url="https://api.opensubtitles.com/api/v1/download",
+            headers={
+                'Accept': "application/json",
+                'Api-Key': self._current_ctx["api_key"],
+                "Authorization": f"Bearer {self._current_ctx['token']}",
+                "User-Agent": self._current_ctx["user_agent"],
+                'Content-Type': "application/json",
+            },
+            data=json.dumps({
+                "file_id": file_id
+            })
+        )
+        if r.ok:
+            response = r.json()
+            remaining = response["remaining"]
+            if remaining <= 0:
+                logging.info("Quota reached, updating user context")
+                self._user_blacklist.add(self._current_ctx["username"])
+                self._update_user_context(**kwargs)
+                logging.info("Trying again...")
+                return self.get_download_link(**kwargs)
+            else:
+                return response["link"]
+        else:
+            logging.error(f"Failed to get download link for file_id={file_id} status={r.status_code}")
+            raise RuntimeError(f"Failed to get download link for file_id={file_id} status={r.status_code}")
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description='Download subtitles from OpenSubtitles')
+    parser.add_argument('--credentials', required=True, help='Path to credentials file')
+    args = parser.parse_args()
+
+    oss = OpenSubtitles(credentials_path=args.credentials)
+    data = oss.get_subtitles(imdb_id="1375666", languages=",".join(("nl", "en")))
+
+    for sub in data["data"]:
+        files = sub["attributes"]["files"]
+        for f in files:
+            fid = f["file_id"]
+            link = oss.get_download_link(file_id=fid)
+            for _ in range(100):
+                try:
+                    r = requests.get(link)
+                    if r.ok:
+                        with open(f"/Users/david/Downloads/subs/{fid}.srt", "wb") as fd:
+                            fd.write(r.content)
+                            break
+                    else:
+                        raise Exception(f"Bad status: {r.status_code}")
+                except Exception as e:
+                    logging.error(e)
+                    time.sleep(0.1)