plexflow 0.0.64__tar.gz → 0.0.66__tar.gz

{plexflow-0.0.64 → plexflow-0.0.66}/PKG-INFO

@@ -1,19 +1,20 @@
 Metadata-Version: 2.1
 Name: plexflow
-Version: 0.0.64
+Version: 0.0.66
 Summary: A short description of the package.
 Home-page: https://www.example.com
 License: MIT
 Keywords: keyword1,keyword2,keyword3
 Author: Your Name
 Author-email: you@example.com
-Requires-Python: ==3.12.4
+Requires-Python: >=3.12,<3.13
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: apache-airflow (>=2.9.3,<3.0.0)
+Requires-Dist: apache-airflow (>=2.10.4,<3.0.0)
 Requires-Dist: apache-airflow-providers-cncf-kubernetes (>=8.3.3,<9.0.0)
 Requires-Dist: apache-airflow-providers-postgres (>=5.10.2,<6.0.0)
 Requires-Dist: apache-airflow-providers-redis (>=3.7.0,<4.0.0)
@@ -42,7 +43,8 @@ Requires-Dist: langchain-google-genai (>=1.0.8,<2.0.0)
 Requires-Dist: langchain-groq (>=0.1.9,<0.2.0)
 Requires-Dist: langcodes (>=3.4.0,<4.0.0)
 Requires-Dist: lxml (>=5.1.0,<6.0.0)
-Requires-Dist: numpy (>=2.0.1,<3.0.0)
+Requires-Dist: mistralai (>=1.2.5,<2.0.0)
+Requires-Dist: numpy (==2.0)
 Requires-Dist: parse-torrent-title (>=2.8.1,<3.0.0)
 Requires-Dist: playwright (>=1.41.2,<2.0.0)
 Requires-Dist: playwright-stealth (>=1.0.6,<2.0.0)

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/core/torrents/auto/auto_torrents.py

@@ -10,17 +10,17 @@ from plexflow.core.torrents.results.torrent import Torrent
 
 class AutoTorrents:
     @staticmethod
-    def movie(imdb_id: str = None, query: str = None, source: str = 'yts', **kwargs) -> List[Torrent]:
+    def movie(imdb_id: str = None, query: str = None, source: str = 'yts', headless: bool = True, **kwargs) -> List[Torrent]:
         if source == 'tpb':
-            return TPB(**kwargs).search(query=imdb_id)
+            return TPB(**kwargs).search(query=imdb_id, headless=headless, **kwargs)
         elif source == 'yts':
-            return YTS(**kwargs).search(query=imdb_id)
+            return YTS(**kwargs).search(query=imdb_id, headless=headless, **kwargs)
         elif source == 'torrentquest':
             return TorrentQuest(**kwargs).search(query=query)
         elif source == 'extratorrent':
             return ExtraTorrent(**kwargs).search(query=query)
         elif source == 'therarbg':
-            return TheRarbg(**kwargs).search(query=imdb_id)
+            return TheRarbg(**kwargs).search(query=imdb_id, headless=headless, **kwargs)
         elif source == 'ext':
             return Ext(**kwargs).search(query=query)
         elif source == 'snowfl':

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/core/torrents/providers/ext/ext.py

@@ -13,6 +13,6 @@ class Ext(AntibotRestful):
             'c': 'movies',
             'q': query
         })
-                        
+
         data = extract_torrent_results(html=capture.html)
         return list(map(lambda t: ExtSearchResult(**t), data))

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/core/torrents/providers/therarbg/therarbg.py

@@ -1,17 +1,17 @@
-from plexflow.utils.api.rest.antibot_restful import AntibotRestful
+from plexflow.utils.api.rest.plexful import Plexful
 from plexflow.utils.api.rest.restful import Restful
 from plexflow.core.torrents.providers.therarbg.utils import TheRarbgSearchResult
 from plexflow.utils.torrent.extract.therarbg import extract_torrent_results
 from typing import List
 
-class TheRarbg(Restful):
+class TheRarbg(Plexful):
     def __init__(self, base_url: str = 'https://therarbg.com'):
         super().__init__(base_url=base_url)
     
-    def search(self, query: str) -> List[TheRarbgSearchResult]:
+    def search(self, query: str, headless: bool = True, **kwargs) -> List[TheRarbgSearchResult]:
         response = self.get(f'/get-posts/keywords:{query}/')
         
         response.raise_for_status
         
-        data = extract_torrent_results(html=response.text)
+        data = extract_torrent_results(html=response.text if not headless else response.html)
         return list(map(lambda t: TheRarbgSearchResult(**t), data))

plexflow-0.0.66/plexflow/core/torrents/providers/tpb/tpb.py

@@ -0,0 +1,19 @@
+from plexflow.utils.api.rest.plexful import Plexful
+from plexflow.core.torrents.providers.tpb.utils import TPBSearchResult
+
+class TPB(Plexful):
+    def __init__(self, base_url: str = 'https://apibay.org'):
+        super().__init__(base_url=base_url)
+    
+    def search(self, query: str, headless: bool = True, **kwargs) -> list[TPBSearchResult]:
+        response = self.get('/q.php', query_params={
+            'q': query,
+        }, headless=headless, **kwargs)
+        
+        if headless:
+            response.raise_for_status()
+            data = response.json()
+        else:
+            data = response.json
+ 
+        return list(map(lambda x: TPBSearchResult(**x), data))

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/core/torrents/providers/yts/yts.py

@@ -1,18 +1,25 @@
-from plexflow.utils.api.rest.restful import Restful
+from plexflow.utils.api.rest.plexful import Plexful
 from plexflow.core.torrents.providers.yts.utils import YTSSearchResult
 
-class YTS(Restful):
+class YTS(Plexful):
     def __init__(self, base_url: str = 'https://yts.mx'):
         super().__init__(base_url=base_url)
     
-    def search(self, query: str):
+    def search(self, query: str, headless: bool = True, **kwargs) -> list[YTSSearchResult]:
+        if 'wait_value' not in kwargs:
+            kwargs['wait_value'] = 'Query was successful'
         response = self.get(url='/api/v2/list_movies.json', query_params={
             'query_term': query,
-        })
+        }, headless=headless, **kwargs)
         
-        response.raise_for_status()
-        
-        data = response.json()
+        if headless:
+            print(response.content)
+            
+            response.raise_for_status()
+            
+            data = response.json()
+        else:
+            data = response.json
         
         data = data.get("data", {})
         movies = data.get("movies", [])

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/utils/antibot/human_like_requests.py

@@ -8,6 +8,8 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.common.exceptions import TimeoutException
 from seleniumbase import SB
+from bs4 import BeautifulSoup
+import json
 
 class HumanLikeRequestCapture:
     def __init__(self, url, html, screenshot, cookies):
@@ -16,6 +18,14 @@ class HumanLikeRequestCapture:
         self.screenshot = screenshot
         self.cookies = cookies
 
+    @property
+    def json(self):
+        try:
+            soup = BeautifulSoup(self.html, 'html.parser')
+            return json.loads(soup.get_text())
+        except json.decoder.JSONDecodeError:
+            return None
+        
 class HumanLikeRequestSession:
     def __init__(self, use_xvfb=False):
         self.use_xvfb = use_xvfb

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/utils/api/context/http.py

@@ -1,21 +1,47 @@
 import requests
+from requests.adapters import HTTPAdapter
 from typing import Optional, Dict
+import dns.resolver
+
+class DNSHTTPAdapter(HTTPAdapter):
+    def __init__(self, dns_servers, *args, **kwargs):
+        self.dns_servers = dns_servers
+        super().__init__(*args, **kwargs)
+
+    def resolve(self, hostname):
+        resolver = dns.resolver.Resolver()
+        resolver.nameservers = self.dns_servers
+        answers = resolver.resolve(hostname, 'A')
+        return [answer.to_text() for answer in answers]
+
+    def send(self, request, **kwargs):
+        # Resolve the hostname using the custom DNS servers
+        hostname = request.url.split('/')[2].split(':')[0]
+        ip_addresses = self.resolve(hostname)
+        if ip_addresses:
+            request.url = request.url.replace(hostname, ip_addresses[0])
+        return super().send(request, **kwargs)
 
 class HttpRequestContext:
     """
     A base class for setting up a default request context for headers, params, etc.
-    
+
     Args:
         base_url (str): The base URL for the API.
         default_headers (dict, optional): The default headers for the API. Defaults to None.
         default_params (dict, optional): The default parameters for the API. Defaults to None.
-        
+        dns_servers (list, optional): A list of DNS servers to use for custom DNS resolution. Defaults to None.
+
     Attributes:
         session (requests.Session): The requests Session instance.
     """
-    
-    def __init__(self, base_url: str, default_headers: Optional[Dict[str, str]] = None, default_params: Optional[Dict[str, str]] = None):
+
+    def __init__(self, base_url: str, default_headers: Optional[Dict[str, str]] = None, default_params: Optional[Dict[str, str]] = None, dns_servers: Optional[list] = None):
         self.session = requests.Session()
+        if dns_servers:
+            adapter = DNSHTTPAdapter(list(dns_servers))
+            self.session.mount('http://', adapter)
+            self.session.mount('https://', adapter)
         self.session.headers.update(default_headers or {})
         self.session.params.update(default_params or {})
         self.base_url = base_url
@@ -58,5 +84,6 @@ class HttpRequestContext:
         # Reset headers and params to defaults after each request
         self.session.headers = self.default_headers or {}
         self.session.params = self.default_params or {}
-        
+
+        return response
         return response

plexflow-0.0.66/plexflow/utils/api/rest/plexful.py

@@ -0,0 +1,64 @@
+from typing import Optional, Dict, Any,Union
+import requests
+from plexflow.utils.api.context.http import HttpRequestContext
+import plexflow.utils.antibot.human_like_requests as human_like_requests
+from plexflow.utils.api.rest.url_builder import build_url
+
+class Plexful:
+    """
+    A class that uses UniversalHttpHook and UniversalPostgresqlHook to create RESTful API interfaces and interact with a PostgreSQL database.
+    
+    Args:
+        http_conn_id (str, optional): The connection ID, used as Airflow connection ID or as the name for the YAML file. Defaults to None.
+        postgres_conn_id (str, optional): The connection ID, used as Airflow connection ID or as the name for the YAML file. Defaults to None.
+        config_folder (str, optional): The folder where the YAML configuration file is located. Defaults to None.
+    """
+    
+    def __init__(self, base_url: str):
+        self._base_url = base_url
+
+    def get(self, url: str, headers: Optional[Dict[str, str]] = None, query_params: Optional[Dict[str, str]] = None, headless: bool = True, **kwargs) -> Union[requests.Response, human_like_requests.HumanLikeRequestCapture]:
+        """
+        Makes a GET request to the resource.
+        
+        Args:
+            url (str): The full URL for the GET request.
+            headers (dict, optional): The headers for the GET request. Defaults to None.
+            query_params (dict, optional): The query parameters for the GET request. Defaults to None.
+            **kwargs: Additional keyword arguments for the GET request.
+        
+        Returns:
+            The response from the GET request.
+        """
+        if headless:
+            context = HttpRequestContext(self._base_url)
+            return context.get(url, headers=headers, params=query_params, **kwargs)
+        else:
+            full_url = build_url(self._base_url, url, query_params)
+            print(full_url)
+            capture = human_like_requests.get(
+                url=full_url,
+                take_screenshot=kwargs.get('take_screenshot', False),
+                use_xvfb=kwargs.get('use_xvfb', False),
+                wait_condition=kwargs.get('wait_condition', 'regex'),
+                wait_value=kwargs.get('wait_value', r'magnet:'),
+                wait_until_not=kwargs.get('wait_until_not', False)
+            )
+            return capture
+
+    def post(self, url: str, data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, query_params: Optional[Dict[str, str]] = None, **kwargs) -> requests.Response:
+        """
+        Makes a POST request to the resource.
+        
+        Args:
+            url (str): The full URL for the POST request.
+            data (dict): The data for the POST request.
+            headers (dict, optional): The headers for the POST request. Defaults to None.
+            query_params (dict, optional): The query parameters for the POST request. Defaults to None.
+            **kwargs: Additional keyword arguments for the POST request.
+        
+        Returns:
+            The response from the POST request.
+        """
+        context = HttpRequestContext(self._base_url)
+        return context.post(url, headers=headers, params=query_params, **kwargs)

plexflow-0.0.66/plexflow/utils/api/rest/url_builder.py

@@ -0,0 +1,9 @@
+import urllib.parse
+
+def build_url(base_url: str, path: str, query_params: dict) -> str:
+    # Construct the full URL
+    url = urllib.parse.urljoin(base_url, path)
+    # Add query parameters
+    query_string = urllib.parse.urlencode(query_params if query_params else {})
+    full_url = f"{url}?{query_string}"
+    return full_url

plexflow-0.0.66/plexflow/utils/llm/mistral.py

@@ -0,0 +1,139 @@
+from mistralai import Mistral
+from io import StringIO
+from plexflow.utils.strings.json_extract import extract_json_from_string
+
+def extract_torrents(content: str, **kwargs):
+    api_key = kwargs.get("api_key", "bZ6vS0p30hiEqpyy6fBZUMyuZuiqaS91")
+    model = kwargs.get("model", "mistral-large-latest")
+
+    client = Mistral(api_key=api_key)
+
+    stream_response = client.chat.stream(
+        model = model,
+        messages = [
+            {
+                "role": "user",
+                "content": f"""
+                Parse the torrents as JSON, designate the total seeds as the field 'seeds', the total peers as the field 'peers', the date added as the field 'date', the torrent title as 'title', the torrent size as 'size', the torrent category as 'category', keep all other fields the same:
+
+                {content}
+                """,
+            },
+        ]
+    )
+
+    buffer = StringIO()
+
+    for chunk in stream_response:
+        chunk = chunk.data.choices[0].delta.content
+        if kwargs.get("log", False):
+            print(chunk, end="")
+        
+        buffer.write(chunk)
+    
+    json_data = buffer.getvalue()
+    
+    return extract_json_from_string(json_data)
+
+
+torrents = extract_torrents(
+    """
+
+
+Login
+Home
+Catalog
+Box Office
+New Selection
+4K Movies
+4K XXX
+ Home  Movies  TV  Games  Music  Anime  Apps  Other  Books  XXX  Pages
+XXX content
+Search title or IMDB ID like tt27932269 ...
+
+»
+Reset
+My 30 Books March 3 2 mature maturenl tt0051532 girlsoutwest Audiobooks heavier trip matrix language issue mummy The Office nacho libre the gentlemen playboyplus the flash 2023 The Comeback The Lord of the Rings twin peaks 720p the work 1080p vikings Windows All 7 8 1 10 11 All Editions With Updates AIO 46in1 glass 2019 Thunderbolts ma the invasion Becastled v0 6002 naughtyamericavr DesignOptimal Editable font and
+Mufasa: The Lion King 2024
+ Play
+
+Title: Mufasa: The Lion King
+EXTERNAL URL: IMDB | TMDB
+Languages: English
+Genres: Animation, Adventure, Drama, Family, Fantasy, Musical,
+Runtime: 2:00:00 Hour
+Cast: Aaron Pierre, Kelvin Harrison Jr., Tiffany Boone
+Directors: Barry Jenkins
+IMDB Rating Rating: 6.8 (1406 Votes)
+Year: 2024
+Summary: Mufasa, a cub lost and alone, meets a sympathetic lion named Taka, the heir to a royal bloodline. The chance meeting sets in motion an expansive journey of a group of misfits searching for their destiny.
+:TRAILER:
+
+Search table...
+C	File	Category	Added	Time	Size	Se.	Le.
+	
+Mufasa The Lion King 2024 720p WEBRiP x264 XoXo M.Q.A
+Movies	2025-02-02	8 hours	4.0 GB	26	50
+	
+Mufasa The Lion King 2024 1080p WEBRiP x264 XoXo M.Q.A
+Movies	2025-02-02	8 hours	8.4 GB	40	34
+	
+Mufasa The Lion King 2024 720p WEB DL 2000MB Feranki1980 M.Q.A
+Movies	2025-01-09	3 weeks, 2 days	2.0 GB	44	22
+	
+Mufasa The Lion King 2024 1080p WEBRip Multi AAC x265 HNY M.Q.A
+Movies	2025-01-02	1 month	1.5 GB	43	48
+	
+Mufasa The Lion King 2024 1080P WEBRiP 24 bit Stereo HEVC X265 POOTLED mkv M.Q.A
+Movies	2025-01-02	1 month	3.1 GB	23	33
+	
+Mufasa The Lion King 2024 1080p WEBRip x265 10bit YTS M.Q.A
+Movies	2025-01-02	1 month	1.7 GB	294	621
+	
+Mufasa The Lion King 2024 1080p WEBRIP H264 HAPPYNEWYEAR COLLECTi M.Q.A
+Movies	2025-01-02	1 month	3.7 GB	35	42
+	
+Mufasa The Lion King 2024 1080p WEBRip YTS M.Q.A
+Movies	2025-01-01	1 month	1.8 GB	424	1301
+	
+Mufasa Il Re Leone 2024 1080p HDRip HQ h264 ADS MD Ita iDN_Cr M.Q.A
+Movies	2025-01-01	1 month	2.5 GB	36	10
+	
+Mufasa The Lion King 2024 720p WEBRip YTS M.Q.A
+Movies	2025-01-01	1 month	1020.6 MB	422	796
+	
+Mufasa The Lion King 2024 1080p WEBRIP H264 HAPPYNEWYEAR COLLECTiVE M.Q.A
+Movies	2025-01-01	1 month	3.7 GB	45	37
+	
+Mufasa The Lion King 2024 1080p WEBRip READNFO x265 AC3 AOC M.Q.A
+Movies	2024-12-31	1 month	5.9 GB	13	13
+	
+Mufasa The Lion King 2024 1080p HDRip ENG HappyNEWYear M.Q.A
+Movies	2024-12-31	1 month	2.5 GB	95	714
+	
+Mufasa The Lion King 2024 1080p Cam X264 COLLECTiVE M.Q.A
+Movies	2024-12-23	1 month, 1 week	3.7 GB	27	21
+	
+Mufasa The Lion King 2024 V 2 1080p HDTS C1NEM4 M.Q.A
+Movies	2024-12-20	1 month, 2 weeks	2.1 GB	14	45
+	
+Mufasa The Lion King 2024 HDCAM c1nem4 x264 SUNSCREEN TGx M.Q.A
+Movies	2024-12-20	1 month, 2 weeks	995.1 MB	12	25
+	
+Mufasa The Lion King 2024 720p HDCAM C1NEM4 M.Q.A
+Movies	2024-12-20	1 month, 2 weeks	1.7 GB	14	19
+Showing 1 to 17 of 17 entries
+    """,
+    log=False
+)
+
+if len(torrents) == 0:
+    print("No torrents found")
+else:
+    if type(torrents[0]) == list:
+        torrents = torrents[0]
+    
+    print(len(torrents), "torrents found")
+
+    for torrent in torrents:
+        print(torrent)

{plexflow-0.0.64 → plexflow-0.0.66}/plexflow/utils/pubsub/produce.py

@@ -4,12 +4,26 @@ from confluent_kafka import Producer
 import os
 
 def delivery_report(err, msg):
+    """
+    Callback function to handle the delivery report of a message.
+
+    Args:
+        err: Error object if message delivery failed, otherwise None.
+        msg: Message object that was delivered.
+    """
     if err is not None:
         logging.error(f"Message delivery failed: {err}")
     else:
         logging.info(f"Message delivered to {msg.topic()} [{msg.partition()}] at offset {msg.offset()}")
 
 def produce_message(topic: str, message):
+    """
+    Produce a single message to the specified Kafka topic.
+
+    Args:
+        topic (str): The Kafka topic to publish the message to.
+        message: The message to be published.
+    """
     logging.info(f"Publishing message to {topic}...")
     producer = Producer({'bootstrap.servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')})
     producer.produce(topic, json.dumps(message).encode('utf-8'), callback=delivery_report)
@@ -17,9 +31,16 @@ def produce_message(topic: str, message):
     logging.info(f"Message published to {topic}.")
 
 def produce_messages(topic: str, messages):
+    """
+    Produce multiple messages to the specified Kafka topic.
+
+    Args:
+        topic (str): The Kafka topic to publish the messages to.
+        messages: A list of messages to be published.
+    """
     logging.info(f"Publishing messages to {topic}...")
     producer = Producer({'bootstrap.servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')})
     for message in messages:
         producer.produce(topic, json.dumps(message).encode('utf-8'), callback=delivery_report)
     producer.flush()
-    logging.info(f"All messages published to {topic}.")
+    logging.info(f"All messages published to {topic}.")