eaf-base-api 2.0__tar.gz → 2.1__tar.gz

{eaf_base_api-2.0/eaf_base_api.egg-info → eaf_base_api-2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eaf_base_api
-Version: 2.0
+Version: 2.1
 Summary: A base API for EchterAlsFake's Porn APIs
 Home-page: https://github.com/EchterAlsFake/eaf_base_api
 Author: Johannes Habel
@@ -10,9 +10,9 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests
 Requires-Dist: ffmpeg-progress-yield
 Requires-Dist: m3u8
+Requires-Dist: httpx
 
 # EAF Base API
 

{eaf_base_api-2.0 → eaf_base_api-2.1}/base_api/base.py

@@ -1,10 +1,10 @@
 import os
-import m3u8
 import time
+import m3u8
+import httpx
 import random
 import string
 import logging
-import requests
 import traceback
 import threading
 
@@ -67,13 +67,26 @@ class BaseCore:
     def __init__(self):
         self.last_request_time = time.time()
         self.total_requests = 0 # Tracks how many requests have been made
-        self.session = requests.Session()
-        self.session.headers.update(consts.HEADERS)
+        self.session = None
+        self.initialize_session()
+
 
     def update_user_agent(self):
         """Updates the User-Agent"""
         self.session.headers.update({"User-Agent": random.choice(consts.USER_AGENTS)})
 
+    def initialize_session(self):
+        verify = True
+
+        if consts.PROXY is not None:
+            verify = False
+
+        self.session = httpx.Client(proxy=consts.PROXY,
+                              headers=consts.HEADERS,
+                              verify=verify,
+                              timeout=consts.TIMEOUT
+                              )
+
     def enforce_delay(self):
         """Enforces the specified delay in consts.REQUEST_DELAY"""
         delay = consts.REQUEST_DELAY
@@ -85,67 +98,77 @@ class BaseCore:
                 logger.debug(f"Enforcing delay of {sleep_time:.2f} seconds.")
                 time.sleep(sleep_time)
 
-    def fetch(self, url: str, get_bytes: bool = False, stream: bool = False, timeout: int = consts.TIMEOUT,
-              get_response: bool = False, save_cache: bool = True, cookies: dict = None,
-              allow_redirects: bool = True) -> Union[bytes, str, requests.Response]:
+    def fetch(
+            self,
+            url: str,
+            get_bytes: bool = False,
+            timeout: int = consts.TIMEOUT,
+            get_response: bool = False,
+            save_cache: bool = True,
+            cookies: dict = None,
+            allow_redirects: bool = True,
+    ) -> Union[bytes, str, httpx.Response, None]:
         """
-        Fetches content in UTF-8 Text, Bytes or as a stream using multiple request attempts, support for proxies
-        and custom timeout.
+        Fetches content in UTF-8 Text, Bytes, or as a stream using multiple request attempts,
+        support for proxies and custom timeout.
         """
+        # Check cache first
         content = cache.handle_cache(url)
         if content is not None:
             logger.info(f"Fetched content for: {url} from cache!")
             return content
 
-        for attempt in range(1, consts.MAX_RETRIES):
-            if self.total_requests % 3 == 0:  # Change user agent after 3 requests to prevent bot detection
-                self.update_user_agent()
-
+        for attempt in range(1, consts.MAX_RETRIES + 1):
             try:
-                self.enforce_delay()
-                if consts.USE_PROXIES:
-                    verify = False  # Disable SSL verification
-                    url = url.replace("https://", "http://")  # Replace https to http to support HTTP proxies
+                # Update user agent periodically
+                if self.total_requests % 3 == 0:
+                    self.update_user_agent()
 
-                else:
-                    verify = True
+                self.enforce_delay()
 
-                response = self.session.get(url, stream=stream, proxies=consts.PROXIES, verify=verify,
-                                            timeout=timeout, allow_redirects=allow_redirects, cookies=cookies)
+                # Perform the request with stream handling
+                with self.session.stream("GET", url, timeout=timeout, cookies=cookies,
+                                         follow_redirects=allow_redirects) as response:
+                    self.total_requests += 1
 
-                if response.status_code != 200:
-                    logger.error(f"Unexpected status code {response.status_code} for URL: {url}")
+                    # Log and handle non-200 status codes
+                    if response.status_code != 200:
+                        logger.warning(
+                            f"Attempt {attempt}: Unexpected status code {response.status_code} for URL: {url}")
 
-                    if response.status_code == 404:
-                        logger.error("Website returned 404. Not an issue for xivdeos, but for all other sites")
-                        return response # Needed for xvideos. Means that video is unavailable lol
+                        if response.status_code == 404:
+                            logger.error("Resource not found (404). This may indicate the content is unavailable.")
+                            return None  # Return None for unavailable resources
 
-                    continue
+                        continue  # Retry for other non-200 status codes
 
-                if get_response:
-                    return response
+                    logger.debug(f"Attempt {attempt}: Successfully fetched URL: {url}")
 
-                self.total_requests += 1
+                    # Return response if requested
+                    if get_response:
+                        return response
 
-            except Exception:
-                error = traceback.format_exc()
-                logger.error(f"Could not fetch: {url} ->: {error}")
-                continue  # Next attempt
+                    # Process and return content
+                    if get_bytes:
+                        content = b"".join([chunk for chunk in response.iter_bytes()])
+                    else:
+                        content = "".join([chunk.decode("utf-8") for chunk in response.iter_bytes()])
+                        if save_cache:
+                            logger.debug(f"Saving content of {url} to local cache.")
+                            cache.save_cache(url, content)
 
-            logger.debug(f"[{attempt}|{consts.MAX_RETRIES}] Fetch ->: {url}")
+                    return content
 
-            logger.debug(f"[{attempt}|{consts.MAX_RETRIES}] Fetch ->: 200 | Success")
-            if get_bytes is False:
-                content = response.content.decode("utf-8")
-                if save_cache:
-                    logger.debug(f"Trying to save content of: {url} in local cache...")
-                    cache.save_cache(url, content)
+            except httpx.RequestError as e:
+                logger.error(f"Attempt {attempt}: Request error for URL {url}: {e}")
 
-                return content
+            except Exception:
+                logger.error(f"Attempt {attempt}: Unexpected error for URL {url}: {traceback.format_exc()}")
 
-            return response.content
+            logger.info(f"Retrying ({attempt}/{consts.MAX_RETRIES}) for URL: {url}")
 
-        raise ConnectionError(f"Could not fetch content for: {url}")
+        logger.error(f"Failed to fetch URL {url} after {consts.MAX_RETRIES} attempts.")
+        return None  # Return None if all attempts fail
 
     @classmethod
     def strip_title(cls, title: str) -> str:
@@ -243,7 +266,8 @@ class BaseCore:
 
         response = BaseCore().fetch(url, timeout=timeout, get_response=True)
         response.raise_for_status()
-        return url, response.content, True  # Success
+        content = b"".join(response.iter_bytes())
+        return url, content, True  # Success
 
     def threaded(self, max_workers: int = 20, timeout: int = 10):
         """
@@ -259,10 +283,10 @@ class BaseCore:
             completed, successful_downloads = 0, 0
 
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                # Map the last part of the URL (filename) to the future
-                future_to_hls_part = {executor.submit(self.download_segment, url, timeout): os.path.basename(url) for
-                                      url in
-                                      segments}
+                future_to_hls_part = {
+                    executor.submit(self.download_segment, url, timeout): os.path.basename(url)
+                    for url in segments
+                }
 
                 for future in as_completed(future_to_hls_part):
                     hls_part = future_to_hls_part[future]
@@ -271,26 +295,25 @@ class BaseCore:
                         completed += 1
                         if success:
                             successful_downloads += 1
-                        callback(completed, length)  # Update callback regardless of success to reflect progress
+                        callback(completed, length)  # Update progress callback
                     except Exception as e:
-                        raise e
+                        logger.error(f"Error processing segment {hls_part}: {e}")
 
-            # Writing only successful downloads to the file
+            # Write only successful segments to the output file
             with open(path, 'wb') as file:
                 for segment_url in segments:
-                    # Find the future object using the HLS part of the URL as the key
-                    matched_futures = [future for future, hls_part in future_to_hls_part.items() if
-                                       hls_part == os.path.basename(segment_url)]
+                    matched_futures = [
+                        future for future, hls_part in future_to_hls_part.items()
+                        if hls_part == os.path.basename(segment_url)
+                    ]
                     if matched_futures:
-                        logging.info("Got a match")
-                        future = matched_futures[0]  # Assuming unique HLS parts, take the first match
+                        future = matched_futures[0]
                         try:
                             _, data, success = future.result()
                             if success:
                                 file.write(data)
-                        except Exception:
-                            error = traceback.format_exc()
-                            logger.error(f"Exception while downloading segment: {error}")
+                        except Exception as e:
+                            logger.error(f"Exception writing segment {segment_url}: {e}")
 
         return wrapper
 
@@ -325,7 +348,7 @@ class BaseCore:
             "-i", new_url,  # Input URL
             "-bsf:a", "aac_adtstoasc",
             "-y",  # Overwrite output files without asking
-            "-c", "copy",  # Copy streams without reencoding
+            "-c", "copy",  # Copy streams without re-encoding
             path  # Output file path
         ]
 
@@ -340,34 +363,47 @@ class BaseCore:
 
         return False
 
-    def legacy_download(self, stream: bool, path: str, url: str, callback=None) -> bool:
-        response = self.fetch(url, stream=stream, get_bytes=True, get_response=True)
-        file_size = int(response.headers.get('content-length', 0))
+    def legacy_download(self, path: str, url: str, callback=None) -> bool:
+        """
+        Download a file using streaming, with support for progress updates.
+        """
+        try:
+            with self.session.stream("GET", url, timeout=30) as response:  # Use a reasonable timeout
+                response.raise_for_status()
 
-        if callback is None:
-            progress_bar = Callback()
+                file_size = int(response.headers.get('content-length', 0))
 
-        downloaded_so_far = 0
+                if callback is None:
+                    progress_bar = Callback()
 
-        if not os.path.exists(path):
-            with open(path, 'wb') as file:
-                for chunk in response.iter_content(chunk_size=1024):
-                    file.write(chunk)
-                    downloaded_so_far += len(chunk)
+                downloaded_so_far = 0
 
-                    if callback:
-                        callback(downloaded_so_far, file_size)
+                # Open file for writing
+                with open(path, 'wb') as file:
+                    for chunk in response.iter_bytes(chunk_size=1024):
+                        if not chunk:
+                            break  # End of stream
 
-                    else:
-                        progress_bar.text_progress_bar(downloaded=downloaded_so_far, total=file_size)
+                        file.write(chunk)
+                        downloaded_so_far += len(chunk)
 
-            if not callback:
-                del progress_bar
+                        # Update progress
+                        if callback:
+                            callback(downloaded_so_far, file_size)
+                        else:
+                            progress_bar.text_progress_bar(downloaded=downloaded_so_far, total=file_size)
 
-            return True
+                if not callback:
+                    del progress_bar
 
-        else:
-            raise FileExistsError("The file already exists.")
+                return True
+
+        except httpx.RequestError as e:
+            logger.error(f"Request failed for URL {url}: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error during download: {e}")
+            return False
 
     @classmethod
     def return_path(cls, video, args) -> str:

{eaf_base_api-2.0 → eaf_base_api-2.1}/base_api/modules/consts.py

@@ -5,8 +5,7 @@ MAX_RETRIES = 4
 REQUEST_DELAY = 0
 TIMEOUT = 20 # Maximal request timeout
 FFMPEG_PATH = "ffmpeg"
-USE_PROXIES = False
-PROXIES = {}
+PROXY = None
 USER_AGENTS = ["Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.1",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",

{eaf_base_api-2.0 → eaf_base_api-2.1/eaf_base_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eaf_base_api
-Version: 2.0
+Version: 2.1
 Summary: A base API for EchterAlsFake's Porn APIs
 Home-page: https://github.com/EchterAlsFake/eaf_base_api
 Author: Johannes Habel
@@ -10,9 +10,9 @@ Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGP
 Classifier: Programming Language :: Python
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests
 Requires-Dist: ffmpeg-progress-yield
 Requires-Dist: m3u8
+Requires-Dist: httpx
 
 # EAF Base API
 

{eaf_base_api-2.0 → eaf_base_api-2.1}/eaf_base_api.egg-info/requires.txt

@@ -1,3 +1,3 @@
-requests
 ffmpeg-progress-yield
 m3u8
+httpx

{eaf_base_api-2.0 → eaf_base_api-2.1}/setup.py

@@ -2,10 +2,9 @@ from setuptools import setup, find_packages
 
 setup(
     name="eaf_base_api",
-    version="2.0",
+    version="2.1",
     packages=find_packages(),
-    install_requires=[
-        "requests", "ffmpeg-progress-yield", "m3u8"
+    install_requires=["ffmpeg-progress-yield", "m3u8", "httpx"
     ],
     entry_points={
         'console_scripts': [