abstract-webtools 0.1.5.81__py3-none-any.whl → 0.1.5.83__py3-none-any.whl

abstract_webtools/managers/videoDownloader.py

@@ -0,0 +1,205 @@
+import os
+class VideoDownloader:
+    """
+    VideoDownloader is a class for downloading videos from URLs using YouTube-DL.
+
+    Args:
+        link (str or list): The URL(s) of the video(s) to be downloaded.
+        temp_directory (str or None): The directory to store temporary video files (default is None, uses video_directory/temp_files).
+        video_directory (str or None): The directory to store downloaded videos (default is None, uses 'videos' in the current working directory).
+        remove_existing (bool): Whether to remove existing video files with the same name (default is True).
+
+    Methods:
+        count_outliers(speed, threshold): Count speed outliers below the threshold.
+        filter_outliers(speeds): Filter out speed outliers in the list of speeds.
+        remove_temps(file_name): Remove temporary video files based on the file name.
+        move_video(): Move the downloaded video to the final directory.
+        yt_dlp_downloader(url, ydl_opts={}, download=True): Download video information using YouTube-DL.
+        progress_callback(d): Callback function to monitor download progress.
+        download(): Download video(s) based on the provided URL(s).
+        monitor(): Monitor the download progress.
+        start(): Start the download and monitoring threads.
+
+    Note:
+        - The VideoDownloader class uses YouTube-DL to download videos.
+        - It allows downloading from multiple URLs.
+        - You need to have YouTube-DL installed to use this class.
+    """
+    def __init__(self, link,temp_directory=None,video_directory=None,remove_existing=True):
+        if video_directory==None:
+            video_directory=os.path.join(os.getcwd(),'videos')
+        if temp_directory == None:
+            temp_directory=os.path.join(video_directory,'temp_files')
+        self.thread_manager = ThreadManager()
+        self.pause_event = self.thread_manager.add_thread('pause_event')
+        self.link = link
+        self.temp_directory = temp_directory
+        self.video_directory = video_directory
+        self.remove_existing=remove_existing
+        self.video_urls=self.link if isinstance(self.link,list) else [self.link]
+        self.starttime = None
+        self.downloaded = 0
+        self.time_interval=60
+        self.monitoring=True
+        self.temp_file_name = None
+        self.file_name = None
+        self.dl_speed = None
+        self.dl_eta=None
+        self.total_bytes_est=None
+        self.percent_speed=None
+        self.percent=None
+        self.speed_track = []
+        self.video_url=None
+        self.last_checked = get_time_stamp()
+        self.num=0
+        self.start()
+    def count_outliers(self,speed,threshold):
+        if speed < threshold:
+            self.outlier_count+=1
+        else:
+            self.outlier_count=0
+    def filter_outliers(self,speeds):
+        # Step 1: Compute initial average
+        initial_avg = sum(speeds) / len(speeds)
+        
+        # Step 2: Remove speeds 25% under the average
+        threshold = initial_avg * 0.75  # 25% under average
+        filtered_speeds = [speed for speed in speeds if speed >= threshold]
+        
+        # Step 3: Compute the new average of the filtered list
+        if filtered_speeds:  # Ensure the list is not empty
+            self.count_outliers(speeds[-1],threshold)
+            return filtered_speeds
+        else:
+            # This can happen if all values are outliers, it's up to you how to handle it
+            self.outlier_count=0
+            return speeds
+    def remove_temps(self,file_name):
+        for temp_vid in os.listdir(self.temp_directory):
+            if len(file_name)<=len(temp_vid):
+                if temp_vid[:len(file_name)] == file_name:
+                    os.remove(os.path.join(self.temp_directory,temp_vid))
+                    print(f"removing {temp_vid} from {self.temp_directory}")
+    def move_video(self):
+        if os.path.exists(self.temp_file_path):
+            shutil.move(self.temp_file_path, self.video_directory)
+            print(f"moving {self.file_name} from {self.temp_directory} to {self.video_directory}")
+            self.remove_temps(self.file_name)
+            return True
+        if os.path.exists(self.complete_file_path):
+            print(f"{self.file_name} already existed in {self.video_directory}; removing it from {self.temp_directory}")
+            self.remove_temps(self.file_name)
+            return True
+        return False
+    def yt_dlp_downloader(self,url,ydl_opts={},download=True):
+        try:
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                self.info_dict=ydl.extract_info(url=url, download=download)
+            return True
+        except:
+            return False
+    def progress_callback(self, d):
+        self.status_dict = d
+        keys = ['status',
+                'downloaded_bytes',
+                'fragment_index',
+                'fragment_count',
+                'filename',
+                'tmpfilename',
+                'max_progress',
+                'progress_idx',
+                'elapsed',
+                'total_bytes_estimate',
+                'speed',
+                'eta',
+                '_eta_str',
+                '_speed_str',
+                '_percent_str',
+                '_total_bytes_str',
+                '_total_bytes_estimate_str',
+                '_downloaded_bytes_str',
+                '_elapsed_str',
+                '_default_template']
+        if self.status_dict['status'] == 'finished':
+            print("Done downloading, moving video to final directory...")
+            self.move_video()
+            return
+        if get_time_stamp()-self.last_checked>5:
+            print(self.status_dict['_default_template'])
+            self.last_checked = get_time_stamp()
+            if (get_time_stamp()-self.start_time/5)>6:
+                self.speed_track.append(self.status_dict['speed'])
+                self.speed_track=self.filter_outliers(self.speed_track)
+                
+    def download(self):
+        if not os.path.exists(self.video_directory):
+            os.makedirs(self.video_directory,exist_ok=True)
+        if not os.path.exists(self.temp_directory):
+            os.makedirs(self.temp_directory,exist_ok=True)
+        for self.num,video_url in enumerate(self.video_urls):
+            if video_url != self.video_url or self.video_url == None:
+                self.video_url=video_url
+                self.info_dict=None
+                result = self.yt_dlp_downloader(url=self.video_url,ydl_opts={'quiet': True, 'no_warnings': True},download=False)
+                if self.info_dict != None and result:
+                    self.start_time = get_time_stamp()
+                    self.downloaded = 0
+                    self.video_title = self.info_dict.get('title', None)
+                    self.video_ext = self.info_dict.get('ext', 'mp4')
+                    self.file_name =f"{self.video_title}.{self.video_ext}"
+                    self.temp_file_path = os.path.join(self.temp_directory, self.file_name)
+                    self.complete_file_path = os.path.join(self.video_directory, self.file_name)
+                    if not self.move_video():
+                        self.dl_speed = []
+                        self.percent=None
+                        self.dl_eta=None
+                        self.total_bytes_est=None
+                        self.percent_speed=None
+                        self.speed_track = []
+                        self.outlier_count=0
+                        ydl_opts = {
+                            'outtmpl': self.temp_file_path,
+                            'noprogress':True,
+                            'progress_hooks': [self.progress_callback]  
+                        }
+                        
+                       
+                        print("Starting download...")  # Check if this point in code is reached
+                        result = self.yt_dlp_downloader(url=self.video_url,ydl_opts=ydl_opts,download=True)
+                        if result:
+                            print("Download finished!")  # Check if download completes
+                        else:
+                            print(f'error downloding {self.video_url}')
+                        self.move_video()
+                    else:
+                        print(f"The video from {self.video_url} already exists in the directory {self.video_directory}. Skipping download.")
+                else:
+                    print(f"could not find video info from {self.video_url} Skipping download.")
+        if self.num==len(self.video_urls)-1:
+            self.monitoring=False
+            self.time_interval=0
+            
+    def monitor(self):
+        while self.monitoring:
+            self.thread_manager.wait(name='pause_event',n=self.time_interval)# check every minute
+            if self.monitoring:
+                if 'eta' in self.status_dict:
+                    if self.outlier_count>=3 and (self.status_dict['eta']/60)>10:
+                        self.start()
+
+    def start(self):
+        download_thread = self.thread_manager.add_thread(name='download_thread',target=self.download)
+        monitor_thread = self.thread_manager.add_thread(name='monitor_thread',target_function=self.monitor)
+        self.thread_manager.start(name='download_thread')
+        self.thread_manager.start(name='monitor_thread')
+        self.thread_manager.join(name='download_thread')
+        self.thread_manager.join(name='monitor_thread')
+class VideoDownloaderSingleton():
+    _instance = None
+    @staticmethod
+    def get_instance(url_manager,request_manager,title=None,video_extention='mp4',download_directory=os.getcwd(),user_agent=None,download=True,get_info=False):
+        if VideoDownloaderSingleton._instance is None:
+            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
+        elif VideoDownloaderSingleton._instance.title != title or video_extention != VideoDownloaderSingleton._instance.video_extention or url != VideoDownloaderSingleton._instance.url or download_directory != VideoDownloaderSingleton._instance.download_directory or user_agent != VideoDownloaderSingleton._instance.user_agent:
+            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
+        return VideoDownloaderSingleton._instance

abstract_webtools/url_grabber.py

@@ -1,7 +1,7 @@
 from abstract_gui import make_component,sg
 import inspect
 import re
-from . import UserAgentManager,UrlManager,SafeRequest,SoupManager,LinkManager,CipherManager,requests,ssl,BeautifulSoup,HTTPAdapter,PoolManager,ssl_
+from .managers import *
 window = None
 
 def get_attrs(values):

{abstract_webtools-0.1.5.81.dist-info → abstract_webtools-0.1.5.83.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: abstract-webtools
-Version: 0.1.5.81
+Name: abstract_webtools
+Version: 0.1.5.83
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: PySimpleGUI (>=4.60.5)
-Requires-Dist: abstract-utilities (>=0.2.2.30)
-Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.0.4)
+Requires-Dist: abstract-utilities >=0.2.2.30
+Requires-Dist: PySimpleGUI >=4.60.5
+Requires-Dist: urllib3 >=2.0.4
+Requires-Dist: requests >=2.31.0
 
 # Abstract WebTools
 Provides utilities for inspecting and parsing web content, including React components and URL utilities, with enhanced capabilities for managing HTTP requests and TLS configurations.

abstract_webtools-0.1.5.83.dist-info/RECORD

@@ -0,0 +1,28 @@
+abstract_webtools/__init__.py,sha256=rPt-BXa6ksplRXmZoJtn-vMGEPZGpEwfxqqOqZo_EWk,68
+abstract_webtools/abstract_webtools.py,sha256=6pYoObMhvOnjLiw1oQaNBoX3ipr7QUJYve0YEjoXvC8,3813
+abstract_webtools/big_user_agent_list.py,sha256=5ZkrUWmfzYL5yaULREslh9ZiRQeITbSjqZlp2KQON3w,131923
+abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,1329
+abstract_webtools/soup_gui.py,sha256=n95YAps1R6DpMwR4UbthSqQby0C5WHUa9tsW-f2qpLg,5184
+abstract_webtools/url_grabber.py,sha256=AJN8F6TQ_Kwv3sd8YWXffmC4EHRRrKizuJY1uEXNMlM,10752
+abstract_webtools/url_grabber_new.py,sha256=Oh2Kc0gBScCo0xpopNsg8JE5lIbPuzZVKM5f5GoZmw0,3454
+abstract_webtools/managers/__init__.py,sha256=5aIpbdUsDWTrhPUAjfIKnG54OULqOKan9LBL5EIUllo,407
+abstract_webtools/managers/cipherManager.py,sha256=NHQGdR11eNSm-1H-GezD5dyQgsPTJwY5kczt8Sher2s,1621
+abstract_webtools/managers/crawlManager.py,sha256=8ico7qFM8sXluIx3Rzgr5TZMmwsb_bnx2eYoS2Vss6I,7661
+abstract_webtools/managers/domainManager.py,sha256=dluzexs4GdI248kXA7sRaRUm020_dGO86ZLo20xLWOI,1788
+abstract_webtools/managers/dynamicRateLimiter.py,sha256=gopQcQo50JG2D0KcyepNCIQ_1uDQEBIHBzWf4R2Wgy0,7617
+abstract_webtools/managers/linkManager.py,sha256=m6y9s8jknrTX8RtOAFKeHd4yd23G7Rgf0T7Sp7wmHUw,12180
+abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
+abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
+abstract_webtools/managers/requestManager.py,sha256=ZAood3uhs6mi--R8RLTtcNFxACKhu-Jf3m-u4fFMY2Q,17029
+abstract_webtools/managers/seleniumManager.py,sha256=toG0aM72UO1LMicXGSVm3PCf88yQLJqLpFEMNT9gLU0,3014
+abstract_webtools/managers/soupManager.py,sha256=03qfg4Ww6T3F9HP0F2RoqxZ-jBFhnwE-lS7m3EOcRCI,14717
+abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQqdwdTs0H4ecc,1370
+abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
+abstract_webtools/managers/urlManager.py,sha256=OtEBV26RHHuxtbYLLX56AlMv83s9kOIcsDvlMZ9bKZc,8647
+abstract_webtools/managers/userAgentManager.py,sha256=33SB2p2FG7EYZl7l2iYm1U4gI9PcdkGTZHw5lg_Ogrw,1653
+abstract_webtools/managers/videoDownloader.py,sha256=6G_aLc05BTMUYUWc7iqYtHF_BaR7DnCNK_NJ-QnjsYY,10531
+abstract_webtools-0.1.5.83.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.5.83.dist-info/METADATA,sha256=HDnvNqx5Ho0uLgBRSBWh_4TeV5E6tGyOLtSH1841TTw,15858
+abstract_webtools-0.1.5.83.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+abstract_webtools-0.1.5.83.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.5.83.dist-info/RECORD,,

{abstract_webtools-0.1.5.81.dist-info → abstract_webtools-0.1.5.83.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

abstract_webtools-0.1.5.81.dist-info/RECORD

@@ -1,12 +0,0 @@
-abstract_webtools/__init__.py,sha256=VtajT5HAiewyI3wKcKU-r5OVDDyMjYqLWVvTmPn8ToY,80
-abstract_webtools/abstract_webtools.py,sha256=gX-jWctjrzx1IJUg8y8dVzSZYWyX0jve4qS4ZthnL0A,90125
-abstract_webtools/big_user_agent_list.py,sha256=5ZkrUWmfzYL5yaULREslh9ZiRQeITbSjqZlp2KQON3w,131923
-abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,1329
-abstract_webtools/soup_gui.py,sha256=n95YAps1R6DpMwR4UbthSqQby0C5WHUa9tsW-f2qpLg,5184
-abstract_webtools/url_grabber.py,sha256=W8nXAu-b3Ywn0ihmN78X1D6vjvMwfgX0Ry-FP5YDT4U,10876
-abstract_webtools/url_grabber_new.py,sha256=Oh2Kc0gBScCo0xpopNsg8JE5lIbPuzZVKM5f5GoZmw0,3454
-abstract_webtools-0.1.5.81.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.5.81.dist-info/METADATA,sha256=Fd-vyebRGojkrOjc6udWELuyDR3gbrD_6Weund2w_nQ,15866
-abstract_webtools-0.1.5.81.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
-abstract_webtools-0.1.5.81.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.5.81.dist-info/RECORD,,