abstract-webtools 0.1.4.24__tar.gz → 0.1.4.25__tar.gz

{abstract_webtools-0.1.4.24 → abstract_webtools-0.1.4.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.4.24
+Version: 0.1.4.25
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.4.24 → abstract_webtools-0.1.4.25}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.4.24',
+    version='0.1.4.25',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.4.24 → abstract_webtools-0.1.4.25}/src/abstract_webtools/abstract_webtools.py

@@ -85,6 +85,7 @@ from bs4 import BeautifulSoup
 import xml.etree.ElementTree as ET
 from abstract_utilities.time_utils import get_time_stamp,get_sleep,sleep_count_down
 from abstract_utilities.string_clean import eatInner,eatAll
+import socket
 logging.basicConfig(level=logging.INFO)
 class DynamicRateLimiterManager:
     def __init__(self):
@@ -641,6 +642,50 @@ class SafeRequestSingleton:
 ##    if source_code:
 ##        print(source_code)
 ## ##
+class MySocketClient:
+    def __init__(self, ip_address, port,domain_name):
+        self.sock
+        self.ip_address= ip_address
+	self.port = port
+        self.sock.connect((host, port))
+        self.domain_name = 
+    def receive_data(self):
+        chunks = []
+        while True:
+            chunk = self.sock.recv(4096)
+            if chunk:
+                chunks.append(chunk)
+            else:
+                break
+        return b''.join(chunks).decode('utf-8')
+    def _parse_socket_response_as_json(self, data, *args, **kwargs):
+        return self._parse_json(data[data.find('{'):data.rfind('}') + 1], *args, **kwargs)
+    def process_data(self):
+        data = self.receive_data()
+        return self._parse_socket_response_as_json(data)
+    def _parse_json(self,json_string):
+        return json.loads(json_string)
+    def get_ip(self,domain=None):
+        try:
+            return self.sock.gethostbyname(domain_name if domain_name != None else self.domain_name)
+        except self.sock.gaierror:
+            return None
+   def grt_host_name(self,ip_address=None):
+   	return self.sock.gethostbyaddr(ip_address if ip_address != None else self.ip_address)
+   def toggle_sock(self):
+   	if self.sock != None:
+	    self.sock.close()
+   	else:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+class MySocketClient():
+    _instance = None
+    @staticmethod
+    def get_instance(ip_address='local_host',port=22,domain_name="example.com"):
+        if MySocketClientSingleton._instance is None:
+            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain_name=domain_name)
+        elif MySocketClientSingleton._instance.ip_address != ip_address or MySocketClientSingleton._instance.port != port or URLManagerSingleton._instance.domain_name != domain_name:
+            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain_name=domain_name)
+        return MySocketClient
 class URLManager:
     def __init__(self,url=None,session=requests):
         self.url = url
@@ -648,8 +693,15 @@ class URLManager:
         self.striped_url = None if url ==  None else self.strip_web()
         self.clean_urls = None if url ==  None else self.clean_url(url=self.url)
         self.correct_url = None if url ==  None else self.get_correct_url()
-        self.domain_name = None if url ==  None else self.get_domain_name(self.correct_url)
+        if self.correct_url != None:
+        	self.url = self.correct_url
+        self.protocol=None
+	self.domain_name= None if url ==  None else self.get_domain_name(self.correct_url)
+	self.path=None
+	self.query=None
+	self.strip_web()
         self.all_urls=[]
+     
     def strip_web(self) -> str:
         """
         Strip the 'http://' or 'https://' prefix from a URL, if present.
@@ -666,6 +718,13 @@ class URLManager:
         elif self.url.startswith("https://"):
             url = self.url.replace("https://", '', 1)
         return url
+    def url_to_pieces(self):
+	    match = re.match(r'^(https?):\/\/([^\/]+)(\/[^?]+)?(\?.+)?', self.correct_url)
+	    if match:
+		self.protocol = match.group(1)
+		self.domain = match.group(2)
+		self.path = match.group(3) if match.group(3) else ""  # Handle None
+		self.query = match.group(4) if match.group(4) else ""  # Handle None
     @staticmethod
     def clean_url(url: str) -> list:
         """
@@ -780,17 +839,20 @@ class URLManagerSingleton:
         return URLManagerSingleton._instance
 
 class VideoDownloader:
-    
     def __init__(self, url,title=None,download_directory=os.getcwd(),user_agent=None,video_extention='mp4',download=True,get_info=False):
         self.url = url
+        self.download = download
+        self.get_info = get_info
+        self.user_agent=user_agent
+        self.download_directory=download_directory
         self.video_extention=video_extention
         self.header = UserAgentManagerSingleton().get_instance(user_agent=user_agent).user_agent_header
         self.base_name = os.path.basename(self.url)
         self.file_name,self.ext = os.path.splitext(self.base_name)
-        self.download_directory=download_directory
         self.title = url.split('/')[3] if title == None else title
-        self.video_urls = []
+        self.video_urls = [self.url]
         self.fetch_video_urls()
+        self.info={}
         self.download_videos()
     def fetch_video_urls(self):
         driver = webdriver.Chrome()
@@ -803,12 +865,12 @@ class VideoDownloader:
         for video_url in self.video_urls:
             ydl_opts = {}
             with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                info = ydl.extract_info(self.url,download=download)
-                if get_info == True:
-                	return info
-                self.base_name = os.path.basename(info['url'])
+                self.info = ydl.extract_info(url=video_url,download=self.download)
+                if self.get_info == True:
+                     return self.info
+                self.base_name = os.path.basename(self.info['url'])
                 self.file_name,self.ext = os.path.splitext(self.base_name)
-                video_content =SafeRequestSingleton().get_instance(url=info['url']).response
+                video_content =SafeRequestSingleton().get_instance(url=self.info['url']).response
                 print("Start downloading")
                 content_length = int(video_content.headers['content-length'])
                 print(f'Size: {content_length / 1024 / 1024:.2f}MB')
@@ -824,9 +886,9 @@ class VideoDownloaderSingleton():
     @staticmethod
     def get_instance(url,title=None,video_extention='mp4',download_directory=os.getcwd(),user_agent=None,download=True,get_info=False):
         if VideoDownloaderSingleton._instance is None:
-            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info)
+            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
         elif VideoDownloaderSingleton._instance.title != title or video_extention != VideoDownloaderSingleton._instance.video_extention or url != VideoDownloaderSingleton._instance.url or download_directory != VideoDownloaderSingleton._instance.download_directory or user_agent != VideoDownloaderSingleton._instance.user_agent:
-            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info)
+            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
         return VideoDownloaderSingleton._instance
 class SoupManager:
     def __init__(self, url=None, source_code=None, parse_type="html.parser"):

{abstract_webtools-0.1.4.24 → abstract_webtools-0.1.4.25}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract-webtools
-Version: 0.1.4.24
+Version: 0.1.4.25
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff