abstract-webtools 0.1.6.101__tar.gz → 0.1.6.103__tar.gz

{abstract_webtools-0.1.6.101 → abstract_webtools-0.1.6.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.101
+Version: 0.1.6.103
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.101 → abstract_webtools-0.1.6.103}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.101',
+    version='0.1.6.103',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.101 → abstract_webtools-0.1.6.103}/src/abstract_webtools/k2s_downloader.py

@@ -13,8 +13,9 @@ from abstract_security import *
 from abstract_webtools import *
 DOWNLOAD_DIR = os.path.abspath("./downloads")
 class K2SDownloader:
-    def __init__(self,env_path=None,download_dir=None):
+    def __init__(self,env_path=None,download_dir=None,json_file_path=None):
         self.download_dir = download_dir or DOWNLOAD_DIR
+        self.json_file_path = json_file_path
         os.makedirs(self.download_dir, exist_ok=True)
         self.env_path = env_path
         self.session = requests.Session()
@@ -95,18 +96,36 @@ class K2SDownloader:
         if 'filename=' in cd:
             return cd.split('filename=')[-1].strip('"')
         return url.split('/')[-1].split('?')[0]
+def get_json_key_value(json_data,key):
+    if json_data and isinstance(json_data,dict):
+        return json_data.get(key)
+def compare_keys(json_data,comp_json_data,key):
+    json_key_value = get_json_key_value(json_data,key)
+    comp_json_key_value = get_json_key_value(comp_json_data,key)
+    if json_key_value and comp_json_key_value and comp_json_key_value==json_key_value:
+        return True
+def check_json_data(json_list,new_data):
+    keys = ['k2s','link','name']
+    for json_data in json_list:
+        for key in keys:
+            result = compare_keys(json_data,new_data,key)
+            if result:
+                return result
 
 class dlsManager:
     def __init__(self, downloader):
         self.downloader = downloader
-        self.all_dls = []
-
+        self.json_file_path = self.downloader.json_file_path
+        all_dls= None
+        if self.json_file_path:
+            all_dls = safe_load_from_json(self.json_file_path)
+        self.all_dls = all_dls or  []
+        self.last_data = None
     def is_prev_dl(self, data):
-        k2s_link = data.get('k2s')
-        for prev_data in self.all_dls:
-            if prev_data.get('k2s') == k2s_link:
-                return True
-        self.all_dls.append(data)
+        if check_json_data(self.all_dls,data):
+            self.last_data = None
+            return True
+        self.last_data = data
         return False
 
     def dl_k2s_link(self, k2s_link):
@@ -114,6 +133,10 @@ class dlsManager:
             print(f"Downloading: {k2s_link}")
             self.downloader.download_file(k2s_link)
             time.sleep(10)
+            if self.json_file_path:
+                self.all_dls.append(self.last_data)
+                safe_dump_to_file(data=self.all_dls,
+                                  file_path=self.json_file_path)
 
 
 def get_soup(url):
@@ -128,3 +151,13 @@ def get_soup(url):
 def get_k2s_link(soup):
     match = re.search(r'https://k2s\.cc/file/[^"<]+', str(soup))
     return match.group(0) if match else None
+
+def get_sections_content(content,get_post_attribute,dls_mgr):
+    if not content:
+        return []
+    for section in content:
+        data = get_post_attribute(section)
+        if data and data.get('k2s') and not dls_mgr.is_prev_dl(data):
+            dls_mgr.dl_k2s_link(data['k2s'])
+            results.append(data)
+    return results

{abstract_webtools-0.1.6.101 → abstract_webtools-0.1.6.103}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.101
+Version: 0.1.6.103
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff