py2ls 0.1.6.7__py3-none-any.whl → 0.1.6.8__py3-none-any.whl

py2ls/ips.py

@@ -1022,15 +1022,30 @@ def fupdate(fpath, content=None):
     with open(fpath, 'w') as file:
         file.write(content)
         file.write(old_content)
+def fappend(fpath, content=None):
+    """
+    append new content at the end.
+    """
+    content = content or ""
+    if os.path.exists(fpath):
+        with open(fpath, 'r') as file:
+            old_content = file.read()
+    else:
+        old_content = '' 
+        
+    with open(fpath, 'w') as file:
+        file.write(old_content)
+        file.write(content)
         
 def fsave(
     fpath,
     content,
+    mode='w',
+    how ='overwrite',
     kind=None,
     font_name="Times",
     font_size=10,
     spacing=6,
-    mode='w',
     **kwargs,
 ):
     """
@@ -1046,9 +1061,14 @@ def fsave(
     Returns:
         None
     """
-    def save_content(fpath, content, mode=mode): 
-        with open(fpath, mode, encoding='utf-8') as file:
-            file.write(content)
+    def save_content(fpath, content, mode=mode, how='overwrite'): 
+        if 'wri' in how.lower():
+            with open(fpath, mode, encoding='utf-8') as file:
+                file.write(content)
+        elif 'upd' in how.lower():
+            fupdate(fpath, content=content)
+        elif 'app' in how.lower():
+            fappend(fpath, content=content)
 
 
     def save_docx(fpath, content, font_name, font_size, spacing):

py2ls/netfinder.py

@@ -1,6 +1,7 @@
 from bs4 import BeautifulSoup
 import requests
 from requests.utils import dict_from_cookiejar
+from requests.exceptions import ChunkedEncodingError, ConnectionError
 import os
 from urllib.parse import urlparse, urljoin
 import base64
@@ -150,7 +151,7 @@ def flatten_json(y):
 def get_proxy():
     list_ = []
     headers = {"User-Agent": user_agent()}
-    response = requests.get("https://free-proxy-list.net", headers=headers)
+    response = requests.get("https://free-proxy-list.net", headers=headers,timeout=30,stream=True)
     content = BeautifulSoup(response.content, "html.parser")
     info = extract_text_from_content(content, where="td", extend=0)[0].split()
     count, pair_proxy = 0, 2
@@ -200,18 +201,18 @@ def fetch_all(url, parser="lxml", driver='request', # request or selenium
         
         headers = {"User-Agent": user_agent()}
         if 'req' in driver.lower():
-            response = requests.get(url, headers=headers,proxies=proxies_glob)
+            response = requests.get(url, headers=headers,proxies=proxies_glob,timeout=30,stream=True)
 
             # If the response is a redirect, follow it
             while response.is_redirect:
                 logger.info(f"Redirecting to: {response.headers['Location']}")
-                response = requests.get(response.headers["Location"], headers=headers,proxies=proxies_glob)
+                response = requests.get(response.headers["Location"], headers=headers,proxies=proxies_glob,timeout=30,stream=True)
             # Check for a 403 error
             if response.status_code == 403:
                 logger.warning("403 Forbidden error. Retrying...")
                 # Retry the request after a short delay
                 sleep(random.uniform(1, 3))
-                response = requests.get(url, headers=headers,proxies=proxies_glob)
+                response = requests.get(url, headers=headers,proxies=proxies_glob,timeout=30,stream=True)
                 # Raise an error if retry also fails
                 response.raise_for_status()
 
@@ -471,7 +472,7 @@ def pdf_detector(url, contains = None, dir_save = None, booster = False):
             idx += 1
         print(f'{len(fnames)} files are downloaded:\n{fnames}\n to local: \n{dir_save}')
 
-def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=False, booster=False,verbose=True):
+def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=False, booster=False,verbose=True, timeout=30, n_try=3,timestamp=False):
     if verbose:
         print("usage: downloader(url, dir_save=None, kind=['.pdf','xls'], contains=None, booster=False)")
     def fname_corrector(fname, ext):
@@ -482,17 +483,21 @@ def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=F
         return fname
     def check_and_modify_filename(directory, filename):
         base, ext = os.path.splitext(filename)
-        counter = 2
+        counter = 1
         new_filename = filename
         while os.path.exists(os.path.join(directory, new_filename)):
-            new_filename = f"{base}_{counter}{ext}"
+            if counter<=9:
+                counter_='0'+str(counter)
+            else:
+                counter_=str(counter)
+            new_filename = f"{base}_{counter_}{ext}"
             counter += 1
         return new_filename
     if not isinstance(kind,list):
         kind=[kind]
     if isinstance(url, list):
         for url_ in url:
-            downloader(url_, dir_save=dir_save, kind=kind, contains=contains, booster=booster,verbose=verbose)
+            downloader(url_, dir_save=dir_save, kind=kind, contains=contains, booster=booster,verbose=verbose,timeout=timeout,n_try=n_try,timestamp=timestamp)
             # sleep(random.uniform(1, 3))
     for i,k in enumerate(kind):
         if not k.startswith('.'):
@@ -544,25 +549,45 @@ def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=F
         fnames = [file_link.split("/")[-1] for file_link in file_links_all]
         for idx, file_link in enumerate(file_links_all):
             headers = {"User-Agent": user_agent()}
-            response = requests.get(file_link, headers=headers)
-            if response.status_code == 200:
-                ext = next((ftype for ftype in kind if ftype in file_link), None)
-                if ext:
-                    corrected_fname = fname_corrector(fnames[idx], ext)
-                    corrected_fname = check_and_modify_filename(dir_save, corrected_fname)
-                    with open(os.path.join(dir_save, corrected_fname), "wb") as file:
-                        file.write(response.content)
-                    if verbose:
-                        print(f"Done! {fnames[idx]}")
-                else:
-                    if verbose:
-                        print(f"Unknown file type for {file_link}")
-            else:
-                if verbose:
-                    print(f"Failed to download file: {response.status_code}")
+            itry = 0 # Retry logic with exception handling
+            while itry < n_try:
+                try:
+                    # streaming to handle large files and reduce memory usage.
+                    response = requests.get(file_link, headers=headers, timeout=timeout, stream=True)
+                    if response.status_code == 200:
+                        ext = next((ftype for ftype in kind if ftype in file_link), None)
+                        if ext:
+                            corrected_fname = fname_corrector(fnames[idx], ext)
+                            corrected_fname = check_and_modify_filename(dir_save, corrected_fname)
+                            if timestamp:
+                                corrected_fname=datetime.now().strftime("%y%m%d_%H%M%S_")+corrected_fname
+                            fpath_tmp = os.path.join(dir_save, corrected_fname)
+                            with open(fpath_tmp, "wb") as file:
+                                for chunk in response.iter_content(chunk_size=8192):
+                                    if chunk:  # Filter out keep-alive chunks
+                                        file.write(chunk)
+                            if verbose:
+                                print(f"Done! {fnames[idx]}")
+                        else:
+                            if verbose:
+                                print(f"Unknown file type for {file_link}")
+                        break  # Exit the retry loop if successful
+                    else:
+                        if verbose:
+                            print(f"Failed to download file: HTTP status code {response.status_code}")
+                except (ChunkedEncodingError, ConnectionError) as e:
+                    print(f"Attempt {itry+1} failed: {e}. Retrying in a few seconds...")
+                    # time.sleep(random.uniform(0, 2))  # Random sleep to mitigate server issues
+                    if os.path.exists(fpath_tmp):
+                                os.remove(fpath_tmp)
+                    itry += 1
+
+            if itry == n_try:
+                print(f"Failed to download {file_link} after {n_try} attempts.")
+
         print(f'\n{len(fnames)} files were downloaded:')
         if verbose:
-            pp(fnames)
+            pp(corrected_fname) if corrected_fname in locals() else pp(fnames)
             print(f"\n\nsaved @:\n{dir_save}")
         
 def find_img(url, driver='request',dir_save="images", rm_folder=False, verbose=True):

{py2ls-0.1.6.7.dist-info → py2ls-0.1.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.6.7
+Version: 0.1.6.8
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

{py2ls-0.1.6.7.dist-info → py2ls-0.1.6.8.dist-info}/RECORD

@@ -134,14 +134,14 @@ py2ls/db2ls.py,sha256=MMfFX47aIPIyu7fU9aPvX9lbPRPYOpJ_VXwlnWk-8qo,13615
 py2ls/doc.py,sha256=xN3g1OWfoaGUhikbJ0NqbN5eKy1VZVvWwRlhHMgyVEc,4243
 py2ls/export_requirements.py,sha256=psZtSe-MOD9L_w3dVpA_VJEKfq3J914g3Y1OtRNAb4g,2324
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
-py2ls/ips.py,sha256=B88jol_EYZ4IERWAeabkHSwMkGytqyBheEbbpM_YDiQ,86146
-py2ls/netfinder.py,sha256=ZsLWGYMeRuGvxj2nqE0Z8ANoaVl18Necfw0HQfh2q7I,45548
+py2ls/ips.py,sha256=jyvui0X41-AyZfpcujfgBMt6NsQNfyh_SiQ5lQ0gk1Q,86744
+py2ls/netfinder.py,sha256=aOrgXp2rqpUDREZMlP_875SuAAcQXu3lhnRMk1cPG5M,47269
 py2ls/plot.py,sha256=8_33-1wpkGZrDUuvRBfTPUi_BRKdf1njoR725OLSLSY,48579
 py2ls/setuptools-70.1.0-py3-none-any.whl,sha256=2bi3cUVal8ip86s0SOvgspteEF8SKLukECi-EWmFomc,882588
 py2ls/sleep_events_detectors.py,sha256=36MCuRrpurn0Uvzpo3p3b3_JlVsRNHSWCXbJxCGM3mg,51546
 py2ls/stats.py,sha256=Wd9yCKQ_61QD29WMEgMuEcreFxF91NmlPW65iWT2B5w,39041
 py2ls/translator.py,sha256=6S7MmTZmjj8NljVmj0W5uEauu4ePxso3AMf2LvGVRQA,30516
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.1.6.7.dist-info/METADATA,sha256=-gKMv_eCD4spLGvNAidRLyfI07hf1eq_ldQsfFmxlsA,20998
-py2ls-0.1.6.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-py2ls-0.1.6.7.dist-info/RECORD,,
+py2ls-0.1.6.8.dist-info/METADATA,sha256=yQ6DRgZ13Lc-Hw3FhI0dbn-nE0N3JwbC9X_8rQlxILQ,20998
+py2ls-0.1.6.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+py2ls-0.1.6.8.dist-info/RECORD,,