py2ls 0.1.6.6__py3-none-any.whl → 0.1.6.8__py3-none-any.whl

py2ls/export_requirements.py

@@ -1,33 +1,71 @@
 import os
 from datetime import datetime
 import subprocess
+import re
 
-def export_requirements(fpath="/Users/macjianfeng/Dropbox/github/python/py2ls/"):
+def export_requirements(fpath="/Users/macjianfeng/Dropbox/github/python/py2ls/", exclude=["py2ls","jaraco"]):
     """
     Main function to generate a timestamped requirements file and convert it to Poetry dependencies.
     """
-    
-    current_time = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
-    fpath_requirements = f"{fpath}{current_time}_requirements.txt"
-    print("Current date and time:", current_time)
+    if not fpath.endswith('/'):
+        fpath+='/'
+    with open(os.path.join(fpath,'pyproject.toml'),'r') as f:
+        txt=f.read()
+    txt_list = re.split('\n',txt)
+
+    idx_start=[i for i,item in enumerate(txt_list) if item.startswith('[tool.poetry.dependencies]')][0]+2
+    idx_stop=[i for i,item in enumerate(txt_list) if item.startswith('[build-system]')][0]
+
+    # keep the head info
+    txt_list_head=txt_list[:idx_start]
+    txt_list_tail=txt_list[idx_stop:]
+
+    # keep the tail info
+    txt_list_head=[i+"\n" for i in txt_list_head]
+    txt_list_tail=[i+"\n" for i in txt_list_tail]
+
+    # filename
+    current_time = datetime.now().strftime("%Y-%m-%d")
+    fpath_requirements = f"{fpath}requirements_{current_time}.txt"
 
     os.makedirs(os.path.dirname(fpath_requirements), exist_ok=True)
 
+    # get the updated requirements.txt
     try:
         with open(fpath_requirements, 'w') as requirements_file:
             subprocess.run(['pip', 'freeze'], stdout=requirements_file, check=True)
         print(f"Requirements have been saved to {fpath_requirements}")
     except subprocess.CalledProcessError as e:
         print(f"An error occurred while creating the requirements file: {e}")
+        
+    # open the orgi toml file
+    try:
+        with open(fpath_requirements, 'r') as file:
+            txt = file.read()
+    except Exception as e:
+        print(f"An error occurred while reading the requirements file: {e}")
+
+    # convert to poetry
     txt_corr=[]
     for txt_ in re.split('\n', txt):
-        if len(txt_) <=40:
+        if len(txt_) <=40 and not txt_=='':
             txt_corr.append(txt_.replace("==",' = "^')+'"\n')
-    with open(fpath_requirements.replace('requirements.txt','for_poetry.txt'), 'w') as f:
-        f.writelines(txt_corr)
+
+    # add a newline
+    txt_corr.append('\n')
+
+    # merge 
+    txt_corr_=[*txt_list_head, *txt_corr, *txt_list_tail]
+
+    # rm .txt
+    # os.remove(fpath_requirements)
+
+    # fsave
+    with open(f"{fpath}pyproject_{current_time}.toml", 'w') as f:
+        f.writelines(txt_corr_)
 
 def main():
-    export_requirements(fpath="/Users/macjianfeng/Dropbox/github/python/py2ls/") 
+    export_requirements(fpath="/Users/macjianfeng/Dropbox/github/python/py2ls/")
 
 if __name__ == "__main__":
     main()

py2ls/ips.py

@@ -1022,15 +1022,30 @@ def fupdate(fpath, content=None):
     with open(fpath, 'w') as file:
         file.write(content)
         file.write(old_content)
+def fappend(fpath, content=None):
+    """
+    append new content at the end.
+    """
+    content = content or ""
+    if os.path.exists(fpath):
+        with open(fpath, 'r') as file:
+            old_content = file.read()
+    else:
+        old_content = '' 
+        
+    with open(fpath, 'w') as file:
+        file.write(old_content)
+        file.write(content)
         
 def fsave(
     fpath,
     content,
+    mode='w',
+    how ='overwrite',
     kind=None,
     font_name="Times",
     font_size=10,
     spacing=6,
-    mode='w',
     **kwargs,
 ):
     """
@@ -1046,9 +1061,14 @@ def fsave(
     Returns:
         None
     """
-    def save_content(fpath, content, mode=mode): 
-        with open(fpath, mode, encoding='utf-8') as file:
-            file.write(content)
+    def save_content(fpath, content, mode=mode, how='overwrite'): 
+        if 'wri' in how.lower():
+            with open(fpath, mode, encoding='utf-8') as file:
+                file.write(content)
+        elif 'upd' in how.lower():
+            fupdate(fpath, content=content)
+        elif 'app' in how.lower():
+            fappend(fpath, content=content)
 
 
     def save_docx(fpath, content, font_name, font_size, spacing):

py2ls/netfinder.py

@@ -1,6 +1,7 @@
 from bs4 import BeautifulSoup
 import requests
 from requests.utils import dict_from_cookiejar
+from requests.exceptions import ChunkedEncodingError, ConnectionError
 import os
 from urllib.parse import urlparse, urljoin
 import base64
@@ -150,7 +151,7 @@ def flatten_json(y):
 def get_proxy():
     list_ = []
     headers = {"User-Agent": user_agent()}
-    response = requests.get("https://free-proxy-list.net", headers=headers)
+    response = requests.get("https://free-proxy-list.net", headers=headers,timeout=30,stream=True)
     content = BeautifulSoup(response.content, "html.parser")
     info = extract_text_from_content(content, where="td", extend=0)[0].split()
     count, pair_proxy = 0, 2
@@ -200,18 +201,18 @@ def fetch_all(url, parser="lxml", driver='request', # request or selenium
         
         headers = {"User-Agent": user_agent()}
         if 'req' in driver.lower():
-            response = requests.get(url, headers=headers,proxies=proxies_glob)
+            response = requests.get(url, headers=headers,proxies=proxies_glob,timeout=30,stream=True)
 
             # If the response is a redirect, follow it
             while response.is_redirect:
                 logger.info(f"Redirecting to: {response.headers['Location']}")
-                response = requests.get(response.headers["Location"], headers=headers,proxies=proxies_glob)
+                response = requests.get(response.headers["Location"], headers=headers,proxies=proxies_glob,timeout=30,stream=True)
             # Check for a 403 error
             if response.status_code == 403:
                 logger.warning("403 Forbidden error. Retrying...")
                 # Retry the request after a short delay
                 sleep(random.uniform(1, 3))
-                response = requests.get(url, headers=headers,proxies=proxies_glob)
+                response = requests.get(url, headers=headers,proxies=proxies_glob,timeout=30,stream=True)
                 # Raise an error if retry also fails
                 response.raise_for_status()
 
@@ -471,7 +472,7 @@ def pdf_detector(url, contains = None, dir_save = None, booster = False):
             idx += 1
         print(f'{len(fnames)} files are downloaded:\n{fnames}\n to local: \n{dir_save}')
 
-def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=False, booster=False,verbose=True):
+def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=False, booster=False,verbose=True, timeout=30, n_try=3,timestamp=False):
     if verbose:
         print("usage: downloader(url, dir_save=None, kind=['.pdf','xls'], contains=None, booster=False)")
     def fname_corrector(fname, ext):
@@ -482,17 +483,21 @@ def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=F
         return fname
     def check_and_modify_filename(directory, filename):
         base, ext = os.path.splitext(filename)
-        counter = 2
+        counter = 1
         new_filename = filename
         while os.path.exists(os.path.join(directory, new_filename)):
-            new_filename = f"{base}_{counter}{ext}"
+            if counter<=9:
+                counter_='0'+str(counter)
+            else:
+                counter_=str(counter)
+            new_filename = f"{base}_{counter_}{ext}"
             counter += 1
         return new_filename
     if not isinstance(kind,list):
         kind=[kind]
     if isinstance(url, list):
         for url_ in url:
-            downloader(url_, dir_save=dir_save, kind=kind, contains=contains, booster=booster,verbose=verbose)
+            downloader(url_, dir_save=dir_save, kind=kind, contains=contains, booster=booster,verbose=verbose,timeout=timeout,n_try=n_try,timestamp=timestamp)
             # sleep(random.uniform(1, 3))
     for i,k in enumerate(kind):
         if not k.startswith('.'):
@@ -544,25 +549,45 @@ def downloader(url, dir_save=dir_save, kind=['.pdf'], contains=None, rm_folder=F
         fnames = [file_link.split("/")[-1] for file_link in file_links_all]
         for idx, file_link in enumerate(file_links_all):
             headers = {"User-Agent": user_agent()}
-            response = requests.get(file_link, headers=headers)
-            if response.status_code == 200:
-                ext = next((ftype for ftype in kind if ftype in file_link), None)
-                if ext:
-                    corrected_fname = fname_corrector(fnames[idx], ext)
-                    corrected_fname = check_and_modify_filename(dir_save, corrected_fname)
-                    with open(os.path.join(dir_save, corrected_fname), "wb") as file:
-                        file.write(response.content)
-                    if verbose:
-                        print(f"Done! {fnames[idx]}")
-                else:
-                    if verbose:
-                        print(f"Unknown file type for {file_link}")
-            else:
-                if verbose:
-                    print(f"Failed to download file: {response.status_code}")
+            itry = 0 # Retry logic with exception handling
+            while itry < n_try:
+                try:
+                    # streaming to handle large files and reduce memory usage.
+                    response = requests.get(file_link, headers=headers, timeout=timeout, stream=True)
+                    if response.status_code == 200:
+                        ext = next((ftype for ftype in kind if ftype in file_link), None)
+                        if ext:
+                            corrected_fname = fname_corrector(fnames[idx], ext)
+                            corrected_fname = check_and_modify_filename(dir_save, corrected_fname)
+                            if timestamp:
+                                corrected_fname=datetime.now().strftime("%y%m%d_%H%M%S_")+corrected_fname
+                            fpath_tmp = os.path.join(dir_save, corrected_fname)
+                            with open(fpath_tmp, "wb") as file:
+                                for chunk in response.iter_content(chunk_size=8192):
+                                    if chunk:  # Filter out keep-alive chunks
+                                        file.write(chunk)
+                            if verbose:
+                                print(f"Done! {fnames[idx]}")
+                        else:
+                            if verbose:
+                                print(f"Unknown file type for {file_link}")
+                        break  # Exit the retry loop if successful
+                    else:
+                        if verbose:
+                            print(f"Failed to download file: HTTP status code {response.status_code}")
+                except (ChunkedEncodingError, ConnectionError) as e:
+                    print(f"Attempt {itry+1} failed: {e}. Retrying in a few seconds...")
+                    # time.sleep(random.uniform(0, 2))  # Random sleep to mitigate server issues
+                    if os.path.exists(fpath_tmp):
+                                os.remove(fpath_tmp)
+                    itry += 1
+
+            if itry == n_try:
+                print(f"Failed to download {file_link} after {n_try} attempts.")
+
         print(f'\n{len(fnames)} files were downloaded:')
         if verbose:
-            pp(fnames)
+            pp(corrected_fname) if corrected_fname in locals() else pp(fnames)
             print(f"\n\nsaved @:\n{dir_save}")
         
 def find_img(url, driver='request',dir_save="images", rm_folder=False, verbose=True):

{py2ls-0.1.6.6.dist-info → py2ls-0.1.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.6.6
+Version: 0.1.6.8
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

{py2ls-0.1.6.6.dist-info → py2ls-0.1.6.8.dist-info}/RECORD

@@ -132,16 +132,16 @@ py2ls/data/docs_links.json,sha256=kXgbbWo0b8bfV4n6iuuUNLnZipIyLzokUO6Lzmf7nO4,10
 py2ls/data/lang_code_iso639.json,sha256=qZiU7H2RLJjDMXK22C-jhwzLJCI5vKmampjB1ys4ek4,2157
 py2ls/db2ls.py,sha256=MMfFX47aIPIyu7fU9aPvX9lbPRPYOpJ_VXwlnWk-8qo,13615
 py2ls/doc.py,sha256=xN3g1OWfoaGUhikbJ0NqbN5eKy1VZVvWwRlhHMgyVEc,4243
-py2ls/export_requirements.py,sha256=qsyYrT_fnBlaMuWRYr3EnDyAKT52yJJBlW0Em_rlvrU,1265
+py2ls/export_requirements.py,sha256=psZtSe-MOD9L_w3dVpA_VJEKfq3J914g3Y1OtRNAb4g,2324
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
-py2ls/ips.py,sha256=B88jol_EYZ4IERWAeabkHSwMkGytqyBheEbbpM_YDiQ,86146
-py2ls/netfinder.py,sha256=ZsLWGYMeRuGvxj2nqE0Z8ANoaVl18Necfw0HQfh2q7I,45548
+py2ls/ips.py,sha256=jyvui0X41-AyZfpcujfgBMt6NsQNfyh_SiQ5lQ0gk1Q,86744
+py2ls/netfinder.py,sha256=aOrgXp2rqpUDREZMlP_875SuAAcQXu3lhnRMk1cPG5M,47269
 py2ls/plot.py,sha256=8_33-1wpkGZrDUuvRBfTPUi_BRKdf1njoR725OLSLSY,48579
 py2ls/setuptools-70.1.0-py3-none-any.whl,sha256=2bi3cUVal8ip86s0SOvgspteEF8SKLukECi-EWmFomc,882588
 py2ls/sleep_events_detectors.py,sha256=36MCuRrpurn0Uvzpo3p3b3_JlVsRNHSWCXbJxCGM3mg,51546
 py2ls/stats.py,sha256=Wd9yCKQ_61QD29WMEgMuEcreFxF91NmlPW65iWT2B5w,39041
 py2ls/translator.py,sha256=6S7MmTZmjj8NljVmj0W5uEauu4ePxso3AMf2LvGVRQA,30516
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.1.6.6.dist-info/METADATA,sha256=L7E5_PaUNRXQzUiwKTqHFTNjDPSA7iUu_wCYOs6SHHE,20998
-py2ls-0.1.6.6.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-py2ls-0.1.6.6.dist-info/RECORD,,
+py2ls-0.1.6.8.dist-info/METADATA,sha256=yQ6DRgZ13Lc-Hw3FhI0dbn-nE0N3JwbC9X_8rQlxILQ,20998
+py2ls-0.1.6.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+py2ls-0.1.6.8.dist-info/RECORD,,

{py2ls-0.1.6.6.dist-info → py2ls-0.1.6.8.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any