py2ls 0.2.4.33__py3-none-any.whl → 0.2.4.35__py3-none-any.whl

py2ls/ips.py

@@ -26,6 +26,22 @@ import re
 import stat
 import platform
 
+# only for backup these scripts
+def backup(
+    src="/Users/macjianfeng/Dropbox/github/python/py2ls/.venv/lib/python3.12/site-packages/py2ls/",
+    tar="/Users/macjianfeng/Dropbox/github/python/py2ls/py2ls/",
+    kind="py",
+    overwrite=True,
+    reverse=False,
+    verbose=False
+):  
+    if reverse:
+        src, tar = tar,src
+        print(f"reversed")
+    f = listdir(src, kind, verbose=verbose)
+    [copy(i, tar, overwrite=overwrite, verbose=verbose) for i in f.path]
+    print(f"all files are copied from {os.path.basename(src)} to {tar}") if verbose else None
+
 def run_once_within(duration=60, reverse=False):  # default 60s
     import time
 
@@ -2683,13 +2699,41 @@ def fload(fpath, kind=None, **kwargs):
     def load_excel(fpath, **kwargs):
         engine = kwargs.get("engine", "openpyxl")
         verbose = kwargs.pop("verbose", False)
-        if run_once_within(reverse=True):
-            use_pd("read_excel", verbose=verbose)
-        df = pd.read_excel(fpath, engine=engine, **kwargs)
+        password=kwargs.pop("password",None)
+
+        if not password:
+            if run_once_within(reverse=True):
+                use_pd("read_excel", verbose=verbose)
+            df = pd.read_excel(fpath, engine=engine, **kwargs)
+            try:
+                meta = pd.ExcelFile(fpath)
+                print(f"n_sheet={len(meta.sheet_names)},\t'sheetname = 0 (default)':")
+                [print(f"{i}:\t{i_}") for i, i_ in enumerate(meta.sheet_names)]
+            except:
+                pass
+            return df
+        #* needs a password?
+        import msoffcrypto # pip install msoffcrypto-tool
+        from io import BytesIO
+
+        # Open the encrypted Excel file
+        with open(fpath, 'rb') as f:
+            try:
+                office_file = msoffcrypto.OfficeFile(f)
+                office_file.load_key(password=password)  # Provide the password
+                decrypted = BytesIO()
+                office_file.decrypt(decrypted)
+            except:
+                office_file = msoffcrypto.OfficeFile(f)
+                office_file.load_key(password=depass(password))  # Provide the password
+                decrypted = BytesIO()
+                office_file.decrypt(decrypted)
+        decrypted.seek(0)
+        df = pd.read_excel(decrypted, engine=engine, **kwargs)
         try:
-            meata = pd.ExcelFile(fpath)
-            print(f"n_sheet={len(meata.sheet_names)},\t'sheetname = 0 (default)':")
-            [print(f"{i}:\t{i_}") for i, i_ in enumerate(meata.sheet_names)]
+            meta = pd.ExcelFile(fpath)
+            print(f"n_sheet={len(meta.sheet_names)},\t'sheetname = 0 (default)':")
+            [print(f"{i}:\t{i_}") for i, i_ in enumerate(meta.sheet_names)]
         except:
             pass
         return df
@@ -3273,11 +3317,16 @@ def fsave(
         df = pd.DataFrame(data)
         df.to_csv(fpath, **kwargs_valid)
 
-    def save_xlsx(fpath, data, **kwargs):
+
+    def save_xlsx(fpath, data, password=None, **kwargs):
+        import msoffcrypto
+        from io import BytesIO
         verbose = kwargs.pop("verbose", False)
         sheet_name = kwargs.pop("sheet_name", "Sheet1")
+        
         if run_once_within(reverse=True):
             use_pd("to_excel", verbose=verbose)
+        
         if any(kwargs):
             format_excel(df=data, filename=fpath, **kwargs)
         else:
@@ -3302,16 +3351,30 @@ def fsave(
                 kwargs.pop(key, None)
 
             df = pd.DataFrame(data)
-            # Check if the file exists, then append the sheet, otherwise create a new file
-            try:
-                # Use ExcelWriter with append mode if the file exists
-                with pd.ExcelWriter(
-                    fpath, engine="openpyxl", mode="a", if_sheet_exists="new"
-                ) as writer:
-                    df.to_excel(writer, sheet_name=sheet_name, index=False, **kwargs)
-            except FileNotFoundError:
-                # If file doesn't exist, create a new one
-                df.to_excel(fpath, sheet_name=sheet_name, index=False, **kwargs)
+            
+            # Write to Excel without password first
+            temp_file = BytesIO()
+            df.to_excel(temp_file, sheet_name=sheet_name, index=False, engine="xlsxwriter", **kwargs)
+
+            # If a password is provided, encrypt the file
+            if password:
+                temp_file.seek(0)
+                office_file = msoffcrypto.OfficeFile(temp_file)
+                office_file.load_key(password=password)  # Provide the password
+
+                # Encrypt and save the file
+                with open(fpath, 'wb') as encrypted_file:
+                    office_file.encrypt(encrypted_file)
+            else:
+                # Save the file without encryption if no password is provided
+                try:
+                    # Use ExcelWriter with append mode if the file exists
+                    with pd.ExcelWriter(fpath, engine="openpyxl", mode="a", if_sheet_exists="new") as writer:
+                        df.to_excel(writer, sheet_name=sheet_name, index=False, **kwargs)
+                except FileNotFoundError:
+                    # If file doesn't exist, create a new one
+                    df.to_excel(fpath, sheet_name=sheet_name, index=False, **kwargs)
+
 
     def save_ipynb(fpath, data, **kwargs):
         # Split the content by code fences to distinguish between code and markdown
@@ -4400,13 +4463,13 @@ def func_list(lib_name, opt="call"):
     return list_func(lib_name, opt=opt)
 
 
-def copy(src, dst, overwrite=False):
+def copy(src, dst, overwrite=False, verbose=True):
     """Copy a file from src to dst."""
     try:
         dir_par_dst = os.path.dirname(dst)
         if not os.path.isdir(dir_par_dst):
             mkdir(dir_par_dst)
-        print(dir_par_dst)
+        print(dir_par_dst) if verbose else None
         src = Path(src)
         dst = Path(dst)
         if not src.is_dir():
@@ -4421,7 +4484,7 @@ def copy(src, dst, overwrite=False):
                         f"{dst.stem}_{datetime.now().strftime('_%H%M%S')}{dst.suffix}"
                     )
             shutil.copy(src, dst)
-            print(f"\n Done! copy to {dst}\n")
+            print(f"\n Done! copy to {dst}\n") if verbose else None
         else:
             dst = dst / src.name
             if dst.exists():
@@ -4432,7 +4495,7 @@ def copy(src, dst, overwrite=False):
                         f"{dst.stem}_{datetime.now().strftime('%H%M%S')}"
                     )
             shutil.copytree(src, dst)
-            print(f"\n Done! copy to {dst}\n")
+            print(f"\n Done! copy to {dst}\n") if verbose else None
 
     except Exception as e:
         logging.error(f"Failed {e}")
@@ -4442,7 +4505,7 @@ def cut(src, dst, overwrite=False):
     return move(src=src, dst=dst, overwrite=overwrite)
 
 
-def move(src, dst, overwrite=False):
+def move(src, dst, overwrite=False, verbose=True):
     try:
         dir_par_dst = os.path.dirname(dst)
         if not os.path.isdir(dir_par_dst):
@@ -4460,7 +4523,7 @@ def move(src, dst, overwrite=False):
                     f"{dst.stem}_{datetime.now().strftime('_%H%M%S')}{dst.suffix}"
                 )
         shutil.move(src, dst)
-        print(f"\n Done! moved to {dst}\n")
+        print(f"\n Done! moved to {dst}\n") if verbose else None
     except Exception as e:
         logging.error(f"Failed to move file from {src} to {dst}: {e}")
 
@@ -4719,6 +4782,19 @@ def figsave(*args, dpi=300):
         plt.savefig(fname, format="emf", dpi=dpi, bbox_inches="tight", pad_inches=0)
     elif ftype.lower() == "fig":
         plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi, pad_inches=0)
+
+    elif ftype.lower() == "ico":
+        # Ensure the image is in a format that can be saved as an icon (e.g., 32x32, 64x64, etc.)
+        if img is None:  # If no image is provided, use the matplotlib figure
+            img = plt.figure()
+            img.savefig(fname, dpi=dpi, format="png", bbox_inches="tight")
+            img = Image.open(fname)  # Load the saved figure image
+
+        # Resize the image to typical icon sizes and save it as .ico
+        icon_sizes = [(32, 32), (64, 64), (128, 128), (256, 256)]
+        img = img.convert("RGBA")  # Ensure it has an alpha channel for transparency
+        img.save(fname, format="ICO", sizes=icon_sizes)
+        print(f"Icon saved @: {fname} with sizes: {icon_sizes}")
     print(f"\nSaved @: dpi={dpi}\n{fname}")
 
 
@@ -5497,7 +5573,6 @@ def detect_angle(image, by="median", template=None):
         print(f"Unknown method {by}: supported methods: {methods}")
         return 0
 
-
 def imgsets(img, 
             auto:bool=True,
             size=None,
@@ -5506,7 +5581,9 @@ def imgsets(img,
             show_axis:bool=False,
             plot_:bool=True,
             verbose:bool=False,
-            **kwargs):
+            model:str="isnet-general-use",
+            **kwargs,
+            ):
     """
     Apply various enhancements and filters to an image using PIL's ImageEnhance and ImageFilter modules.
 
@@ -5562,6 +5639,26 @@ def imgsets(img,
         "BOX_BLUR",
         "MEDIAN_FILTER",
     ]
+    # *Rembg is a tool to remove images background.
+    # https://github.com/danielgatis/rembg
+    rem_models = {
+        "u2net": "general use cases.",
+        "u2netp": "A lightweight version of u2net model.",
+        "u2net_human_seg": "human segmentation.",
+        "u2net_cloth_seg": "Cloths Parsing from human portrait. Here clothes are parsed into 3 category: Upper body, Lower body and Full body.",
+        "silueta": "Same as u2net but the size is reduced to 43Mb.",
+        "isnet-general-use": "A new pre-trained model for general use cases.",
+        "isnet-anime": "A high-accuracy segmentation for anime character.",
+        "sam": "any use cases.",
+        "birefnet-general": "general use cases.",
+        "birefnet-general-lite": "A light pre-trained model for general use cases.",
+        "birefnet-portrait": "human portraits.",
+        "birefnet-dis": "dichotomous image segmentation (DIS).",
+        "birefnet-hrsod": "high-resolution salient object detection (HRSOD).",
+        "birefnet-cod": "concealed object detection (COD).",
+        "birefnet-massive": "A pre-trained model with massive dataset.",
+    }
+    models_support_rem=list(rem_models.keys())
     str_usage="""
     imgsets(dir_img, auto=1, color=1.5, plot_=0)
     imgsets(dir_img, color=2)
@@ -5569,6 +5666,7 @@ def imgsets(img,
     imgsets(dir_img, contrast=0, color=1.2, plot_=0)
     imgsets(get_clip(), flip="tb")# flip top and bottom
     imgsets(get_clip(), contrast=1, rm=[100, 5, 2]) #'foreground_threshold', 'background_threshold' and 'erode_structure_size'
+    imgsets(dir_img, rm="birefnet-portrait") # with using custom model
     """
     if run_once_within():
         print(str_usage)
@@ -5577,28 +5675,7 @@ def imgsets(img,
         # adjust gama value
         inv_gamma = 1.0 / gamma
         lut = [int((i / float(v_max)) ** inv_gamma * int(v_max)) for i in range(int(v_max))]
-        return lut #image.point(lut)
-
-    def confirm_rembg_models(model_name):
-        models_support = [
-            "u2net",
-            "u2netp",
-            "u2net_human_seg",
-            "u2net_cloth_seg",
-            "silueta",
-            "isnet-general-use",
-            "isnet-anime",
-            "sam",
-        ]
-        if model_name in models_support:
-            print(f"model_name: {model_name}")
-            return model_name
-        else:
-            print(
-                f"{model_name} cannot be found, check the name:{models_support}, default('isnet-general-use') has been used"
-            )
-            return "isnet-general-use"
-
+        return lut #image.point(lut) 
     def auto_enhance(img):
         """
         Automatically enhances the image based on its characteristics, including brightness,
@@ -5701,7 +5778,7 @@ def imgsets(img,
         kwargs = {**auto_enhance(img_update), **kwargs}
     params=["sharp","color","contrast","bright","crop","rotate",'size',"resize",
             "thumbnail","cover","contain","filter","fit","pad",
-            "rem","rm","back","bg_color","cut",'gamma','flip']
+            "rem","rm","back","bg_color","cut","gamma","flip","booster"]
     for k, value in kwargs.items():
         k = strcmp(k, params)[0] # correct the param name
         if "shar" in k.lower():
@@ -5722,7 +5799,7 @@ def imgsets(img,
                     img_update = ImageOps.autocontrast(img_update)
                     print("autocontrasted")
                 except Exception as e:
-                    print(f"Failed 'autocontrasted':{e}")
+                    print(f"Failed 'auto-contrasted':{e}")
         elif "bri" in k.lower():
             enhancer = ImageEnhance.Brightness(img_update)
             img_update = enhancer.enhance(value)
@@ -5765,9 +5842,14 @@ def imgsets(img,
             img_update = ImageOps.pad(img_update, size=value)
         elif "rem" in k.lower() or "rm" in k.lower() or "back" in k.lower():
             from rembg import remove, new_session
-
+            if verbose:
+                preview(rem_models)
+            
+            print(f"supported modles: {models_support_rem}")
+            model=strcmp(model, models_support_rem)[0]
+            session = new_session(model)
             if isinstance(value, bool):
-                session = new_session("isnet-general-use")
+                print(f"using model:{model}")
                 img_update = remove(img_update, session=session)
             elif value and isinstance(value, (int, float, list)):
                 if verbose:
@@ -5779,14 +5861,14 @@ def imgsets(img,
                     img_update = remove(
                         img_update,
                         alpha_matting=True,
-                        alpha_matting_background_threshold=value,
+                        alpha_matting_background_threshold=value, session=session
                     )
                 elif 2 <= len(value) < 3:
                     img_update = remove(
                         img_update,
                         alpha_matting=True,
                         alpha_matting_background_threshold=value[0],
-                        alpha_matting_foreground_threshold=value[1],
+                        alpha_matting_foreground_threshold=value[1], session=session
                     )
                 elif 3 <= len(value) < 4:
                     img_update = remove(
@@ -5794,17 +5876,16 @@ def imgsets(img,
                         alpha_matting=True,
                         alpha_matting_background_threshold=value[0],
                         alpha_matting_foreground_threshold=value[1],
-                        alpha_matting_erode_size=value[2],
+                        alpha_matting_erode_size=value[2], session=session
                     )
             elif isinstance(value, tuple):  # replace the background color
                 if len(value) == 3:
                     value += (255,)
-                img_update = remove(img_update, bgcolor=value)
+                img_update = remove(img_update, bgcolor=value, session=session)
             elif isinstance(value, str):
-                if confirm_rembg_models(value):
-                    img_update = remove(img_update, session=new_session(value))
-                else:
-                    img_update = remove(img_update)
+                # use custom model
+                print(f"using model:{strcmp(value, models_support_rem)[0]}")
+                img_update = remove(img_update, session=new_session(strcmp(value, models_support_rem)[0]))
         elif "bg" in k.lower() and "color" in k.lower():
             from rembg import remove
 
@@ -5814,7 +5895,33 @@ def imgsets(img,
                 if len(value) == 3:
                     value += (255,)
                 img_update = remove(img_update, bgcolor=value)
-        
+        elif 'boost' in k.lower():
+            import torch
+            from realesrgan import RealESRGANer
+            if verbose:
+                print("Applying Real-ESRGAN for image reconstruction...")
+            if isinstance(value, bool):
+                scale=4
+            elif isinstance(value, (float, int)):
+                scale=value
+            else:
+                scale=4
+
+            # try:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            dir_curr_script = os.path.dirname(os.path.abspath(__file__))
+            model_path = dir_curr_script + "/data/RealESRGAN_x4plus.pth"
+            model_RealESRGAN = RealESRGANer(device=device, 
+                                            scale=scale, 
+                                            model_path=model_path,
+                                            model="RealESRGAN_x4plus"
+                                            )
+            # https://github.com/xinntao/Real-ESRGAN?tab=readme-ov-file#python-script
+
+            img_update = model_RealESRGAN.enhance(np.array(img_update))[0] 
+            # except Exception as e:
+            #     print(f"Failed to apply Real-ESRGAN: {e}")
+
         # elif "ga" in k.lower() and "m" in k.lower():
         #     img_update = gamma_correction(img_update, gamma=value)
     # Display the image if requested
@@ -10715,4 +10822,227 @@ def mouse(
     #         "Image not found. Ensure the image is visible and parameters are correct."
     #     )
     # except Exception as e:
-    #     print(f"An error occurred: {e}")
+    #     print(f"An error occurred: {e}")
+
+
+
+def py2installer(
+    script_path: str=None,
+    flatform:str="mingw64",
+    output_dir: str = "dist",
+    icon_path: str = None,
+    extra_data: list = None,
+    hidden_imports: list = None,
+    plugins:list=None,
+    use_nuitka: bool = True,
+    onefile: bool = True,
+    console: bool = True,
+    clean_build: bool = False,
+    additional_args: list = None,
+    verbose: bool = True,
+    use_docker: bool = False,
+    docker_image: str = "python:3.12-slim",
+):
+    """
+    to package Python scripts into standalone application.
+
+    script_path (str): Path to the Python script to package.
+    output_dir (str): Directory where the executable will be stored.
+    icon_path (str): Path to the .ico file for the executable icon.
+    extra_data (list): List of additional data files or directories in "source:dest" format.
+    hidden_imports (list): List of hidden imports to include.
+    plugins (list): List of plugins imports to include.e.g., 'tk-inter'
+    use_nuitka (bool): Whether to use Nuitka instead of PyInstaller.
+    onefile (bool): If True, produces a single executable file.
+    console (bool): If False, hides the console window (GUI mode).
+    clean_build (bool): If True, cleans previous build and dist directories.
+    additional_args (list): Additional arguments for PyInstaller/Nuitka.
+    verbose (bool): If True, provides detailed logs.
+    use_docker (bool): If True, uses Docker to package the script.
+    docker_image (str): Docker image to use for packaging.
+
+    """
+
+    import os
+    import sys
+    import shutil
+    import subprocess
+    import sys
+    import glob
+    from pathlib import Path
+    if run_once_within():
+        usage_str="""
+            # build locally 
+            py2installer(
+                script_path="update_tab.py",
+                output_dir="dist",
+                icon_path="icon4app.ico",
+                extra_data=["dat/*.xlsx:dat"],
+                hidden_imports=["msoffcrypto", "tkinter", "pandas", "numpy"],
+                onefile=True,
+                console=False,
+                clean_build=True,
+                verbose=True,
+            )
+            # build via docker
+            py2installer(
+                "my_script.py",
+                output_dir="dist",
+                onefile=True,
+                clean_build=True,
+                use_docker=True,
+                docker_image="python:3.12-slim"
+            )
+        """
+        print(usage_str)
+        if verbose:
+            return
+        else:
+            pass
+    # Check if the script path exists
+    script_path = Path(script_path)
+    if not script_path.exists():
+        raise FileNotFoundError(f"Script '{script_path}' not found.")
+
+     # Clean build and dist directories if requested
+    if clean_build:
+        for folder in ["build", "dist"]:
+            folder_path = Path(folder)
+            if folder_path.exists():
+                shutil.rmtree(folder_path, ignore_errors=True)
+        # Recreate the folders
+        for folder in ["build", "dist"]:
+            folder_path = Path(folder)
+            folder_path.mkdir(parents=True, exist_ok=True)
+                
+
+    if use_docker:
+        # Ensure Docker is installed
+        try:
+            subprocess.run(["docker", "--version"], check=True, capture_output=True, text=True)
+        except FileNotFoundError:
+            raise EnvironmentError("Docker is not installed or not in the PATH.")
+
+        # Prepare Docker volume mappings
+        script_dir = script_path.parent.resolve()
+        dist_path = Path(output_dir).resolve()
+        volumes = [
+            f"{script_dir}:/app:rw",
+            f"{dist_path}:/output:rw",
+        ]
+        docker_cmd = [
+            "docker", "run", "--rm",
+            "-v", volumes[0],
+            "-v", volumes[1],
+            docker_image,
+            "bash", "-c",
+        ]
+
+        # Build the packaging command inside the container
+        cmd = ["nuitka"] if use_nuitka else ["pyinstaller"]
+        if onefile:
+            cmd.append("--onefile")
+        if not console:
+            cmd.append("--windowed")
+        cmd.extend(["--distpath", "/output"])
+        if icon_path:
+            cmd.extend(["--icon", f"/app/{Path(icon_path).name}"])
+        if extra_data:
+            for data in extra_data:
+                cmd.extend(["--add-data", f"/app/{data}"])
+        if hidden_imports:
+            for hidden in hidden_imports:
+                cmd.extend(["--hidden-import", hidden])
+        if additional_args:
+            cmd.extend(additional_args)
+        cmd.append(f"/app/{script_path.name}")
+
+        # Full command to execute inside the container
+        docker_cmd.append(" ".join(cmd))
+
+        if verbose:
+            print(f"Running Docker command: {' '.join(docker_cmd)}")
+
+        # Run Docker command
+        try:
+            subprocess.run(
+                docker_cmd,
+                capture_output=not verbose,
+                text=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Error during Docker packaging:\n{e.stderr}", file=sys.stderr)
+            raise
+    else:
+        # Handle local packaging (native build)
+        cmd = ["nuitka"]
+        cmd.append("--standalone")  # Make sure to use --standalone for independent environments
+        if 'min' in flatform.lower():
+            cmd.append("--mingw64")
+        if onefile:
+            cmd.append("--onefile")
+        if not console:
+            # cmd.append("--windows-disable-console")  # Disabled based on your request
+            pass
+
+        cmd.extend([f"--output-dir={output_dir}"])  # Correct the space issue here
+        if icon_path:
+            icon_path = Path(icon_path)
+            if not icon_path.exists():
+                raise FileNotFoundError(f"Icon file '{icon_path}' not found.")
+            cmd.extend([f"--windows-icon-from-ico={icon_path}"])
+
+        if extra_data:
+            for data in extra_data:
+                if "*" in data:
+                    matches = glob.glob(data.split(":")[0])
+                    for match in matches:
+                        dest = data.split(":")[1]
+                        cmd.extend(
+                            [
+                                "--include-data-file=" if use_nuitka else "--add-data",
+                                f"{match}:{dest}",
+                            ]
+                        )
+                else:
+                    cmd.extend(
+                        ["--include-data-file=" if use_nuitka else "--add-data", data]
+                    )
+
+        if hidden_imports:
+            cmd.extend([f"--nofollow-import-to={','.join(hidden_imports)}"])
+
+        if plugins:
+            for plugin in plugins:
+                cmd.extend([f"--plugin-enable={plugin}"])
+
+        if additional_args:
+            cmd.extend(additional_args)
+
+        # Add the script path (final positional argument)
+        cmd.append(str(script_path))
+
+        # Ensure Windows shell compatibility
+        shell_flag = sys.platform.startswith("win")
+
+        # Run the command
+        if verbose:
+            print(f"Running command: {' '.join(cmd)}")
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=not verbose,
+                text=True,
+                shell=shell_flag,
+                check=True,
+            )
+            if verbose:
+                print(result.stdout)
+        except subprocess.CalledProcessError as e:
+            print(f"Error during packaging:\n{e.stderr}", file=sys.stderr)
+            print(" ".join(cmd))
+            raise
+
+    print("\nPackaging complete. Check the output directory for the executable.")
+ 

py2ls/netfinder.py

@@ -333,32 +333,6 @@ def parse_cookies(cookies_str):
 
     return cookies_dict
 
-class FetchSpider(scrapy.Spider):
-    name = "fetch_spider"
-    
-    def __init__(self, url, parser="html.parser", cookies=None, headers=None, *args, **kwargs):
-        super(FetchSpider, self).__init__(*args, **kwargs)
-        self.start_urls = [url]
-        self.cookies = cookies
-        self.headers = headers
-        self.parser = parser
-    
-    def start_requests(self):
-        for url in self.start_urls:
-            yield scrapy.Request(
-                url,
-                cookies=self.cookies,
-                headers=self.headers,
-                callback=self.parse
-            )
-    
-    def parse(self, response):
-        # Use the desired parser (default: html.parser)
-        from bs4 import BeautifulSoup
-        soup = BeautifulSoup(response.text, self.parser)
-        yield {"content": soup}
-
-
 def fetch_scrapy(
     url,
     parser="html.parser",
@@ -367,7 +341,7 @@ def fetch_scrapy(
     settings=None,
 ):
     """
-    Fetches content using Scrapy.
+    Fetches content using Scrapy with proper reactor handling.
 
     Args:
         url (str): The URL to scrape.
@@ -380,9 +354,10 @@ def fetch_scrapy(
         dict: Parsed content as a dictionary.
     """
     from scrapy.utils.project import get_project_settings
-    from scrapy.crawler import CrawlerProcess
+    from scrapy.crawler import CrawlerRunner
     from scrapy.signalmanager import dispatcher
     from scrapy import signals
+    from twisted.internet import reactor, defer
     import scrapy
 
     # Container for scraped content
@@ -403,19 +378,30 @@ def fetch_scrapy(
         }
     )
 
-    # Initialize and configure Scrapy process
-    process = CrawlerProcess(settings=process_settings)
+    # Connect item scraped signal
     dispatcher.connect(handle_item, signal=signals.item_scraped)
 
-    # Start the Scrapy crawl
-    process.crawl(
-        FetchSpider,
-        url=url,
-        parser=parser,
-        cookies=cookies,
-        headers=headers,
-    )
-    process.start()  # Blocks until all crawls are finished
+    # Asynchronous Twisted function
+    @defer.inlineCallbacks
+    def crawl():
+        runner = CrawlerRunner(settings=process_settings)
+        yield runner.crawl(
+            FetchSpider,
+            url=url,
+            parser=parser,
+            cookies=cookies,
+            headers=headers,
+        )
+        reactor.stop()
+
+    # Start the reactor if not already running
+    if not reactor.running:
+        crawl()
+        reactor.run()  # Blocks until the crawl finishes
+    else:
+        # Run the crawl if the reactor is already running
+        d = crawl()
+        d.addBoth(lambda _: reactor.stop())
 
     # Return the first scraped content or None if empty
     return content[0] if content else None
@@ -654,7 +640,11 @@ def fetch_all(
                 "COOKIES_ENABLED": True if cookies else False,
                 "LOG_LEVEL": "WARNING",  # Reduce log verbosity
             }
-            content=fetch_scrapy(url, parser=parser, cookies=cookies, headers=headers, settings=settings)
+            content=fetch_scrapy(url, 
+                                 parser=parser, 
+                                 cookies=cookies, 
+                                 headers=headers, 
+                                 settings=settings)
             return parser, content 
 
     except requests.RequestException as e:

py2ls/ocr.py

@@ -9,8 +9,6 @@ from py2ls.ips import (
     isa
 )
 import logging
-#logging.getLogger("ppocr").setLevel(logging.ERROR)
-logging.getLogger("ppocr").setLevel(logging.WARNING)
 
 """
     Optical Character Recognition (OCR)
@@ -563,8 +561,8 @@ def get_text(
     font=cv2.FONT_HERSHEY_SIMPLEX,# draw_box
     fontsize=8,# draw_box
     figsize=[10,10],
-    box_color = (0, 255, 0),  # draw_box
-    fontcolor = (0, 0, 0),# draw_box
+    box_color = (0, 255, 0), # draw_box
+    fontcolor = (116,173,233), # draw_box
     bg_color=(133, 203, 245, 100),# draw_box
     usage=False,
     **kwargs,
@@ -618,7 +616,6 @@ def get_text(
         image = cv2.imread(image)
     elif isa(image,'image'):
         cvt_cmp=False
-        print(1)
         image = np.array(image)
     else:
         raise ValueError(f"not support image with {type(image)} type")
@@ -704,6 +701,8 @@ def get_text(
                 return detections
     elif "pad" in model.lower():
         from paddleocr import PaddleOCR
+        logging.getLogger("ppocr").setLevel(logging.ERROR)
+        
         lang=strcmp(lang, ['ch','en','french','german','korean','japan'])[0]
         ocr = PaddleOCR(
             use_angle_cls=True,

{py2ls-0.2.4.33.dist-info → py2ls-0.2.4.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.2.4.33
+Version: 0.2.4.35
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Provides-Extra: extr
+Provides-Extra: full
 Requires-Dist: CacheControl (>=0.13.1)
 Requires-Dist: Cython (>=3.0.10)
 Requires-Dist: Deprecated (>=1.2.14)
@@ -85,7 +85,7 @@ Requires-Dist: h2 (>=3.2.0)
 Requires-Dist: h5py (>=3.11.0)
 Requires-Dist: hpack (>=3.0.0)
 Requires-Dist: hstspreload (>=2024.7.1)
-Requires-Dist: httpcore (>=0.9.1)
+Requires-Dist: httpcore (>=0.9.0)
 Requires-Dist: httpx (>=0.13.3)
 Requires-Dist: humanfriendly (>=10.0)
 Requires-Dist: hyperframe (>=5.2.0)
@@ -123,6 +123,7 @@ Requires-Dist: mne (>=1.7.1)
 Requires-Dist: more-itertools (>=10.3.0)
 Requires-Dist: mpmath (>=1.3.0)
 Requires-Dist: msgpack (>=1.0.8)
+Requires-Dist: msoffcrypto-tool (>=5.4.2)
 Requires-Dist: mtscomp (>=1.0.2)
 Requires-Dist: nbclient (>=0.10.0)
 Requires-Dist: nbconvert (>=7.16.4)
@@ -131,6 +132,7 @@ Requires-Dist: neo (>=0.13.1)
 Requires-Dist: nest-asyncio (>=1.6.0)
 Requires-Dist: networkx (>=3.3)
 Requires-Dist: nltk (>=3.8.1)
+Requires-Dist: nuitka (>=2.5.9)
 Requires-Dist: numba (>=0.59.1)
 Requires-Dist: numcodecs (>=0.13.0)
 Requires-Dist: numerizer (>=0.2.3)
@@ -139,6 +141,7 @@ Requires-Dist: onnxruntime (>=1.18.1)
 Requires-Dist: opencv-contrib-python (>=4.10.0.84)
 Requires-Dist: opencv-python (>=4.10.0.84)
 Requires-Dist: opencv-python-headless (>=4.10.0.84)
+Requires-Dist: openpyxl (>=3.1.5)
 Requires-Dist: outcome (>=1.3.0.post0)
 Requires-Dist: packaging (>=24.1)
 Requires-Dist: pandas (>=2.2.2)
@@ -199,6 +202,7 @@ Requires-Dist: rpds-py (>=0.18.1)
 Requires-Dist: scikit-image (>=0.23.2)
 Requires-Dist: scikit-learn (>=1.5.1)
 Requires-Dist: scipy (>=1.14.0)
+Requires-Dist: scrapy (>=2.12.0)
 Requires-Dist: seaborn (>=0.13.2)
 Requires-Dist: selenium (>=4.23.1)
 Requires-Dist: setuptools (>=70.3.0)

{py2ls-0.2.4.33.dist-info → py2ls-0.2.4.35.dist-info}/RECORD

@@ -182,6 +182,7 @@ py2ls/chat.py,sha256=Yr22GoIvoWhpV3m4fdwV_I0Mn77La346_ymSinR-ORA,3793
 py2ls/corr.py,sha256=RbOaJIPLCHJtUm5SFi_4dCJ7VFUPWR0PErfK3K26ad4,18243
 py2ls/correlators.py,sha256=RbOaJIPLCHJtUm5SFi_4dCJ7VFUPWR0PErfK3K26ad4,18243
 py2ls/data/.DS_Store,sha256=0vLN27VSbtJbHNG5DG-oGqs9h_ubWh3xHz-baHnyfck,6148
+py2ls/data/RealESRGAN_x4plus.pth,sha256=T6DTiQX3WsButJp5UbQmZwAhvjAYJl_RkdISXfnWgvE,67040989
 py2ls/data/db2ls_sql_chtsht.json,sha256=ls9d7Sm8TLeujanWHfHlWhU85Qz1KnAizO_9X3wUH7E,6933
 py2ls/data/docs_links.json,sha256=kXgbbWo0b8bfV4n6iuuUNLnZipIyLzokUO6Lzmf7nO4,101829
 py2ls/data/email/email_html_template.html,sha256=UIg3aixWfdNsvVx-j2dX1M5N3G-6DgrnV1Ya1cLjiUQ,2809
@@ -242,18 +243,18 @@ py2ls/export_requirements.py,sha256=x2WgUF0jYKz9GfA1MVKN-MdsM-oQ8yUeC6Ua8oCymio,
 py2ls/fetch_update.py,sha256=9LXj661GpCEFII2wx_99aINYctDiHni6DOruDs_fdt8,4752
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
 py2ls/ich2ls.py,sha256=3E9R8oVpyYZXH5PiIQgT3CN5NxLe4Dwtm2LwaeacE6I,21381
-py2ls/ips.py,sha256=Fap4Iqt3QCq-td6CSz2dPBwI65TLfTRApy2QYFO7TIY,403036
+py2ls/ips.py,sha256=2tFfYrkRXdR-jwWYK1goE2zhSfG_iHhkDc73gLje6hI,416145
 py2ls/ml2ls.py,sha256=I-JFPdikgEtfQjhv5gBz-QSeorpTJI_Pda_JwkTioBY,209732
 py2ls/mol.py,sha256=AZnHzarIk_MjueKdChqn1V6e4tUle3X1NnHSFA6n3Nw,10645
-py2ls/netfinder.py,sha256=6XZWxFCo5PNOVKdr5qGL_250AoKLfz6CuVmhGkDwkFM,69266
+py2ls/netfinder.py,sha256=OhqD3S9PuwweL2013D-q4GNP1WvJjuYfZzq5BZgGddE,68980
 py2ls/nl2ls.py,sha256=UEIdok-OamFZFIvvz_PdZenu085zteMdaJd9mLu3F-s,11485
-py2ls/ocr.py,sha256=xNNQd_cHjb4HLSZvRQPHJGPm29de6urZhoOddBKnV8M,33989
+py2ls/ocr.py,sha256=WDFvx1oVxXjlyyFs2a6pizdu4-jEL5pTFLP960-HbyM,33939
 py2ls/plot.py,sha256=7C1x6KX0Fvmbll4IStIzlNjxLnrRBNSPaLJRgGjF3Ok,239172
 py2ls/setuptools-70.1.0-py3-none-any.whl,sha256=2bi3cUVal8ip86s0SOvgspteEF8SKLukECi-EWmFomc,882588
 py2ls/sleep_events_detectors.py,sha256=bQA3HJqv5qnYKJJEIhCyhlDtkXQfIzqksnD0YRXso68,52145
 py2ls/stats.py,sha256=qBn2rJmNa_QLLUqjwYqXUlGzqmW94sgA1bxJU2FC3r0,39175
 py2ls/translator.py,sha256=77Tp_GjmiiwFbEIJD_q3VYpQ43XL9ZeJo6Mhl44mvh8,34284
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.2.4.33.dist-info/METADATA,sha256=YHx2Yww0F2wy0fPxBKC-pAkpLBq9db2D-McYtW7r8FI,20332
-py2ls-0.2.4.33.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-py2ls-0.2.4.33.dist-info/RECORD,,
+py2ls-0.2.4.35.dist-info/METADATA,sha256=j9QC5tXPpOp4VTvG6onwy3BaoPw_ROq0KGPGw2bo8Gk,20473
+py2ls-0.2.4.35.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+py2ls-0.2.4.35.dist-info/RECORD,,