py2ls 0.2.4.34__tar.gz → 0.2.4.36__tar.gz

{py2ls-0.2.4.34 → py2ls-0.2.4.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.2.4.34
+Version: 0.2.4.36
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Provides-Extra: extr
+Provides-Extra: full
 Requires-Dist: CacheControl (>=0.13.1)
 Requires-Dist: Cython (>=3.0.10)
 Requires-Dist: Deprecated (>=1.2.14)
@@ -25,11 +25,6 @@ Requires-Dist: MarkupSafe (>=2.1.5)
 Requires-Dist: PyMatting (>=1.1.12)
 Requires-Dist: PyOpenGL (>=3.1.6)
 Requires-Dist: PyPDF2 (>=3.0.1)
-Requires-Dist: PyQt5 (>=5.15.0)
-Requires-Dist: PyQt5-Qt5 (>=5.15.0)
-Requires-Dist: PyQt5_sip (>=12.15.0)
-Requires-Dist: PyQtWebEngine (>=5.15.0)
-Requires-Dist: PyQtWebEngine-Qt5 (>=5.15.0)
 Requires-Dist: PySocks (>=1.7.1)
 Requires-Dist: PyYAML (>=6.0.1)
 Requires-Dist: Pygments (>=2.18.0)
@@ -123,6 +118,7 @@ Requires-Dist: mne (>=1.7.1)
 Requires-Dist: more-itertools (>=10.3.0)
 Requires-Dist: mpmath (>=1.3.0)
 Requires-Dist: msgpack (>=1.0.8)
+Requires-Dist: msoffcrypto-tool (>=5.4.2)
 Requires-Dist: mtscomp (>=1.0.2)
 Requires-Dist: nbclient (>=0.10.0)
 Requires-Dist: nbconvert (>=7.16.4)
@@ -131,6 +127,7 @@ Requires-Dist: neo (>=0.13.1)
 Requires-Dist: nest-asyncio (>=1.6.0)
 Requires-Dist: networkx (>=3.3)
 Requires-Dist: nltk (>=3.8.1)
+Requires-Dist: nuitka (>=2.5.9)
 Requires-Dist: numba (>=0.59.1)
 Requires-Dist: numcodecs (>=0.13.0)
 Requires-Dist: numerizer (>=0.2.3)
@@ -139,6 +136,7 @@ Requires-Dist: onnxruntime (>=1.18.1)
 Requires-Dist: opencv-contrib-python (>=4.10.0.84)
 Requires-Dist: opencv-python (>=4.10.0.84)
 Requires-Dist: opencv-python-headless (>=4.10.0.84)
+Requires-Dist: openpyxl (>=3.1.5)
 Requires-Dist: outcome (>=1.3.0.post0)
 Requires-Dist: packaging (>=24.1)
 Requires-Dist: pandas (>=2.2.2)
@@ -199,6 +197,7 @@ Requires-Dist: rpds-py (>=0.18.1)
 Requires-Dist: scikit-image (>=0.23.2)
 Requires-Dist: scikit-learn (>=1.5.1)
 Requires-Dist: scipy (>=1.14.0)
+Requires-Dist: scrapy (>=2.12.0)
 Requires-Dist: seaborn (>=0.13.2)
 Requires-Dist: selenium (>=4.23.1)
 Requires-Dist: setuptools (>=70.3.0)

{py2ls-0.2.4.34 → py2ls-0.2.4.36}/py2ls/ips.py

@@ -32,10 +32,16 @@ def backup(
     tar="/Users/macjianfeng/Dropbox/github/python/py2ls/py2ls/",
     kind="py",
     overwrite=True,
-):
-    f = listdir(src, kind)
-    [copy(i, tar, overwrite=overwrite) for i in f.path]
-    print(f"all files are copied from {os.path.basename(src)} to {tar}")
+    reverse=False,
+    verbose=False
+):  
+    if reverse:
+        src, tar = tar,src
+        print(f"reversed")
+    f = listdir(src, kind, verbose=verbose)
+    [copy(i, tar, overwrite=overwrite, verbose=verbose) for i in f.path]
+    print(f"all files are copied from {os.path.basename(src)} to {tar}") if verbose else None
+
 def run_once_within(duration=60, reverse=False):  # default 60s
     import time
 
@@ -2693,13 +2699,41 @@ def fload(fpath, kind=None, **kwargs):
     def load_excel(fpath, **kwargs):
         engine = kwargs.get("engine", "openpyxl")
         verbose = kwargs.pop("verbose", False)
-        if run_once_within(reverse=True):
-            use_pd("read_excel", verbose=verbose)
-        df = pd.read_excel(fpath, engine=engine, **kwargs)
+        password=kwargs.pop("password",None)
+
+        if not password:
+            if run_once_within(reverse=True):
+                use_pd("read_excel", verbose=verbose)
+            df = pd.read_excel(fpath, engine=engine, **kwargs)
+            try:
+                meta = pd.ExcelFile(fpath)
+                print(f"n_sheet={len(meta.sheet_names)},\t'sheetname = 0 (default)':")
+                [print(f"{i}:\t{i_}") for i, i_ in enumerate(meta.sheet_names)]
+            except:
+                pass
+            return df
+        #* needs a password?
+        import msoffcrypto # pip install msoffcrypto-tool
+        from io import BytesIO
+
+        # Open the encrypted Excel file
+        with open(fpath, 'rb') as f:
+            try:
+                office_file = msoffcrypto.OfficeFile(f)
+                office_file.load_key(password=password)  # Provide the password
+                decrypted = BytesIO()
+                office_file.decrypt(decrypted)
+            except:
+                office_file = msoffcrypto.OfficeFile(f)
+                office_file.load_key(password=depass(password))  # Provide the password
+                decrypted = BytesIO()
+                office_file.decrypt(decrypted)
+        decrypted.seek(0)
+        df = pd.read_excel(decrypted, engine=engine, **kwargs)
         try:
-            meata = pd.ExcelFile(fpath)
-            print(f"n_sheet={len(meata.sheet_names)},\t'sheetname = 0 (default)':")
-            [print(f"{i}:\t{i_}") for i, i_ in enumerate(meata.sheet_names)]
+            meta = pd.ExcelFile(fpath)
+            print(f"n_sheet={len(meta.sheet_names)},\t'sheetname = 0 (default)':")
+            [print(f"{i}:\t{i_}") for i, i_ in enumerate(meta.sheet_names)]
         except:
             pass
         return df
@@ -3283,11 +3317,16 @@ def fsave(
         df = pd.DataFrame(data)
         df.to_csv(fpath, **kwargs_valid)
 
-    def save_xlsx(fpath, data, **kwargs):
+
+    def save_xlsx(fpath, data, password=None, **kwargs):
+        import msoffcrypto
+        from io import BytesIO
         verbose = kwargs.pop("verbose", False)
         sheet_name = kwargs.pop("sheet_name", "Sheet1")
+        
         if run_once_within(reverse=True):
             use_pd("to_excel", verbose=verbose)
+        
         if any(kwargs):
             format_excel(df=data, filename=fpath, **kwargs)
         else:
@@ -3312,16 +3351,30 @@ def fsave(
                 kwargs.pop(key, None)
 
             df = pd.DataFrame(data)
-            # Check if the file exists, then append the sheet, otherwise create a new file
-            try:
-                # Use ExcelWriter with append mode if the file exists
-                with pd.ExcelWriter(
-                    fpath, engine="openpyxl", mode="a", if_sheet_exists="new"
-                ) as writer:
-                    df.to_excel(writer, sheet_name=sheet_name, index=False, **kwargs)
-            except FileNotFoundError:
-                # If file doesn't exist, create a new one
-                df.to_excel(fpath, sheet_name=sheet_name, index=False, **kwargs)
+            
+            # Write to Excel without password first
+            temp_file = BytesIO()
+            df.to_excel(temp_file, sheet_name=sheet_name, index=False, engine="xlsxwriter", **kwargs)
+
+            # If a password is provided, encrypt the file
+            if password:
+                temp_file.seek(0)
+                office_file = msoffcrypto.OfficeFile(temp_file)
+                office_file.load_key(password=password)  # Provide the password
+
+                # Encrypt and save the file
+                with open(fpath, 'wb') as encrypted_file:
+                    office_file.encrypt(encrypted_file)
+            else:
+                # Save the file without encryption if no password is provided
+                try:
+                    # Use ExcelWriter with append mode if the file exists
+                    with pd.ExcelWriter(fpath, engine="openpyxl", mode="a", if_sheet_exists="new") as writer:
+                        df.to_excel(writer, sheet_name=sheet_name, index=False, **kwargs)
+                except FileNotFoundError:
+                    # If file doesn't exist, create a new one
+                    df.to_excel(fpath, sheet_name=sheet_name, index=False, **kwargs)
+
 
     def save_ipynb(fpath, data, **kwargs):
         # Split the content by code fences to distinguish between code and markdown
@@ -4410,13 +4463,13 @@ def func_list(lib_name, opt="call"):
     return list_func(lib_name, opt=opt)
 
 
-def copy(src, dst, overwrite=False):
+def copy(src, dst, overwrite=False, verbose=True):
     """Copy a file from src to dst."""
     try:
         dir_par_dst = os.path.dirname(dst)
         if not os.path.isdir(dir_par_dst):
             mkdir(dir_par_dst)
-        print(dir_par_dst)
+        print(dir_par_dst) if verbose else None
         src = Path(src)
         dst = Path(dst)
         if not src.is_dir():
@@ -4431,7 +4484,7 @@ def copy(src, dst, overwrite=False):
                         f"{dst.stem}_{datetime.now().strftime('_%H%M%S')}{dst.suffix}"
                     )
             shutil.copy(src, dst)
-            print(f"\n Done! copy to {dst}\n")
+            print(f"\n Done! copy to {dst}\n") if verbose else None
         else:
             dst = dst / src.name
             if dst.exists():
@@ -4442,7 +4495,7 @@ def copy(src, dst, overwrite=False):
                         f"{dst.stem}_{datetime.now().strftime('%H%M%S')}"
                     )
             shutil.copytree(src, dst)
-            print(f"\n Done! copy to {dst}\n")
+            print(f"\n Done! copy to {dst}\n") if verbose else None
 
     except Exception as e:
         logging.error(f"Failed {e}")
@@ -4452,7 +4505,7 @@ def cut(src, dst, overwrite=False):
     return move(src=src, dst=dst, overwrite=overwrite)
 
 
-def move(src, dst, overwrite=False):
+def move(src, dst, overwrite=False, verbose=True):
     try:
         dir_par_dst = os.path.dirname(dst)
         if not os.path.isdir(dir_par_dst):
@@ -4470,7 +4523,7 @@ def move(src, dst, overwrite=False):
                     f"{dst.stem}_{datetime.now().strftime('_%H%M%S')}{dst.suffix}"
                 )
         shutil.move(src, dst)
-        print(f"\n Done! moved to {dst}\n")
+        print(f"\n Done! moved to {dst}\n") if verbose else None
     except Exception as e:
         logging.error(f"Failed to move file from {src} to {dst}: {e}")
 
@@ -4729,6 +4782,19 @@ def figsave(*args, dpi=300):
         plt.savefig(fname, format="emf", dpi=dpi, bbox_inches="tight", pad_inches=0)
     elif ftype.lower() == "fig":
         plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi, pad_inches=0)
+
+    elif ftype.lower() == "ico":
+        # Ensure the image is in a format that can be saved as an icon (e.g., 32x32, 64x64, etc.)
+        if img is None:  # If no image is provided, use the matplotlib figure
+            img = plt.figure()
+            img.savefig(fname, dpi=dpi, format="png", bbox_inches="tight")
+            img = Image.open(fname)  # Load the saved figure image
+
+        # Resize the image to typical icon sizes and save it as .ico
+        icon_sizes = [(32, 32), (64, 64), (128, 128), (256, 256)]
+        img = img.convert("RGBA")  # Ensure it has an alpha channel for transparency
+        img.save(fname, format="ICO", sizes=icon_sizes)
+        print(f"Icon saved @: {fname} with sizes: {icon_sizes}")
     print(f"\nSaved @: dpi={dpi}\n{fname}")
 
 
@@ -5507,7 +5573,6 @@ def detect_angle(image, by="median", template=None):
         print(f"Unknown method {by}: supported methods: {methods}")
         return 0
 
-
 def imgsets(img, 
             auto:bool=True,
             size=None,
@@ -5594,7 +5659,6 @@ def imgsets(img,
         "birefnet-massive": "A pre-trained model with massive dataset.",
     }
     models_support_rem=list(rem_models.keys())
-
     str_usage="""
     imgsets(dir_img, auto=1, color=1.5, plot_=0)
     imgsets(dir_img, color=2)
@@ -5714,7 +5778,7 @@ def imgsets(img,
         kwargs = {**auto_enhance(img_update), **kwargs}
     params=["sharp","color","contrast","bright","crop","rotate",'size',"resize",
             "thumbnail","cover","contain","filter","fit","pad",
-            "rem","rm","back","bg_color","cut",'gamma','flip']
+            "rem","rm","back","bg_color","cut","gamma","flip","booster"]
     for k, value in kwargs.items():
         k = strcmp(k, params)[0] # correct the param name
         if "shar" in k.lower():
@@ -5735,7 +5799,7 @@ def imgsets(img,
                     img_update = ImageOps.autocontrast(img_update)
                     print("autocontrasted")
                 except Exception as e:
-                    print(f"Failed 'autocontrasted':{e}")
+                    print(f"Failed 'auto-contrasted':{e}")
         elif "bri" in k.lower():
             enhancer = ImageEnhance.Brightness(img_update)
             img_update = enhancer.enhance(value)
@@ -5780,9 +5844,12 @@ def imgsets(img,
             from rembg import remove, new_session
             if verbose:
                 preview(rem_models)
+            
+            print(f"supported modles: {models_support_rem}")
             model=strcmp(model, models_support_rem)[0]
             session = new_session(model)
             if isinstance(value, bool):
+                print(f"using model:{model}")
                 img_update = remove(img_update, session=session)
             elif value and isinstance(value, (int, float, list)):
                 if verbose:
@@ -5817,6 +5884,7 @@ def imgsets(img,
                 img_update = remove(img_update, bgcolor=value, session=session)
             elif isinstance(value, str):
                 # use custom model
+                print(f"using model:{strcmp(value, models_support_rem)[0]}")
                 img_update = remove(img_update, session=new_session(strcmp(value, models_support_rem)[0]))
         elif "bg" in k.lower() and "color" in k.lower():
             from rembg import remove
@@ -5827,7 +5895,33 @@ def imgsets(img,
                 if len(value) == 3:
                     value += (255,)
                 img_update = remove(img_update, bgcolor=value)
-        
+        elif 'boost' in k.lower():
+            import torch
+            from realesrgan import RealESRGANer
+            if verbose:
+                print("Applying Real-ESRGAN for image reconstruction...")
+            if isinstance(value, bool):
+                scale=4
+            elif isinstance(value, (float, int)):
+                scale=value
+            else:
+                scale=4
+
+            # try:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            dir_curr_script = os.path.dirname(os.path.abspath(__file__))
+            model_path = dir_curr_script + "/data/RealESRGAN_x4plus.pth"
+            model_RealESRGAN = RealESRGANer(device=device, 
+                                            scale=scale, 
+                                            model_path=model_path,
+                                            model="RealESRGAN_x4plus"
+                                            )
+            # https://github.com/xinntao/Real-ESRGAN?tab=readme-ov-file#python-script
+
+            img_update = model_RealESRGAN.enhance(np.array(img_update))[0] 
+            # except Exception as e:
+            #     print(f"Failed to apply Real-ESRGAN: {e}")
+
         # elif "ga" in k.lower() and "m" in k.lower():
         #     img_update = gamma_correction(img_update, gamma=value)
     # Display the image if requested
@@ -10728,4 +10822,227 @@ def mouse(
     #         "Image not found. Ensure the image is visible and parameters are correct."
     #     )
     # except Exception as e:
-    #     print(f"An error occurred: {e}")
+    #     print(f"An error occurred: {e}")
+
+
+
+def py2installer(
+    script_path: str=None,
+    flatform:str="mingw64",
+    output_dir: str = "dist",
+    icon_path: str = None,
+    extra_data: list = None,
+    hidden_imports: list = None,
+    plugins:list=None,
+    use_nuitka: bool = True,
+    onefile: bool = True,
+    console: bool = True,
+    clean_build: bool = False,
+    additional_args: list = None,
+    verbose: bool = True,
+    use_docker: bool = False,
+    docker_image: str = "python:3.12-slim",
+):
+    """
+    to package Python scripts into standalone application.
+
+    script_path (str): Path to the Python script to package.
+    output_dir (str): Directory where the executable will be stored.
+    icon_path (str): Path to the .ico file for the executable icon.
+    extra_data (list): List of additional data files or directories in "source:dest" format.
+    hidden_imports (list): List of hidden imports to include.
+    plugins (list): List of plugins imports to include.e.g., 'tk-inter'
+    use_nuitka (bool): Whether to use Nuitka instead of PyInstaller.
+    onefile (bool): If True, produces a single executable file.
+    console (bool): If False, hides the console window (GUI mode).
+    clean_build (bool): If True, cleans previous build and dist directories.
+    additional_args (list): Additional arguments for PyInstaller/Nuitka.
+    verbose (bool): If True, provides detailed logs.
+    use_docker (bool): If True, uses Docker to package the script.
+    docker_image (str): Docker image to use for packaging.
+
+    """
+
+    import os
+    import sys
+    import shutil
+    import subprocess
+    import sys
+    import glob
+    from pathlib import Path
+    if run_once_within():
+        usage_str="""
+            # build locally 
+            py2installer(
+                script_path="update_tab.py",
+                output_dir="dist",
+                icon_path="icon4app.ico",
+                extra_data=["dat/*.xlsx:dat"],
+                hidden_imports=["msoffcrypto", "tkinter", "pandas", "numpy"],
+                onefile=True,
+                console=False,
+                clean_build=True,
+                verbose=True,
+            )
+            # build via docker
+            py2installer(
+                "my_script.py",
+                output_dir="dist",
+                onefile=True,
+                clean_build=True,
+                use_docker=True,
+                docker_image="python:3.12-slim"
+            )
+        """
+        print(usage_str)
+        if verbose:
+            return
+        else:
+            pass
+    # Check if the script path exists
+    script_path = Path(script_path)
+    if not script_path.exists():
+        raise FileNotFoundError(f"Script '{script_path}' not found.")
+
+     # Clean build and dist directories if requested
+    if clean_build:
+        for folder in ["build", "dist"]:
+            folder_path = Path(folder)
+            if folder_path.exists():
+                shutil.rmtree(folder_path, ignore_errors=True)
+        # Recreate the folders
+        for folder in ["build", "dist"]:
+            folder_path = Path(folder)
+            folder_path.mkdir(parents=True, exist_ok=True)
+                
+
+    if use_docker:
+        # Ensure Docker is installed
+        try:
+            subprocess.run(["docker", "--version"], check=True, capture_output=True, text=True)
+        except FileNotFoundError:
+            raise EnvironmentError("Docker is not installed or not in the PATH.")
+
+        # Prepare Docker volume mappings
+        script_dir = script_path.parent.resolve()
+        dist_path = Path(output_dir).resolve()
+        volumes = [
+            f"{script_dir}:/app:rw",
+            f"{dist_path}:/output:rw",
+        ]
+        docker_cmd = [
+            "docker", "run", "--rm",
+            "-v", volumes[0],
+            "-v", volumes[1],
+            docker_image,
+            "bash", "-c",
+        ]
+
+        # Build the packaging command inside the container
+        cmd = ["nuitka"] if use_nuitka else ["pyinstaller"]
+        if onefile:
+            cmd.append("--onefile")
+        if not console:
+            cmd.append("--windowed")
+        cmd.extend(["--distpath", "/output"])
+        if icon_path:
+            cmd.extend(["--icon", f"/app/{Path(icon_path).name}"])
+        if extra_data:
+            for data in extra_data:
+                cmd.extend(["--add-data", f"/app/{data}"])
+        if hidden_imports:
+            for hidden in hidden_imports:
+                cmd.extend(["--hidden-import", hidden])
+        if additional_args:
+            cmd.extend(additional_args)
+        cmd.append(f"/app/{script_path.name}")
+
+        # Full command to execute inside the container
+        docker_cmd.append(" ".join(cmd))
+
+        if verbose:
+            print(f"Running Docker command: {' '.join(docker_cmd)}")
+
+        # Run Docker command
+        try:
+            subprocess.run(
+                docker_cmd,
+                capture_output=not verbose,
+                text=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Error during Docker packaging:\n{e.stderr}", file=sys.stderr)
+            raise
+    else:
+        # Handle local packaging (native build)
+        cmd = ["nuitka"]
+        cmd.append("--standalone")  # Make sure to use --standalone for independent environments
+        if 'min' in flatform.lower():
+            cmd.append("--mingw64")
+        if onefile:
+            cmd.append("--onefile")
+        if not console:
+            # cmd.append("--windows-disable-console")  # Disabled based on your request
+            pass
+
+        cmd.extend([f"--output-dir={output_dir}"])  # Correct the space issue here
+        if icon_path:
+            icon_path = Path(icon_path)
+            if not icon_path.exists():
+                raise FileNotFoundError(f"Icon file '{icon_path}' not found.")
+            cmd.extend([f"--windows-icon-from-ico={icon_path}"])
+
+        if extra_data:
+            for data in extra_data:
+                if "*" in data:
+                    matches = glob.glob(data.split(":")[0])
+                    for match in matches:
+                        dest = data.split(":")[1]
+                        cmd.extend(
+                            [
+                                "--include-data-file=" if use_nuitka else "--add-data",
+                                f"{match}:{dest}",
+                            ]
+                        )
+                else:
+                    cmd.extend(
+                        ["--include-data-file=" if use_nuitka else "--add-data", data]
+                    )
+
+        if hidden_imports:
+            cmd.extend([f"--nofollow-import-to={','.join(hidden_imports)}"])
+
+        if plugins:
+            for plugin in plugins:
+                cmd.extend([f"--plugin-enable={plugin}"])
+
+        if additional_args:
+            cmd.extend(additional_args)
+
+        # Add the script path (final positional argument)
+        cmd.append(str(script_path))
+
+        # Ensure Windows shell compatibility
+        shell_flag = sys.platform.startswith("win")
+
+        # Run the command
+        if verbose:
+            print(f"Running command: {' '.join(cmd)}")
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=not verbose,
+                text=True,
+                shell=shell_flag,
+                check=True,
+            )
+            if verbose:
+                print(result.stdout)
+        except subprocess.CalledProcessError as e:
+            print(f"Error during packaging:\n{e.stderr}", file=sys.stderr)
+            print(" ".join(cmd))
+            raise
+
+    print("\nPackaging complete. Check the output directory for the executable.")
+ 

{py2ls-0.2.4.34 → py2ls-0.2.4.36}/py2ls/netfinder.py

@@ -333,32 +333,6 @@ def parse_cookies(cookies_str):
 
     return cookies_dict
 
-class FetchSpider(scrapy.Spider):
-    name = "fetch_spider"
-    
-    def __init__(self, url, parser="html.parser", cookies=None, headers=None, *args, **kwargs):
-        super(FetchSpider, self).__init__(*args, **kwargs)
-        self.start_urls = [url]
-        self.cookies = cookies
-        self.headers = headers
-        self.parser = parser
-    
-    def start_requests(self):
-        for url in self.start_urls:
-            yield scrapy.Request(
-                url,
-                cookies=self.cookies,
-                headers=self.headers,
-                callback=self.parse
-            )
-    
-    def parse(self, response):
-        # Use the desired parser (default: html.parser)
-        from bs4 import BeautifulSoup
-        soup = BeautifulSoup(response.text, self.parser)
-        yield {"content": soup}
-
-
 def fetch_scrapy(
     url,
     parser="html.parser",
@@ -367,7 +341,7 @@ def fetch_scrapy(
     settings=None,
 ):
     """
-    Fetches content using Scrapy.
+    Fetches content using Scrapy with proper reactor handling.
 
     Args:
         url (str): The URL to scrape.
@@ -380,9 +354,10 @@ def fetch_scrapy(
         dict: Parsed content as a dictionary.
     """
     from scrapy.utils.project import get_project_settings
-    from scrapy.crawler import CrawlerProcess
+    from scrapy.crawler import CrawlerRunner
     from scrapy.signalmanager import dispatcher
     from scrapy import signals
+    from twisted.internet import reactor, defer
     import scrapy
 
     # Container for scraped content
@@ -403,19 +378,30 @@ def fetch_scrapy(
         }
     )
 
-    # Initialize and configure Scrapy process
-    process = CrawlerProcess(settings=process_settings)
+    # Connect item scraped signal
     dispatcher.connect(handle_item, signal=signals.item_scraped)
 
-    # Start the Scrapy crawl
-    process.crawl(
-        FetchSpider,
-        url=url,
-        parser=parser,
-        cookies=cookies,
-        headers=headers,
-    )
-    process.start()  # Blocks until all crawls are finished
+    # Asynchronous Twisted function
+    @defer.inlineCallbacks
+    def crawl():
+        runner = CrawlerRunner(settings=process_settings)
+        yield runner.crawl(
+            FetchSpider,
+            url=url,
+            parser=parser,
+            cookies=cookies,
+            headers=headers,
+        )
+        reactor.stop()
+
+    # Start the reactor if not already running
+    if not reactor.running:
+        crawl()
+        reactor.run()  # Blocks until the crawl finishes
+    else:
+        # Run the crawl if the reactor is already running
+        d = crawl()
+        d.addBoth(lambda _: reactor.stop())
 
     # Return the first scraped content or None if empty
     return content[0] if content else None
@@ -654,7 +640,11 @@ def fetch_all(
                 "COOKIES_ENABLED": True if cookies else False,
                 "LOG_LEVEL": "WARNING",  # Reduce log verbosity
             }
-            content=fetch_scrapy(url, parser=parser, cookies=cookies, headers=headers, settings=settings)
+            content=fetch_scrapy(url, 
+                                 parser=parser, 
+                                 cookies=cookies, 
+                                 headers=headers, 
+                                 settings=settings)
             return parser, content 
 
     except requests.RequestException as e: