py2ls 0.1.4.4__py3-none-any.whl → 0.1.4.5__py3-none-any.whl

py2ls/.git/COMMIT_EDITMSG

@@ -1,4 +1 @@
-listdir type
-
-listdir type 大类别和精确类别,
-e.g.,  kind='img', means filtter all kinds of images, .tiff, tif, jpg, png...
+update xample_imgsets

py2ls/.git/FETCH_HEAD

@@ -1 +1 @@
-6427a4edff08f93d98f511418423f09f2ab90bcd		branch 'main' of https://github.com/Jianfengliu0413/py2ls
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f		branch 'main' of https://github.com/Jianfengliu0413/py2ls

py2ls/.git/logs/HEAD

@@ -8,3 +8,7 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
 48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type
+6427a4edff08f93d98f511418423f09f2ab90bcd d9dfa5aee51e92a541b707e8e7baea6f06deff98 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718570098 +0200	commit: imgsets update
+d9dfa5aee51e92a541b707e8e7baea6f06deff98 e31356f90ea6dd0577b5e0b40b206319adcbf085 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718618413 +0200	commit: add thumbnail
+e31356f90ea6dd0577b5e0b40b206319adcbf085 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718619568 +0200	commit: Update ips.py
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 627c81b23b4e56e87b042b650b0103653cc9e34a Jianfeng <Jianfeng.Liu0413@gmail.com> 1718698925 +0200	commit: update xample_imgsets

py2ls/.git/logs/refs/heads/main

@@ -8,3 +8,7 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
 48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type
+6427a4edff08f93d98f511418423f09f2ab90bcd d9dfa5aee51e92a541b707e8e7baea6f06deff98 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718570098 +0200	commit: imgsets update
+d9dfa5aee51e92a541b707e8e7baea6f06deff98 e31356f90ea6dd0577b5e0b40b206319adcbf085 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718618413 +0200	commit: add thumbnail
+e31356f90ea6dd0577b5e0b40b206319adcbf085 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718619568 +0200	commit: Update ips.py
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 627c81b23b4e56e87b042b650b0103653cc9e34a Jianfeng <Jianfeng.Liu0413@gmail.com> 1718698925 +0200	commit: update xample_imgsets

py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -24,3 +24,14 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553371 +0200	remote set-head
 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553465 +0200	remote set-head
 6427a4edff08f93d98f511418423f09f2ab90bcd 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718555183 +0200	remote set-head
+6427a4edff08f93d98f511418423f09f2ab90bcd 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718570095 +0200	remote set-head
+d9dfa5aee51e92a541b707e8e7baea6f06deff98 d9dfa5aee51e92a541b707e8e7baea6f06deff98 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718570102 +0200	remote set-head
+e31356f90ea6dd0577b5e0b40b206319adcbf085 e31356f90ea6dd0577b5e0b40b206319adcbf085 Jianfeng Liu <macjianfeng@JFLMBP.local> 1718618416 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718619571 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718698084 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718699917 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718701729 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718703544 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718705358 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718707170 +0200	remote set-head
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng Liu <macjianfeng@JFLMBP.local> 1718708982 +0200	remote set-head

py2ls/.git/logs/refs/remotes/origin/main

@@ -8,3 +8,6 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526959 +0200	update by push
 48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553464 +0200	update by push
 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555183 +0200	update by push
+6427a4edff08f93d98f511418423f09f2ab90bcd d9dfa5aee51e92a541b707e8e7baea6f06deff98 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718570102 +0200	update by push
+d9dfa5aee51e92a541b707e8e7baea6f06deff98 e31356f90ea6dd0577b5e0b40b206319adcbf085 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718618415 +0200	update by push
+e31356f90ea6dd0577b5e0b40b206319adcbf085 1ab4585881a6a42889f01aa0cfe25fd5acfaf46f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718619570 +0200	update by push

py2ls/.git/objects/62/d90ccf8cbefdc2e4fd475e7c6f4f76e9fdf801

@@ -0,0 +1,3 @@
+x
M��
+�0���S�]�����tMY�Kk���s��\��w~\�n��a�Y9����DP�\a�����$�p�8�,
+���#[Ԅ��2fJ�Y��݄�I#���RcGb����~�k"*���Z�'��/Y�n]-��v�)�6{m����]�J�

py2ls/.git/refs/heads/main

@@ -1 +1 @@
-6427a4edff08f93d98f511418423f09f2ab90bcd
+627c81b23b4e56e87b042b650b0103653cc9e34a

py2ls/.git/refs/remotes/origin/main

@@ -1 +1 @@
-6427a4edff08f93d98f511418423f09f2ab90bcd
+1ab4585881a6a42889f01aa0cfe25fd5acfaf46f

py2ls/__init__.py

@@ -7,11 +7,10 @@ from . import (
     correlators, 
     dbhandler, 
     freqanalysis, 
-    internet_finder, 
     ips, 
     netfinder, 
     sleep_events_detectors, 
     translator, 
-    version, 
     wb_detector, 
     )
+from .ips import *

py2ls/ips.py

@@ -3,7 +3,6 @@ import numpy as np
 import pandas as pd
 import json
 import matplotlib.pyplot as plt
-# from functools import partial
 import seaborn as sns
 import scienceplots
 import matplotlib
@@ -15,7 +14,7 @@ from scipy import stats
 import matplotlib.ticker as tck 
 from cycler import cycler
 import re
-from PIL import ImageEnhance, ImageOps,ImageFilter
+from PIL import Image,ImageEnhance, ImageOps,ImageFilter
 from rembg import remove,new_session
 from mpl_toolkits.mplot3d import Axes3D
 import docx
@@ -37,6 +36,7 @@ from box import Box, BoxList
 from numerizer import numerize
 from tqdm import tqdm
 import mimetypes
+from pprint import pp
 
 def str2num(s, *args):
     delimiter = None
@@ -2107,7 +2107,6 @@ def load_img(fpath):
         FileNotFoundError: If the specified file is not found.
         OSError: If the specified file cannot be opened or is not a valid image file.
     """
-    from PIL import Image
 
     try:
         img = Image.open(fpath)
@@ -2188,6 +2187,7 @@ def apply_filter(img, *args):
             raise ValueError(
                 f"Unsupported filter: {filter_name}, should be one of: {supported_filters}"
             )
+
     for arg in args:
         if isinstance(arg, str):
             filter_name = arg
@@ -2286,6 +2286,29 @@ def imgsets(
     Note:
         The "color" and "enhance" enhancements are not implemented in this function.
     """
+    supported_filters = [
+                "BLUR",
+                "CONTOUR",
+                "DETAIL",
+                "EDGE_ENHANCE",
+                "EDGE_ENHANCE_MORE",
+                "EMBOSS",
+                "FIND_EDGES",
+                "SHARPEN",
+                "SMOOTH",
+                "SMOOTH_MORE",
+                "MIN_FILTER",
+                "MAX_FILTER",
+                "MODE_FILTER",
+                "MULTIBAND_FILTER",
+                "GAUSSIAN_BLUR",
+                "BOX_BLUR",
+                "MEDIAN_FILTER",
+            ]
+    print("sets: a dict,'sharp:1.2','color','contrast:'auto' or 1.2','bright', 'crop: x_upperleft,y_upperleft, x_lowerright, y_lowerright','rotation','resize','rem or background'")
+    print(f"usage: filter_kws 'dict' below:")
+    pp([str(i).lower() for i in supported_filters])
+    print("\nlog:\n")
     def confirm_rembg_models(model_name):
         models_support = [
             "u2net",
@@ -2734,6 +2757,46 @@ def figsets(*args):
         plt.tight_layout()
         plt.gcf().align_labels()
 
+def thumbnail(dir_img_list,figsize=(10,10),dpi=100, dir_save=None, kind='.png'):
+    """
+    Display a thumbnail figure of all images in the specified directory.
+    Args:
+        dir_img_list (list): List of the Directory containing the images.
+    """
+    num_images = len(dir_img_list)
+    if not kind.startswith('.'):
+        kind='.'+kind
+
+    if num_images == 0:
+        print("No images found to display.")
+        return
+    grid_size = int(num_images ** 0.5) + 1 # Determine grid size
+    fig, axs = plt.subplots(grid_size, grid_size, figsize=figsize,dpi=dpi)
+    for ax, image_file in zip(axs.flatten(), dir_img_list):
+        try:
+            img = Image.open(image_file)
+            ax.imshow(img)
+            ax.axis('off')
+        except:
+            continue
+    # for ax in axs.flatten():
+    #     ax.axis('off')
+    [ax.axis("off") for ax in axs.flatten()]
+    plt.tight_layout()
+    if dir_save is None:
+        plt.show()
+    else:
+        if basename(dir_save):
+            fname= basename(dir_save) +kind
+        else:
+            fname= "_thumbnail_"+basename(dirname(dir_save)[:-1])+'.png'
+        if dirname(dir_img_list[0]) == dirname(dir_save):
+            figsave(dirname(dir_save[:-1]),fname)
+        else:
+            figsave(dirname(dir_save),fname)
+# usage:
+# fpath = "/Users/macjianfeng/Dropbox/github/python/py2ls/tests/xample_netfinder/images/"
+# thumbnail(listdir(fpath,'png').fpath.to_list(),dir_save=dirname(fpath))
 def read_mplstyle(style_file):
     # Load the style file
     plt.style.use(style_file)

py2ls/netfinder.py

@@ -21,8 +21,11 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-import pprint
+from pprint import pp
 import mimetypes
+import io
+import matplotlib.pyplot as plt
+from PIL import Image
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -313,7 +316,7 @@ def pdf_detector(url, contains=None, dir_save=None,booster=False):
             pdf_links = filter_links(links=links_all, contains=["pdf"])
 
     if pdf_links:
-        pprint.pp(f"pdf detected\n{pdf_links}")
+        pp(f"pdf detected{pdf_links}")
     else:
         print('no pdf file')
     if dir_save:
@@ -399,9 +402,13 @@ def find_img(url, dir_save="images", verbose=True):
                 print(f"Failed to process image {image_url}: {e}")
         print(f"images were saved at\n{dir_save}")
         if verbose:
-            display_thumbnail_figure(flist(dir_save,filter='img'),dpi=200)
+            display_thumbnail_figure(flist(dir_save,filter='img'),dpi=100)
     return content
-
+def svg_to_png(svg_file):
+    with WandImage(filename=svg_file, resolution=300) as img:
+        img.format = 'png'
+        png_image = img.make_blob()
+    return Image.open(io.BytesIO(png_image))
 def display_thumbnail_figure(dir_img_list,figsize=(10,10),dpi=100):
     import matplotlib.pyplot as plt
     from PIL import Image
@@ -422,13 +429,14 @@ def display_thumbnail_figure(dir_img_list,figsize=(10,10),dpi=100):
     fig, axs = plt.subplots(grid_size, grid_size, figsize=figsize,dpi=dpi)
 
     for ax, image_file in zip(axs.flatten(), dir_img_list):
-        img = Image.open(image_file)
-        ax.imshow(img)
-        ax.axis('off')  # Hide axes
-
+        try:
+            img = Image.open(image_file)
+            ax.imshow(img)
+            ax.axis('off')  # Hide axes
+        except:
+            continue
     # Hide remaining subplots
-    for ax in axs.flatten()[num_images:]:
-        ax.axis('off')
+    [ax.axis("off") for ax in axs.flatten()]
 
     plt.tight_layout()
     plt.show()

{py2ls-0.1.4.4.dist-info → py2ls-0.1.4.5.dist-info}/METADATA

@@ -1,12 +1,149 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.4.4
+Version: 0.1.4.5
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: Deprecated (>=1.2.14,<2.0.0)
+Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: MarkupSafe (>=2.1.5,<3.0.0)
+Requires-Dist: PyMatting (>=1.1.12,<2.0.0)
+Requires-Dist: PyPDF2 (>=3.0.1,<4.0.0)
+Requires-Dist: PySocks (>=1.7.1,<2.0.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Pygments (>=2.18.0,<3.0.0)
+Requires-Dist: SciencePlots (>=2.1.1,<3.0.0)
+Requires-Dist: appnope (>=0.1.4,<0.2.0)
+Requires-Dist: appscript (>=1.2.5,<2.0.0)
+Requires-Dist: asttokens (>=2.4.1,<3.0.0)
+Requires-Dist: attrs (>=23.2.0,<24.0.0)
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: certifi (>=2024.6.2,<2025.0.0)
+Requires-Dist: chardet (>=3.0.4,<4.0.0)
+Requires-Dist: charset-normalizer (>=3.3.2,<4.0.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: colorcet (>=3.1.0,<4.0.0)
+Requires-Dist: coloredlogs (>=15.0.1,<16.0.0)
+Requires-Dist: comm (>=0.2.2,<0.3.0)
+Requires-Dist: contourpy (>=1.2.1,<2.0.0)
+Requires-Dist: cycler (>=0.12.1,<0.13.0)
+Requires-Dist: debugpy (>=1.8.1,<2.0.0)
+Requires-Dist: decorator (>=5.1.1,<6.0.0)
+Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
+Requires-Dist: docx (>=0.2.4,<0.3.0)
+Requires-Dist: docx2pdf (>=0.1.8,<0.2.0)
+Requires-Dist: executing (>=2.0.1,<3.0.0)
+Requires-Dist: fake-useragent (>=1.5.1,<2.0.0)
+Requires-Dist: flatbuffers (>=24.3.25,<25.0.0)
+Requires-Dist: fonttools (>=4.53.0,<5.0.0)
+Requires-Dist: fpdf (>=1.7.2,<2.0.0)
+Requires-Dist: googletrans (>=4.0.0rc1,<5.0.0)
+Requires-Dist: h11 (>=0.9.0,<0.10.0)
+Requires-Dist: h2 (>=3.2.0,<4.0.0)
+Requires-Dist: hpack (>=3.0.0,<4.0.0)
+Requires-Dist: hstspreload (>=2024.6.1,<2025.0.0)
+Requires-Dist: httpcore (>=0.9.1,<0.10.0)
+Requires-Dist: httpx (>=0.13.3,<0.14.0)
+Requires-Dist: humanfriendly (>=10.0,<11.0)
+Requires-Dist: hyperframe (>=5.2.0,<6.0.0)
+Requires-Dist: idna (>=2.10,<3.0)
+Requires-Dist: imageio (>=2.34.1,<3.0.0)
+Requires-Dist: img2pdf (>=0.5.1,<0.6.0)
+Requires-Dist: ipykernel (>=6.29.4,<7.0.0)
+Requires-Dist: ipython (>=8.25.0,<9.0.0) ; python_version >= "3.9" and python_version < "4.0"
+Requires-Dist: jedi (>=0.19.1,<0.20.0)
+Requires-Dist: joblib (>=1.3.2,<2.0.0)
+Requires-Dist: jsonschema (>=4.22.0,<5.0.0)
+Requires-Dist: jsonschema-specifications (>=2023.12.1,<2024.0.0)
+Requires-Dist: jupyter_client (>=8.6.2,<9.0.0)
+Requires-Dist: jupyter_core (>=5.7.2,<6.0.0)
+Requires-Dist: kiwisolver (>=1.4.5,<2.0.0)
+Requires-Dist: langdetect (>=1.0.9,<2.0.0)
+Requires-Dist: lazy_loader (>=0.4,<0.5)
+Requires-Dist: libretranslatepy (>=2.1.1,<3.0.0)
+Requires-Dist: llvmlite (>=0.42.0,<0.43.0)
+Requires-Dist: lxml (>=4.9.4,<5.0.0)
+Requires-Dist: matplotlib (>=3.8.4,<4.0.0)
+Requires-Dist: matplotlib-inline (>=0.1.7,<0.2.0)
+Requires-Dist: mne (>=1.6.0,<2.0.0)
+Requires-Dist: mpmath (>=1.3.0,<2.0.0)
+Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
+Requires-Dist: networkx (>=3.3,<4.0) ; python_version >= "3.10" and python_version < "4.0"
+Requires-Dist: nltk (>=3.8.1,<4.0.0)
+Requires-Dist: numba (>=0.59.1,<0.60.0)
+Requires-Dist: numerizer (>=0.2.3,<0.3.0)
+Requires-Dist: numpy (>=1.26.4,<2.0.0)
+Requires-Dist: onnxruntime (>=1.18.0,<2.0.0)
+Requires-Dist: opencv-contrib-python (>=4.9.0.80,<5.0.0.0)
+Requires-Dist: opencv-python (>=4.9.0.80,<5.0.0.0)
+Requires-Dist: opencv-python-headless (>=4.9.0.80,<5.0.0.0)
+Requires-Dist: outcome (>=1.3.0.post0,<2.0.0)
+Requires-Dist: packaging (>=24.1,<25.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: pandas-flavor (>=0.6.0,<0.7.0)
+Requires-Dist: parso (>=0.8.4,<0.9.0)
+Requires-Dist: patsy (>=0.5.6,<0.6.0)
+Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
+Requires-Dist: pdf2img (>=0.1.2,<0.2.0)
+Requires-Dist: pexpect (>=4.9.0,<5.0.0)
+Requires-Dist: pikepdf (>=9.0.0,<10.0.0)
+Requires-Dist: pillow (>=10.3.0,<11.0.0)
+Requires-Dist: pingouin (>=0.5.4,<0.6.0)
+Requires-Dist: platformdirs (>=4.2.2,<5.0.0)
+Requires-Dist: pooch (>=1.8.2,<2.0.0)
+Requires-Dist: prompt_toolkit (>=3.0.47,<4.0.0)
+Requires-Dist: protobuf (>=5.27.1,<6.0.0)
+Requires-Dist: psutil (>=5.9.8,<6.0.0)
+Requires-Dist: ptyprocess (>=0.7.0,<0.8.0)
+Requires-Dist: pure-eval (>=0.2.2,<0.3.0)
+Requires-Dist: pyparsing (>=3.1.2,<4.0.0)
+Requires-Dist: python-box (>=7.2.0,<8.0.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
+Requires-Dist: python-docx (>=1.1.0,<2.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: pytz (>=2024.1,<2025.0)
+Requires-Dist: pyzmq (>=26.0.3,<27.0.0)
+Requires-Dist: referencing (>=0.35.1,<0.36.0)
+Requires-Dist: regex (>=2024.5.15,<2025.0.0)
+Requires-Dist: rembg (>=2.0.56,<3.0.0) ; python_version >= "3.9" and python_version < "3.13"
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: rfc3986 (>=1.5.0,<2.0.0)
+Requires-Dist: rpds-py (>=0.18.1,<0.19.0)
+Requires-Dist: scikit-image (>=0.23.2,<0.24.0) ; python_version >= "3.10" and python_version < "4.0"
+Requires-Dist: scikit-learn (>=1.3.2,<2.0.0)
+Requires-Dist: scipy (>=1.13.1,<2.0.0)
+Requires-Dist: seaborn (>=0.13.2,<0.14.0)
+Requires-Dist: selenium (>=4.21.0,<5.0.0)
+Requires-Dist: six (>=1.16.0,<2.0.0)
+Requires-Dist: sniffio (>=1.3.1,<2.0.0)
+Requires-Dist: sortedcontainers (>=2.4.0,<3.0.0)
+Requires-Dist: soupsieve (>=2.5,<3.0)
+Requires-Dist: stack-data (>=0.6.3,<0.7.0)
+Requires-Dist: statsmodels (>=0.14.1,<0.15.0)
+Requires-Dist: stem (>=1.8.2,<2.0.0)
+Requires-Dist: sympy (>=1.12.1,<2.0.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: threadpoolctl (>=3.5.0,<4.0.0)
+Requires-Dist: tifffile (>=2024.5.22,<2025.0.0)
+Requires-Dist: tornado (>=6.4.1,<7.0.0)
+Requires-Dist: tqdm (>=4.66.4,<5.0.0)
+Requires-Dist: traitlets (>=5.14.3,<6.0.0)
+Requires-Dist: translate (>=3.6.1,<4.0.0)
+Requires-Dist: trio (>=0.25.1,<0.26.0)
+Requires-Dist: trio-websocket (>=0.11.1,<0.12.0)
+Requires-Dist: typing_extensions (>=4.12.2,<5.0.0)
+Requires-Dist: tzdata (>=2024.1,<2025.0)
+Requires-Dist: urllib3 (>=2.2.1,<3.0.0)
+Requires-Dist: wcwidth (>=0.2.13,<0.3.0)
+Requires-Dist: webdriver-manager (>=4.0.1,<5.0.0)
+Requires-Dist: wrapt (>=1.16.0,<2.0.0)
+Requires-Dist: wsproto (>=1.2.0,<2.0.0)
+Requires-Dist: xarray (>=2024.6.0,<2025.0.0)
 Description-Content-Type: text/markdown
 
 # Install

{py2ls-0.1.4.4.dist-info → py2ls-0.1.4.5.dist-info}/RECORD

@@ -1,5 +1,6 @@
-py2ls/.git/COMMIT_EDITMSG,sha256=E6ggbgaSjmHbt33GuQe6KX5ipK0Dp29QiuxxpBkqm60,131
-py2ls/.git/FETCH_HEAD,sha256=Cdzu13JkjE_s_0YvmkAIac4l2WuokfA-YimhsDkYo1Q,100
+py2ls/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
+py2ls/.git/COMMIT_EDITMSG,sha256=411ztgfI0SRPtVH7bl1HcadUmgTDrnUC7vUpClHTuhg,22
+py2ls/.git/FETCH_HEAD,sha256=JftrmaQ9tRhDaUKGGYHLcduOOWrCUcIXNgXZnFtj7MI,100
 py2ls/.git/HEAD,sha256=KNJb-Cr0wOK3L1CVmyvrhZ4-YLljCl6MYD2tTdsrboA,21
 py2ls/.git/config,sha256=XswTg1Ts7_7IBDlKHh4OF_0Tq7v4wW7BXb6xSVInSec,345
 py2ls/.git/description,sha256=ZzMxc0Ca26m45Twn1DDnOHqin5VHEZ9uOTBrScIXSjE,16
@@ -16,15 +17,16 @@ py2ls/.git/hooks/pre-receive.sample,sha256=pMPSuce7P9jRRBwxvU7nGlldZrRPz0ndsxAlI
 py2ls/.git/hooks/prepare-commit-msg.sample,sha256=6d3KpBif3dJe2X_Ix4nsp7bKFjkLI5KuMnbwyOGqRhk,1492
 py2ls/.git/hooks/push-to-checkout.sample,sha256=pT0HQXmLKHxt16-mSu5HPzBeZdP0lGO7nXQI7DsSv18,2783
 py2ls/.git/hooks/update.sample,sha256=jV8vqD4QPPCLV-qmdSHfkZT0XL28s32lKtWGCXoU0QY,3650
-py2ls/.git/index,sha256=Bxygoi0rCfEqm0Lnpaba5ZsAFg_NS8TYUbR2o4uwnnI,1338
+py2ls/.git/index,sha256=ELDmOGBjKw_yg4T29FlGfjGCtOPyp-0HeobCvVn2FEk,1242
 py2ls/.git/info/exclude,sha256=ZnH-g7egfIky7okWTR8nk7IxgFjri5jcXAbuClo7DsE,240
-py2ls/.git/logs/HEAD,sha256=Wiu2pJYW-hD-oZ452dEJyYjYf9wfjUjvJONH7wAi8Ug,1638
-py2ls/.git/logs/refs/heads/main,sha256=Wiu2pJYW-hD-oZ452dEJyYjYf9wfjUjvJONH7wAi8Ug,1638
-py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=032ZeLPOt_lRcwk8gTcVOzQd81zTAkkTk9P1PtPr6Aw,4212
-py2ls/.git/logs/refs/remotes/origin/main,sha256=YU2rplaPYPyRcfSuRQ65bFZ8oQwKogJrBlSXYhD9Pa4,1520
+py2ls/.git/logs/HEAD,sha256=atxmhKxlt0NtNNva_ke0lzelzyiplpA7eq8VWbYCppU,2283
+py2ls/.git/logs/refs/heads/main,sha256=atxmhKxlt0NtNNva_ke0lzelzyiplpA7eq8VWbYCppU,2283
+py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=PdHzZi_OInxBkrCeCoKkc_wYjhAi8CYJcHdb1ZBskbw,5931
+py2ls/.git/logs/refs/remotes/origin/main,sha256=BwjVsMFWPHygHFr2LkjMCNoxelgIAOBSd0SgxIEnOy4,1976
 py2ls/.git/objects/0b/409e1bc918277010f5679b402d1d1dda53e15c,sha256=y5S1XaGxJz1NXi-SPWjPC_NKIqqSbZv9oOg74MzBihY,156
 py2ls/.git/objects/14/449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed,sha256=PomZFmCUCQM1ii0wH-OJGSHLQCTqRtIwE5w3C0TtzSY,171
 py2ls/.git/objects/15/a8e468aacfcb440e090020f36d0b985d45da23,sha256=xiRunMcN5I_B2hHgBUFupR-F0b8H_CQTmmAZG9XkZik,3215
+py2ls/.git/objects/1a/b4585881a6a42889f01aa0cfe25fd5acfaf46f,sha256=iQsKMPNKUs4WQwhiLgXmG5V3xKyIgxmc13ZwbBATvhQ,165
 py2ls/.git/objects/1d/fe9d9633b24ea560354f4f93d39c6e5f163ea0,sha256=mV_84wLqIitnSYmzfrNpTzwVP9AmksiRI0Fjltwl0Pg,8872
 py2ls/.git/objects/24/6b368b986f758630c46dc02b7fa512b53422f7,sha256=sw7ERFCFu7m6fnURAqQfQ4GWShaARr-Vc6GRnlOPkxU,8512
 py2ls/.git/objects/25/b796accd261b9135fd32a2c00785f68edf6c46,sha256=4ic5vOwEdfbGL8oARSVEeAnSoDs14-gggGZEL-61nYE,564
@@ -37,16 +39,25 @@ py2ls/.git/objects/3c/bbe5f4173d165127b9ad96119f1ec24c306ffc,sha256=S1BXemROYtzR
 py2ls/.git/objects/3f/d6561300938afbb3d11976cf9c8f29549280d9,sha256=91oqbTWfUE1d_hT_1ptYmRUb5pOQ1X4oxQxpF6NXjKU,8501
 py2ls/.git/objects/43/dbd49b2ee367c5434dd545e3b5795434f2ef0b,sha256=DAzt0dWp2KsuuImCKp7N9ia7KaCDNqwB-tYIx3Wf_c0,565
 py2ls/.git/objects/48/a88fc5806305d0bb0755ee6801161b79696972,sha256=f3JStE39k_hPGE-WRwqZtDTjQkfOmBVb_6-ELBbScjI,203
+py2ls/.git/objects/50/08ddfcf53c02e82d7eee2e57c38e5672ef89f6,sha256=p0M2WLqiTe6X2FI_k5Aj0IEsE85jqLa58sVdmV8x1vU,255
+py2ls/.git/objects/53/e0deb1cb4c2c606bced6e7f9a66b0fda60980d,sha256=muq6m7_XRSFPzypW-m9mhpKfsomCr4s7GfkgM3gh2pc,482344
+py2ls/.git/objects/56/e4e8b2d5545e0256090f45aa8fc42c5fe067d0,sha256=VsjKo1biAzCV-iIfwCDTPzyfP63K43hdZqJpDP70Iik,529
 py2ls/.git/objects/58/20a729045d4dc7e37ccaf8aa8eec126850afe2,sha256=3Pf6NS8OTK4EdHZGVeJ421BtK7w4WJncQDBauZI_wW4,34
 py2ls/.git/objects/60/f273eb1c412d916fa3f11318a7da7a9911b52a,sha256=aJD9iF_LmYSrqDepXFBZKN1yMYbQczVkN_wnrDosBdI,5620
 py2ls/.git/objects/61/570cec8c061abe74121f27f5face6c69b98f99,sha256=IQZi5MkbRu3ToRUPsRcXuh1Xa3pkAz_HDRCVhNL89ds,5753
+py2ls/.git/objects/62/7c81b23b4e56e87b042b650b0103653cc9e34a,sha256=pv9wgBxnvJUFSrk9G7vApA6lnSykQSMJ4yXT7YnlSDU,167
+py2ls/.git/objects/62/d90ccf8cbefdc2e4fd475e7c6f4f76e9fdf801,sha256=1L473QanNpnumCkE8tG6wtbvLqFtNeoagL9SJmasXNY,155
 py2ls/.git/objects/64/27a4edff08f93d98f511418423f09f2ab90bcd,sha256=RyNngwk9fvdvvvywmNfllnim718fWNjVauH9U2y8Q2s,258
 py2ls/.git/objects/69/13c452ca319f7cbf6a0836dc10a5bb033c84e4,sha256=NYLQQZTfd0htZst42ALS2dmryv1q_l1N19ZfHEbz_38,3193
 py2ls/.git/objects/6b/7fde264d93a7a0986d394c46c7650d0ce2ab92,sha256=iIl0-RF0wd6BSEjzczgUyApxc899PbdTl04JbDn6_-Q,166
+py2ls/.git/objects/6c/cebb29b7f3f5b0c889f6dadbf9ff066554587d,sha256=UylkFWAfhStNVQRQuC9CzpaWaT9uHCVs1mn7ecOma8I,609
 py2ls/.git/objects/78/3d4167bc95c9d2175e0df03ef1c1c880ba75ab,sha256=SK2QDjDBiDhVMG1I5p19g4RbEm2Rax7mYnxawmVZYxs,15523
 py2ls/.git/objects/79/7ae089b2212a937840e215276005ce76881307,sha256=lQOKF2pb1JvipI3eT79X0-TuMGWsy1A-Yw4BCgKZNOM,33472
 py2ls/.git/objects/7e/5956c806b5edc344d46dab599dec337891ba1f,sha256=sfqJBiSNj-gyJo4D7xkmRAo76mC2ztjqeZZsl4ifULA,162
+py2ls/.git/objects/81/8f26b7bf042269729020cf944fc362d66ba27e,sha256=mg6FGEyv6EcOgurR8CEvHGovaWrUgMUxTtACAy7-ei4,34960
+py2ls/.git/objects/84/59071b722a255b774a80b27746033f8141ab39,sha256=0pYGJOXFfp4MSu4n5MzE1XN--t0lSs7wcdqboADWMx0,9792
 py2ls/.git/objects/87/ef1fc3f7f1ddc4d0ab9b3e65381ce9f3388621,sha256=OFrpW6lu31qGBvD3ijPUBSG9JrdU1_mKzeYBzidn9VM,3748
+py2ls/.git/objects/8b/84f56978e1de8f2ae82abce5f8b3e182d365cd,sha256=a8XequnUMBSv9zIQJdcdgDvMQ7PLGdIrgZ-MqQGF87c,573
 py2ls/.git/objects/8e/55a7d2b96184030211f20c9b9af201eefcac82,sha256=yW-jVYeCTWR-nX3JJgA1g9YLPjzNsKlDmEOH290Ywx0,1221
 py2ls/.git/objects/91/c69ad88fe0ba94aa7859fb5f7edac5e6f1a3f7,sha256=Kk2MWCO1OcShYuABGzp2O9LiWGDfDkcZtd0oy4nY6RU,9529
 py2ls/.git/objects/9d/0df52899fe95279059286d9c0ec42287edc168,sha256=67nV3TLo-fwe4lt0wwvxoDnVNHc1IpapRyAY2STP3iI,564
@@ -55,38 +66,42 @@ py2ls/.git/objects/a7/3e13eafee65c5b8d73ad2d3ea46d0eee82f0d3,sha256=iv3uTzna5XBz
 py2ls/.git/objects/b0/56be4be89ba6b76949dd641df45bb7036050c8,sha256=8Y7z30eNceBd5QIx09QfMp5cYBbrgUllmats0kvJEJ4,132
 py2ls/.git/objects/b0/9cd7856d58590578ee1a4f3ad45d1310a97f87,sha256=82dx4hIdMpdcB64e5PU1s2gZFVkTvrj1cPwwJ_kasNU,4444
 py2ls/.git/objects/b2/18e6a0f0f1c4df8cdefa9852058348abc713b7,sha256=hOQfdyzDZctjoge0-pAcEDel5XHVPNfOtrMNyFPUOIE,564
+py2ls/.git/objects/b5/61831c7dce8ea51e7ee6b6fa35745f14d8242d,sha256=wUqxlKjLN1vOUj2tkYStado64QewdcF3CHlSICds1ik,34415
 py2ls/.git/objects/bb/934eb33bc1a8b85630bf680caffd99560c1b8f,sha256=ggehjexUsWlskHJvHxW7u6U0otB0OCItmIZdT9O-3OU,9670
 py2ls/.git/objects/c4/cba65f1163661999ee4b8ed23342b63bc1300c,sha256=rwSdKt-C98nUQ_B-7imY4fYRYmn29MQc4SIu9wruHeo,566
 py2ls/.git/objects/c6/7f17e5707313600efcb85e9a3fedea35dba591,sha256=TL7rDIWiaWlk8iIwqPst7St5Xr2otPs-vp17GPlET7o,565
 py2ls/.git/objects/cf/0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d,sha256=T_nV0GrgpVu3mOJ4fYcCW98oCunzgqy0DnSX0luy04Q,183
+py2ls/.git/objects/d6/9ab1c4aadf279936dd778e8346ba60f74705b6,sha256=WcfdSMKqfiWT5TOWVUcDj0XDaD2hYxDnyIRNlYGutL8,34976
 py2ls/.git/objects/d9/005f2cc7fc4e65f14ed5518276007c08cf2fd0,sha256=IJIoz93V7pf9yx43U1JdN8gBq_LWtw8A9Z2YMPnq_B0,1450
 py2ls/.git/objects/d9/c2403fd166ce791b4e9d0c6792ed8342c71fcd,sha256=uD7BsKdrmN-9FStTpwsRWh-XxVXeDsV4dGjFkaMIIs8,170
+py2ls/.git/objects/d9/dfa5aee51e92a541b707e8e7baea6f06deff98,sha256=jMdhZ1i_L5q_UgjOtjLN15PCSCz3pE51FhD3z74ZUr8,163
 py2ls/.git/objects/db/ffa8ea7bda721d0cee7b9e4ce5b2ef927733ff,sha256=GhDkvP6JYV26qVg5ETPys1ZEnGlsct9hiXCc24Ky4Xg,565
 py2ls/.git/objects/df/e0770424b2a19faf507a501ebfc23be8f54e7b,sha256=vCdlxwEidekh8i-5TVMVgSLGk9DPZCZAbWqvGYSKQ9c,76
+py2ls/.git/objects/e3/1356f90ea6dd0577b5e0b40b206319adcbf085,sha256=I9_QNwmmtoqSwq29Ixdfv_PgF2x14u2M6sX1eQumwoY,161
 py2ls/.git/objects/e3/5a4dafc50850cacac7bf76c56db2715cbda2c4,sha256=GAcBj3YSEbm6tm7fGD6al16uBo8LtEtjZ2Hi-UgIsUg,3290
 py2ls/.git/objects/e9/391ffe371f1cc43b42ef09b705d9c767c2e14f,sha256=RWTy2n8L2XxZQknBFyPczA0Aa_4gSG_Ybcr8e8v4ccc,10264
 py2ls/.git/objects/f4/b64d3107b39e3ad6f540c6607004ea34e6c024,sha256=0egAtqc0x8hc7U1z91tIjcRhSd_BT2a_gxZxo_7NTJA,564
 py2ls/.git/objects/f7/c98ba5c2f903e603b1f5e63d49fbc8a43815cc,sha256=tYbi3A7irrIPB_11bwItuof0Vc9a0MDuLFMNAzRsG3A,33467
+py2ls/.git/objects/fa/147e6bb78a2e8db241d231295fd7f1ed061af8,sha256=G9pg5LXv7AdxnPIQsTm2AF3Un314dLRJQYwxmZem9rQ,574
 py2ls/.git/objects/fc/292e793ecfd42240ac43be407023bd731fa9e7,sha256=hGIYoxKWNT3IPwk3DE4l3FLBbUYF-kXcHcx7KrH9uS0,1971
-py2ls/.git/refs/heads/main,sha256=XWF5fg25wz0SqFKCM1pFdyA5I_3pkrwsU8Ql6LH7X0E,41
+py2ls/.git/refs/heads/main,sha256=3RjrJjIDvFj75BVJA5LA1QcuJulTJWk6nGYinmb21q4,41
 py2ls/.git/refs/remotes/origin/HEAD,sha256=K7aiSqD8bEhBAPXVGim7rYQc0sdV9dk_qiBOXbtOsrQ,30
-py2ls/.git/refs/remotes/origin/main,sha256=XWF5fg25wz0SqFKCM1pFdyA5I_3pkrwsU8Ql6LH7X0E,41
+py2ls/.git/refs/remotes/origin/main,sha256=lZgord5ZoIag3eQnw5oqNUSn85Nh7qyeconduoPcMls,41
 py2ls/.gitattributes,sha256=Gh2-F2vCM7SZ01pX23UT8pQcmauXWfF3gwyRSb6ZAFs,66
 py2ls/.gitignore,sha256=y7GvbD_zZkjPVVIue8AyiuFkDMuUbvMaV65Lgu89To8,2763
 py2ls/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
 py2ls/README.md,sha256=CwvJWAnSXnCnrVHlnEbrxxi6MbjbE_MT6DH2D53S818,11572
-py2ls/__init__.py,sha256=k05sUEKwWnAaOAn4VKCiPopcWrbmLLsZOuAt7vmUKWc,260
+py2ls/__init__.py,sha256=ESXjQ9tnqg5mqYH4Gfgs76AoT1HHF_BkJUgnstiVwR8,243
 py2ls/brain_atlas.py,sha256=w1o5EelRjq89zuFJUNSz4Da8HnTCwAwDAZ4NU4a-bAY,5486
 py2ls/correlators.py,sha256=RbOaJIPLCHJtUm5SFi_4dCJ7VFUPWR0PErfK3K26ad4,18243
 py2ls/dbhandler.py,sha256=i9dNrpHyx0oIaFieHI4X4tsrCdN-aFxudPTDOgy9Ppo,3574
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
-py2ls/internet_finder.py,sha256=I-jPepbBhpDaOAsD2MqbKMe1CBN8w1PYo0CjNFkaeeU,19149
-py2ls/ips.py,sha256=MH_TRP_lz9AuWMLjPNgTTxNFHcvLLN2XKGQcG9txjP0,121806
-py2ls/netfinder.py,sha256=EbNH-QMFNYsu_eJ68zqEEJRJ5uAYOCrtkqfoC9ZJpo4,31207
+py2ls/ips.py,sha256=yf4UECX0bhzyoKqyRWBOGUrNrzu-BoO2f8G293eR2Do,124079
+py2ls/netfinder.py,sha256=dt6hkYeH-ivCHInoUi92MhJMLlXtjRXT3ewKzOwGtWk,31506
+py2ls/setuptools-70.1.0-py3-none-any.whl,sha256=2bi3cUVal8ip86s0SOvgspteEF8SKLukECi-EWmFomc,882588
 py2ls/sleep_events_detectors.py,sha256=36MCuRrpurn0Uvzpo3p3b3_JlVsRNHSWCXbJxCGM3mg,51546
 py2ls/translator.py,sha256=QfDUO0-pXHGMBFZBefiBHzOrC93-__N5sUQY_VP4wes,29734
-py2ls/version.py,sha256=CactNZqrHHYTPrkHKccy2WKXmaiUdtTgPqSjFyVXnJk,18
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.1.4.4.dist-info/METADATA,sha256=sar2UcFB1uR1NAUfvx0pxCP-VO_KyT8n_fS6Yu4K0r4,11877
-py2ls-0.1.4.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-py2ls-0.1.4.4.dist-info/RECORD,,
+py2ls-0.1.4.5.dist-info/METADATA,sha256=Lbz_REQylOCgSwziBFP6oEELfCEU2cK01PCM4kKo5MI,17943
+py2ls-0.1.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+py2ls-0.1.4.5.dist-info/RECORD,,

py2ls/internet_finder.py

@@ -1,405 +0,0 @@
-from bs4 import BeautifulSoup
-import requests
-import os
-from urllib.parse import urlparse, urljoin
-import base64
-import pandas as pd
-from collections import Counter
-import random
-import logging
-from time import sleep
-import stem.process
-from stem import Signal
-from stem.control import Controller
-import json
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Define supported content types and corresponding parsers
-CONTENT_PARSERS = {
-    "text/html": lambda text, parser: BeautifulSoup(text, parser),
-    "application/json": lambda text, parser: json.loads(text),
-    "text/xml": lambda text, parser: BeautifulSoup(text, parser),
-    "text/plain": lambda text, parser: text.text,
-}
-
-def fetch_all(url, parser="lxml"): # lxml is faster, # parser="html.parser"
-    try:
-        # Generate a random user-agent string
-        headers = {"User-Agent": user_agent()}
-
-        # Send the initial request
-        response = requests.get(url, headers=headers)
-
-        # If the response is a redirect, follow it
-        while response.is_redirect:
-            logger.info(f"Redirecting to: {response.headers['Location']}")
-            response = requests.get(response.headers["Location"], headers=headers)
-        # Check for a 403 error
-        if response.status_code == 403:
-            logger.warning("403 Forbidden error. Retrying...")
-            # Retry the request after a short delay
-            sleep(random.uniform(1, 3))
-            response = requests.get(url, headers=headers)
-            # Raise an error if retry also fails
-            response.raise_for_status()
-
-        # Raise an error for other HTTP status codes
-        response.raise_for_status()
-
-        # Get the content type
-        content_type = response.headers.get("content-type", "").split(";")[0].lower()
-        content = response.content.decode(response.encoding)
-        # logger.info(f"Content type: {content_type}")
-
-        # Check if content type is supported
-        if content_type in CONTENT_PARSERS:
-            return content_type, CONTENT_PARSERS[content_type](content, parser)
-        else:
-            logger.warning("Unsupported content type")
-            return None, None
-    except requests.RequestException as e:
-        logger.error(f"Error fetching URL '{url}': {e}")
-        return None, None
-def user_agent():
-    # Example of generating a random user-agent string
-    user_agents = [
-        # Windows (Intel)
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Safari/537.36",
-        # Windows (ARM)
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Safari/537.36",
-        # Linux (x86_64)
-        "Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
-        "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0",
-        "Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4895.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
-        # macOS (Intel)
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15",
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",
-        # macOS (ARM)
-        "Mozilla/5.0 (Macintosh; ARM Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15",
-        "Mozilla/5.0 (Macintosh; ARM Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",
-        # iOS Devices
-        "Mozilla/5.0 (iPad; CPU OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
-        "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
-        # Android Devices
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Mobile Safari/537.36",
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Mobile Safari/537.36",
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Mobile Safari/537.36",
-        # Smart TVs
-        "Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) SmartTV/1.0",
-        "Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) WebAppManager/1.0",
-        # Game Consoles
-        "Mozilla/5.0 (PlayStation 5 3.01) AppleWebKit/605.1.15 (KHTML, like Gecko)",
-        "Mozilla/5.0 (Xbox One 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edge/44.18363.8740",
-    ]
-    agents = random.choice(user_agents)
-    return agents
-
-# # Function to change Tor IP address
-# def renew_tor_ip():
-#     with Controller.from_port(port=9051) as controller:
-#         controller.authenticate()
-#         controller.signal(Signal.NEWNYM)
-
-# # Function to make requests through Tor
-# def make_tor_request(url, max_retries=3):
-#     renew_tor_ip()
-#     headers = {"User-Agent": user_agent()}
-#     session = requests.Session()
-#     session.proxies = {"http": "socks5h://localhost:9050", "https": "socks5h://localhost:9050"}
-
-#     for i in range(max_retries):
-#         try:
-#             response = session.get(url, headers=headers, timeout=10)
-#             if response.status_code == 200:
-#                 return response.text
-#         except requests.exceptions.RequestException as e:
-#             print(f"Error: {e}")
-#         time.sleep(2)  # Add a delay between retries
-
-#     return None
-
-
-def find_links(url):
-    links_href = []  # Initialize list to store extracted links
-    content_type, content = fetch_all(url)
-    base_url = urlparse(url)
-    links = content.find_all("a", href=True)
-    for link in links:
-        link_href = link["href"]
-        if not link_href.startswith(('http://', 'https://')):
-            # Convert relative links to absolute links
-            link_href = urljoin(base_url.geturl(), link_href)
-        links_href.append(link_href)
-    return links_href
-
-def find_domain(links):
-    domains = [urlparse(link).netloc for link in links]
-    domain_counts = Counter(domains)
-    most_common_domain = domain_counts.most_common(1)[0][0]
-    # print(f"Most_frequent_domain:{most_common_domain}")
-    return most_common_domain
-
-# To determine which links are related to target domains(e.g., pages) you are interested in
-def filter_links(links, domain=None, kind='html'):
-    filtered_links = []
-    if isinstance(kind, (str, list)):
-        kind = tuple(kind)
-    if domain is None:
-        domain = find_domain(links)
-    for link in links:
-        parsed_link = urlparse(link)
-        if parsed_link.netloc == domain and parsed_link.path.endswith(kind) and 'javascript:' not in parsed_link:
-            filtered_links.append(link)
-    return filtered_links
-
-def find_img(url, dir_save="images"):
-    """
-    Save images referenced in HTML content locally.
-    Args:
-        content (str or BeautifulSoup): HTML content or BeautifulSoup object.
-        url (str): URL of the webpage.
-        content_type (str): Type of content. Default is "html".
-        dir_save (str): Directory to save images. Default is "images".
-    Returns:
-        str: HTML content with updated image URLs pointing to local files.
-    """
-    content_type, content = fetch_all(url)
-    if "html" in content_type.lower():
-        # Create the directory if it doesn't exist
-        os.makedirs(dir_save, exist_ok=True)
-
-        # Parse HTML content if it's not already a BeautifulSoup object
-        if isinstance(content, str):
-            content = BeautifulSoup(content, "html.parser")
-        image_links=[]
-        # Extracting images
-        images = content.find_all("img", src=True)
-        for i, image in enumerate(images):
-            try:
-                # Get the image URL
-                image_url = image["src"]
-
-                if image_url.startswith("data:image"):
-                    # Extract the image data from the data URI
-                    mime_type, base64_data = image_url.split(",", 1)
-                    # Determine the file extension from the MIME type
-                    if ":" in mime_type:
-                        # image_extension = mime_type.split(":")[1].split(";")[0]
-                        image_extension = mime_type.split(":")[1].split(";")[0].split("/")[-1]
-                    else:
-                        image_extension = "png"  # Default to PNG if extension is not specified
-                    # if 'svg+xml' in image_extension:
-                    #     image_extension='svg'
-                    image_data = base64.b64decode(base64_data)
-                    # Save the image data to a file
-                    image_filename = os.path.join(
-                        dir_save, f"image_{i}.{image_extension}"
-                    )
-                    with open(image_filename, "wb") as image_file:
-                        image_file.write(image_data)
-
-                    # Update the src attribute of the image tag to point to the local file
-                    image["src"] = image_filename
-                else:
-                    # Construct the absolute image URL
-                    absolute_image_url = urljoin(url, image_url)
-
-                    # Parse the image URL to extract the file extension
-                    parsed_url = urlparse(absolute_image_url)
-                    image_extension = os.path.splitext(parsed_url.path)[1]
-
-                    # Download the image
-                    image_response = requests.get(absolute_image_url)
-
-                    # Save the image to a file
-                    image_filename = os.path.join(
-                        dir_save, f"image_{i}{image_extension}"
-                    )
-                    with open(image_filename, "wb") as image_file:
-                        image_file.write(image_response.content)
-
-                    # Update the src attribute of the image tag to point to the local file
-                    image["src"] = image_filename
-            except (requests.RequestException, KeyError) as e:
-                print(f"Failed to process image {image_url}: {e}")
-        print(f"images were saved at\n{dir_save}")
-    # Return the HTML content with updated image URLs
-    return content
-
-def content_div_class(content, div="div", div_class="highlight"):
-    texts = [div.text for div in content.find_all(div, class_=div_class)]
-    return texts
-def find(url, where="div", what="highlight"):
-    _,content = fetch_all(url, parser="html.parser")
-    texts = [div.text for div in content.find_all(where, class_=what)]
-    return texts
-# usage example:
-#### img2local(url, "/Users/macjianfeng/Desktop/@tmp/dd/")
-def find_forms(url):
-    content_type, content = fetch_all(url)
-    df=pd.DataFrame()
-    # Extracting forms and inputs
-    forms = content.find_all("form")
-    form_data = []
-    for form in forms:
-        form_inputs = form.find_all("input")
-        input_data = {}
-        for input_tag in form_inputs:
-            input_type = input_tag.get("type")
-            input_name = input_tag.get("name")
-            input_value = input_tag.get("value")
-            input_data[input_name] = {"type": input_type, "value": input_value}
-        form_data.append(input_data)
-    return form_data
-#  to clean strings
-def clean_string(value):
-    if isinstance(value, str):
-        return value.replace('\n', '').replace('\r', '').replace('\t', '')
-    else:
-        return value
-def find_all(url, dir_save=None):
-    content_type, content = fetch_all(url)
-
-    # Extracting paragraphs
-    paragraphs_text = [paragraph.text for paragraph in content.find_all("p")]
-
-    # Extracting specific elements by class
-    specific_elements_text = [element.text for element in content.find_all(class_="specific-class")]
-
-    # Extracting links (anchor tags)
-    links_href = find_links(url)
-    links_href = filter_links(links_href)
-
-    # Extracting images
-    images_src = [image['src'] for image in content.find_all("img", src=True)]
-
-    # Extracting headings (h1, h2, h3, etc.)
-    headings = [f'h{i}' for i in range(1, 7)]
-    headings_text = {heading: [tag.text for tag in content.find_all(heading)] for heading in headings}
-
-    # Extracting lists (ul, ol, li)
-    list_items_text = [item.text for list_ in content.find_all(["ul", "ol"]) for item in list_.find_all("li")]
-
-    # Extracting tables (table, tr, td)
-    table_cells_text = [cell.text for table in content.find_all("table") for row in table.find_all("tr") for cell in row.find_all("td")]
-
-    # Extracting other elements
-    divs_content = [div.text.strip() for div in content.find_all("div")]
-    headers_footer_content = [tag.text for tag in content.find_all(["header", "footer"])]
-    meta_tags_content = [(tag.name, tag.attrs) for tag in content.find_all("meta")]
-    spans_content = [span.text for span in content.find_all("span")]
-    bold_text_content = [text.text for text in content.find_all("b")]
-    italic_text_content = [text.text for text in content.find_all("i")]
-    code_snippets_content = [code.text for code in content.find_all("code")]
-    blockquotes_content = [blockquote.text for blockquote in content.find_all("blockquote")]
-    preformatted_text_content = [pre.text for pre in content.find_all("pre")]
-    buttons_content = [button.text for button in content.find_all("button")]
-    navs_content = [nav.text for nav in content.find_all("nav")]
-    sections_content = [section.text for section in content.find_all("section")]
-    articles_content = [article.text for article in content.find_all("article")]
-    figures_content = [figure.text for figure in content.find_all("figure")]
-    captions_content = [caption.text for caption in content.find_all("figcaption")]
-    abbreviations_content = [abbr.text for abbr in content.find_all("abbr")]
-    definitions_content = [definition.text for definition in content.find_all("dfn")]
-    addresses_content = [address.text for address in content.find_all("address")]
-    time_elements_content = [time.text for time in content.find_all("time")]
-    progress_content = [progress.text for progress in content.find_all("progress")]
-    meter_content = [meter.text for meter in content.find_all("meter")]
-    forms = find_forms(url)
-
-    lists_to_fill = [
-        paragraphs_text, specific_elements_text, links_href, images_src,
-        headings_text["h1"], headings_text["h2"], headings_text["h3"], headings_text["h4"],
-        headings_text["h5"], headings_text["h6"], list_items_text, table_cells_text,
-        divs_content, headers_footer_content, meta_tags_content, spans_content,
-        bold_text_content, italic_text_content, code_snippets_content,
-        blockquotes_content, preformatted_text_content, buttons_content,
-        navs_content, sections_content, articles_content, figures_content,
-        captions_content, abbreviations_content, definitions_content,
-        addresses_content, time_elements_content, progress_content,
-        meter_content,forms
-    ]
-    # add new features
-    script_texts=content_div_class(content, div="div", div_class="highlight")
-    lists_to_fill.append(script_texts)
-    
-    audio_src = [audio['src'] for audio in content.find_all("audio", src=True)]
-    video_src = [video['src'] for video in content.find_all("video", src=True)]
-    iframe_src = [iframe['src'] for iframe in content.find_all("iframe", src=True)]
-    lists_to_fill.extend([audio_src, video_src, iframe_src])
-    
-    rss_links = [link['href'] for link in content.find_all('link', type=['application/rss+xml', 'application/atom+xml'])]
-    lists_to_fill.append(rss_links)
-
-    # Find the maximum length among all lists
-    max_length = max(len(lst) for lst in lists_to_fill)
-
-    # Fill missing data with empty strings for each list
-    for lst in lists_to_fill:
-        lst += [""] * (max_length - len(lst))
-
-    # Create DataFrame
-    df = pd.DataFrame({
-        "headings1": headings_text["h1"],
-        "headings2": headings_text["h2"],
-        "headings3": headings_text["h3"],
-        "headings4": headings_text["h4"],
-        "headings5": headings_text["h5"],
-        "headings6": headings_text["h6"],
-        "paragraphs": paragraphs_text,
-        "list_items": list_items_text,
-        "table_cells": table_cells_text,
-        "headers_footer": headers_footer_content,
-        "meta_tags": meta_tags_content,
-        "spans": spans_content,
-        "bold_text": bold_text_content,
-        "italic_text": italic_text_content,
-        "code_snippets": code_snippets_content,
-        "blockquotes": blockquotes_content,
-        "preformatted_text": preformatted_text_content,
-        "buttons": buttons_content,
-        "navs": navs_content,
-        "sections": sections_content,
-        "articles": articles_content,
-        "figures": figures_content,
-        "captions": captions_content,
-        "abbreviations": abbreviations_content,
-        "definitions": definitions_content,
-        "addresses": addresses_content,
-        "time_elements": time_elements_content,
-        "progress": progress_content,
-        "specific_elements": specific_elements_text,
-        "meter": meter_content,
-        "forms":forms,
-        "scripts":script_texts,
-        "audio":audio_src,
-        "video":video_src,
-        "iframe":iframe_src,
-        "rss": rss_links,
-        "images": images_src,
-        "links": links_href,
-        "divs": divs_content,
-    })
-    # to remove the '\n\t\r'
-    df=df.apply(lambda x: x.map(clean_string) if x.dtype == "object" else x) # df=df.applymap(clean_string)
-    if dir_save:
-        if not dir_save.endswith(".csv"):
-            dir_save=dir_save+"_df.csv"
-            df.to_csv(dir_save)
-        else:
-            df.to_csv(dir_save)
-        print(f"file has been saved at\n{dir_save}")
-    return df

py2ls/version.py

@@ -1 +0,0 @@
-version = "0.0.1"