py2ls 0.1.4.2__py3-none-any.whl → 0.1.4.4__py3-none-any.whl

py2ls/.git/COMMIT_EDITMSG

@@ -1,3 +1,4 @@
-Update netfinder.py
+listdir type
 
-new features: iframe, disable_image, capabilities
+listdir type 大类别和精确类别,
+e.g.,  kind='img', means filtter all kinds of images, .tiff, tif, jpg, png...

py2ls/.git/FETCH_HEAD

@@ -1 +1 @@
-48a88fc5806305d0bb0755ee6801161b79696972		branch 'main' of https://github.com/Jianfengliu0413/py2ls
+6427a4edff08f93d98f511418423f09f2ab90bcd		branch 'main' of https://github.com/Jianfengliu0413/py2ls

py2ls/.git/logs/HEAD

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type

py2ls/.git/logs/refs/heads/main

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type

py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -9,3 +9,18 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718393737 +0200	remote set-head
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718491090 +0200	remote set-head
 48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718526960 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718529062 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718531693 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718533521 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718535485 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718537443 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718539464 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718541421 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718543586 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718545544 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718547501 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718549457 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718551415 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553371 +0200	remote set-head
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553465 +0200	remote set-head
+6427a4edff08f93d98f511418423f09f2ab90bcd 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718555183 +0200	remote set-head

py2ls/.git/logs/refs/remotes/origin/main

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393736 +0200	update by push
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491090 +0200	update by push
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526959 +0200	update by push
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553464 +0200	update by push
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555183 +0200	update by push

py2ls/.git/objects/cf/0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d

@@ -0,0 +1 @@
+x
�NIj1�Y��{����4&�C�`�Qk,�4f�!������K-PU�Zk�,��F�����n&�MJd�u$7=eV�1�{ظu0>x�g�h5R��1[�RZ�d�a��u��Z���ʎF��RC��~�t�ic|�B�7;�] �B����O���~�~�klc���2�!^Q�

py2ls/.git/refs/heads/main

@@ -1 +1 @@
-48a88fc5806305d0bb0755ee6801161b79696972
+6427a4edff08f93d98f511418423f09f2ab90bcd

py2ls/.git/refs/remotes/origin/main

@@ -1 +1 @@
-48a88fc5806305d0bb0755ee6801161b79696972
+6427a4edff08f93d98f511418423f09f2ab90bcd

py2ls/ips.py

@@ -36,6 +36,7 @@ import time
 from box import Box, BoxList
 from numerizer import numerize
 from tqdm import tqdm
+import mimetypes
 
 def str2num(s, *args):
     delimiter = None
@@ -778,14 +779,15 @@ def dir_name(fpath):
     return dirname(fpath)
 def basename(fpath):
     return os.path.basename(fpath)
+
 def listdir(
     rootdir,
     kind="folder",
     sort_by="name",
     ascending=True,
     contains=None,
-    orient = "list",
-    output='df'
+    orient="list",
+    output="df"
 ):
     def sort_kind(df, by="name", ascending=True):
         if df[by].dtype == 'object':  # Check if the column contains string values
@@ -801,6 +803,69 @@ def listdir(
         sorted_df = df.iloc[sorted_index].reset_index(drop=True)
         return sorted_df
 
+    def flist(fpath, filter="all"):
+        all_files = [os.path.join(fpath, f) for f in os.listdir(fpath) if os.path.isfile(os.path.join(fpath, f))]
+        if isinstance(filter, list):
+            filt_files = []
+            for filter_ in filter:
+                filt_files.extend(flist(fpath, filter_))
+            return filt_files
+        else:
+            if 'all' in filter.lower():
+                return all_files
+            else:
+                filt_files = [f for f in all_files if istype(f, filter)]
+                return filt_files
+
+    def istype(fpath, filter='img'):
+        """
+        Filters file paths based on the specified filter.
+        Args:
+            fpath (str): Path to the file.
+            filter (str): Filter of file to filter. Default is 'img' for images. Other options include 'doc' for documents,
+                        'zip' for ZIP archives, and 'other' for other types of files.
+        Returns:
+            bool: True if the file matches the filter, False otherwise.
+        """
+        if 'img' in filter.lower():
+            return is_image(fpath)
+        elif 'doc' in filter.lower():
+            return is_document(fpath)
+        elif 'zip' in filter.lower():
+            return is_zip(fpath)
+        else:
+            return False
+
+    def is_image(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type and mime_type.startswith('image'):
+            return True
+        else:
+            return False
+
+    def is_document(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type and (
+            mime_type.startswith('text/') or
+            mime_type == 'application/pdf' or
+            mime_type == 'application/msword' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' or
+            mime_type == 'application/vnd.ms-excel' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' or
+            mime_type == 'application/vnd.ms-powerpoint' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+        ):
+            return True
+        else:
+            return False
+
+    def is_zip(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type == 'application/zip':
+            return True
+        else:
+            return False
+
     if not kind.startswith("."):
         kind = "." + kind
 
@@ -814,7 +879,7 @@ def listdir(
             "path": [],
             "created_time": [],
             "modified_time": [],
-            "last_open_time":[],
+            "last_open_time": [],
             "size": [],
             "fname": [],
             "fpath": [],
@@ -828,16 +893,20 @@ def listdir(
             is_file = kind.lower() in file_extension.lower() and (
                 os.path.isfile(item_path)
             )
-            if not is_folder and not is_file:
-                continue
+            if kind in ['.doc','.img','.zip']: #选择大的类别
+                if kind != ".folder" and not istype(item_path, kind):
+                    continue
+            else: #精确到文件的后缀
+                if not is_folder and not is_file:
+                    continue
             f["name"].append(filename)
             f["length"].append(len(filename))
             f["path"].append(os.path.join(os.path.dirname(item_path), item))
-            fpath=os.path.join(os.path.dirname(item_path), item)
+            fpath = os.path.join(os.path.dirname(item_path), item)
             f["size"].append(round(os.path.getsize(fpath) / 1024 / 1024, 3))
-            f["created_time"].append(pd.to_datetime(os.path.getctime(item_path),unit='s'))
-            f["modified_time"].append(pd.to_datetime(os.path.getmtime(item_path),unit='s'))
-            f['last_open_time'].append(pd.to_datetime(os.path.getatime(item_path),unit='s'))
+            f["created_time"].append(pd.to_datetime(os.path.getctime(item_path), unit='s'))
+            f["modified_time"].append(pd.to_datetime(os.path.getmtime(item_path), unit='s'))
+            f['last_open_time'].append(pd.to_datetime(os.path.getatime(item_path), unit='s'))
             f["fname"].append(filename)  # will be removed
             f["fpath"].append(fpath)  # will be removed
             i += 1
@@ -850,32 +919,35 @@ def listdir(
             'The directory "{}" does NOT exist. Please check the directory "rootdir".'.format(
                 rootdir
             )
-        ) 
+        )
+
     f = pd.DataFrame(f)
+
     if contains is not None:
-        f = f[f["name"].str.contains(contains,case=False)]
+        f = f[f["name"].str.contains(contains, case=False)]
+
     if "nam" in sort_by.lower():
-        # f.sort_values(by="name", ascending=ascending, ignore_index=True, inplace=True)
-        f=sort_kind(f, by="name", ascending=ascending)
+        f = sort_kind(f, by="name", ascending=ascending)
     elif "crea" in sort_by.lower():
-        f=sort_kind(f, by="created_time", ascending=ascending)
+        f = sort_kind(f, by="created_time", ascending=ascending)
     elif "modi" in sort_by.lower():
-        f=sort_kind(f, by="modified_time", ascending=ascending)
+        f = sort_kind(f, by="modified_time", ascending=ascending)
     elif "s" in sort_by.lower() and "z" in sort_by.lower():
-        f=sort_kind(f, by="size", ascending=ascending)
+        f = sort_kind(f, by="size", ascending=ascending)
+
     if 'df' in output:
         return f
     else:
-        if 'l' in orient.lower(): # list # default
+        if 'l' in orient.lower():  # list # default
             res_output = Box(f.to_dict(orient="list"))
             return res_output
-        if 'd' in orient.lower(): # dict
+        if 'd' in orient.lower():  # dict
             return Box(f.to_dict(orient="dict"))
-        if 'r' in orient.lower(): # records
+        if 'r' in orient.lower():  # records
             return Box(f.to_dict(orient="records"))
-        if 'in' in orient.lower(): # records
+        if 'in' in orient.lower():  # records
             return Box(f.to_dict(orient="index"))
-        if 'se' in orient.lower(): # records
+        if 'se' in orient.lower():  # records
             return Box(f.to_dict(orient="series"))
 
 # Example usage:

py2ls/netfinder.py

@@ -20,7 +20,9 @@ from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 import pprint
+import mimetypes
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -42,61 +44,22 @@ def user_agent(browsers=["chrome", "edge", "firefox", "safari"], platforms=["pc"
     output_ua = ua.random
     if verbose:
         print(output_ua)
-    return output_ua
-# def extract_text_from_content(content,where,what,extend=False):
-#     if extend:
-#         texts = ""
-
-#         def extract_text(element):
-#             nonlocal texts
-#             if isinstance(element, str) and element.strip():
-#                 texts += element.strip()
-#             elif hasattr(element, "children"):
-#                 for child in element.children:
-#                     extract_text(child)
-
-#         result_set = (
-#             content.find_all(where, class_=what)
-#             if what
-#             else content.find_all(where)
-#         )
-#         for tag in result_set:
-#             extract_text(tag)
-
-#         text = [tx for tx in texts.split("\n") if tx]
-#         return text
-#     else:
-#         result_set = (
-#             content.find_all(where, class_=what)
-#             if what
-#             else content.find_all(where)
-#         )
-#         texts_ = " ".join(tag.get_text() + "\n" for tag in result_set)
-#         texts = [tx.strip() for tx in texts_.split("\n") if tx]
-#         return texts
-# def extract_text_from_content(content, where, what=None, extend=True):
-#     if extend:
-#         def extract_text(element):
-#             texts = ""
-#             if isinstance(element, str) and element.strip():
-#                 texts += element.strip()
-#             elif hasattr(element, "children"):
-#                 for child in element.children:
-#                     texts += extract_text(child)
-#             return texts
-
-#         result_set = content.find_all(where, class_=what) if what else content.find_all(where)
-#         texts = ""
-#         for tag in result_set:
-#             texts += extract_text(tag) + "\n"
-#         text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
-#         return text_list
-#     else:
-#         result_set = content.find_all(where, class_=what) if what else content.find_all(where)
-#         texts_ = " ".join(tag.get_text() for tag in result_set)
-#         texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
-#         return texts
+    return output_ua 
 def extract_text_from_content(content, content_type="text/html", where=None, what=None, extend=True, **kwargs):
+    """
+    Extracts text from the given content based on the specified content type and search criteria.
+
+    Parameters:
+    - content (str/BeautifulSoup): The content to extract text from.
+    - content_type (str): The type of content, e.g., "text/html" or "application/json".
+    - where (str/list): The HTML tag or list of tags to search for.
+    - what (str): The class name to filter the tags (optional).
+    - extend (bool): Whether to recursively extract text from child elements.
+    - **kwargs: Additional keyword arguments for the search (e.g., id, attributes).
+
+    Returns:
+    - list: A list of extracted text segments.
+    """
     if content is None:
         logger.error("Content is None, cannot extract text.")
         return []
@@ -109,35 +72,41 @@ def extract_text_from_content(content, content_type="text/html", where=None, wha
         where = None
         return extract_text_from_json(content, where)
     elif 'text' in content_type:
-        if extend:
-            def extract_text(element):
-                texts = ""
-                if isinstance(element, str) and element.strip():
-                    texts += element.strip()
-                elif hasattr(element, "children"):
-                    for child in element.children:
-                        texts += extract_text(child)
-                return texts
-
-            search_kwargs = {**kwargs}
-            if what:
-                search_kwargs["class_"] = what
-
-            result_set = content.find_all(where, **search_kwargs)
-            texts = ""
-            for tag in result_set:
-                texts += extract_text(tag) + "\n"
-            text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
-            return text_list
+        if isinstance(where, list):
+            res=[]
+            for where_ in where:
+                res.extend(extract_text_from_content(content, content_type="text/html", where=where_, what=what, extend=extend, **kwargs))
+            return res
         else:
-            search_kwargs = {**kwargs}
-            if what:
-                search_kwargs["class_"] = what
+            if extend:
+                def extract_text(element):
+                    texts = ""
+                    if isinstance(element, str) and element.strip():
+                        texts += element.strip()
+                    elif hasattr(element, "children"):
+                        for child in element.children:
+                            texts += extract_text(child)
+                    return texts
+
+                search_kwargs = {**kwargs}
+                if what:
+                    search_kwargs["class_"] = what
+
+                result_set = content.find_all(where, **search_kwargs)
+                texts = ""
+                for tag in result_set:
+                    texts += extract_text(tag) + "\n"
+                text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
+                return text_list
+            else:
+                search_kwargs = {**kwargs}
+                if what:
+                    search_kwargs["class_"] = what
 
-            result_set = content.find_all(where, **search_kwargs)
-            texts_ = " ".join(tag.get_text() for tag in result_set)
-            texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
-            return texts
+                result_set = content.find_all(where, **search_kwargs)
+                texts_ = " ".join(tag.get_text() for tag in result_set)
+                texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
+                return texts
 
 def extract_text_from_json(content, key=None):
     if key:
@@ -366,7 +335,7 @@ def pdf_detector(url, contains=None, dir_save=None,booster=False):
         print(f'{len(fnames)} files are downloaded:\n{fnames}\n to local: \n{dir_save}')
 
 
-def find_img(url, dir_save="images"):
+def find_img(url, dir_save="images", verbose=True):
     """
     Save images referenced in HTML content locally.
     Args:
@@ -381,7 +350,6 @@ def find_img(url, dir_save="images"):
     if "html" in content_type.lower():
         # Create the directory if it doesn't exist
         os.makedirs(dir_save, exist_ok=True)
-
         # Parse HTML content if it's not already a BeautifulSoup object
         if isinstance(content, str):
             content = BeautifulSoup(content, "html.parser")
@@ -390,13 +358,9 @@ def find_img(url, dir_save="images"):
         images = content.find_all("img", src=True)
         for i, image in enumerate(images):
             try:
-                # Get the image URL
                 image_url = image["src"]
-
                 if image_url.startswith("data:image"):
-                    # Extract the image data from the data URI
                     mime_type, base64_data = image_url.split(",", 1)
-                    # Determine the file extension from the MIME type
                     if ":" in mime_type:
                         # image_extension = mime_type.split(":")[1].split(";")[0]
                         image_extension = (
@@ -406,44 +370,68 @@ def find_img(url, dir_save="images"):
                         image_extension = (
                             "png"  # Default to PNG if extension is not specified
                         )
-                    # if 'svg+xml' in image_extension:
-                    #     image_extension='svg'
                     image_data = base64.b64decode(base64_data)
-                    # Save the image data to a file
                     image_filename = os.path.join(
                         dir_save, f"image_{i}.{image_extension}"
                     )
                     with open(image_filename, "wb") as image_file:
                         image_file.write(image_data)
-
-                    # Update the src attribute of the image tag to point to the local file
                     image["src"] = image_filename
+                    if verbose:
+                        plt.imshow(image_data)
                 else:
                     # Construct the absolute image URL
                     absolute_image_url = urljoin(url, image_url)
-
                     # Parse the image URL to extract the file extension
                     parsed_url = urlparse(absolute_image_url)
                     image_extension = os.path.splitext(parsed_url.path)[1]
-
                     # Download the image
                     image_response = requests.get(absolute_image_url,proxies=proxies_glob)
-
                     # Save the image to a file
                     image_filename = os.path.join(
                         dir_save, f"image_{i}{image_extension}"
                     )
                     with open(image_filename, "wb") as image_file:
                         image_file.write(image_response.content)
-
                     # Update the src attribute of the image tag to point to the local file
                     image["src"] = image_filename
             except (requests.RequestException, KeyError) as e:
                 print(f"Failed to process image {image_url}: {e}")
         print(f"images were saved at\n{dir_save}")
-    # Return the HTML content with updated image URLs
+        if verbose:
+            display_thumbnail_figure(flist(dir_save,filter='img'),dpi=200)
     return content
 
+def display_thumbnail_figure(dir_img_list,figsize=(10,10),dpi=100):
+    import matplotlib.pyplot as plt
+    from PIL import Image
+    """
+    Display a thumbnail figure of all images in the specified directory.
+    Args:
+        dir_img_list (list): List of the Directory containing the images.
+    """
+    num_images = len(dir_img_list)
+
+    if num_images == 0:
+        print("No images found to display.")
+        return
+
+    # Determine grid size
+    grid_size = int(num_images ** 0.5) + 1
+
+    fig, axs = plt.subplots(grid_size, grid_size, figsize=figsize,dpi=dpi)
+
+    for ax, image_file in zip(axs.flatten(), dir_img_list):
+        img = Image.open(image_file)
+        ax.imshow(img)
+        ax.axis('off')  # Hide axes
+
+    # Hide remaining subplots
+    for ax in axs.flatten()[num_images:]:
+        ax.axis('off')
+
+    plt.tight_layout()
+    plt.show()
 
 def content_div_class(content, div="div", div_class="highlight"):
     texts = [div.text for div in content.find_all(div, class_=div_class)]
@@ -467,7 +455,7 @@ def fetch_selenium(
     username_by=By.NAME,
     password_by=By.NAME,
     submit_by=By.NAME,
-    cap_mode='eager', # eager or none
+    # capability='eager', # eager or none
     proxy=None,  # Add proxy parameter
     javascript=True,  # Add JavaScript option
     disable_images=False,  # Add option to disable images
@@ -479,21 +467,16 @@ def fetch_selenium(
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--disable-dev-shm-usage")
     chrome_options.add_argument(f"user-agent={user_agent()}")
-    
     if proxy:
         chrome_options.add_argument(f'--proxy-server={proxy}')
-    
     if disable_images:
         prefs = {"profile.managed_default_content_settings.images": 2}
         chrome_options.add_experimental_option("prefs", prefs)
-    
-    caps = DesiredCapabilities().CHROME
-    caps["pageLoadStrategy"] = "eager"  # You can set this to "none" if needed
-
+    # chrome_options.page_load_strategy = capability
     service = Service(ChromeDriverManager().install())
     for attempt in range(retry):
         try:
-            driver = webdriver.Chrome(service=service, options=chrome_options, desired_capabilities=caps)
+            driver = webdriver.Chrome(service=service, options=chrome_options)
             
             if not javascript:
                 driver.execute_cdp_cmd("Emulation.setScriptExecutionDisabled", {"value": True})
@@ -540,36 +523,19 @@ def fetch_selenium(
 
 
 def fetch(url, where="div", what=None, extend=True, booster=False,retry=2,verbose=False, **kws):
-    # for attempt in range(retry):
-    #     if verbose and attempt==0:
-    #         xample = 'fetch(url,where="div",what=None,extend=True,by=By.TAG_NAME,timeout=10,retry=3,login_url=None,username=None,password=None,username_field="username",password_field="password",submit_field="submit",username_by=By.NAME,password_by=By.NAME,submit_by=By.NAME)'
-    #         print(xample)
-    #     content_type, content = fetch_all(url, parser="html.parser")
-    #     texts=extract_text_from_content(content,content_type=content_type,where=where,what=what,extend=extend, **kws)
-    #     if isinstance(texts,pd.core.frame.DataFrame):
-    #         condition=[texts.empty, attempt != retry - 1]
-    #     else:
-    #         condition=[not texts, attempt != retry - 1]
-    #     if all(condition):
-    #         texts = fetch(url=url, where=where, what=what, extend=extend, retry=retry-1, **kws)
-    #     sleep(random.uniform(0.5, 1.5))
     for attempt in range(retry):
         if verbose and attempt==0:
             xample = 'fetch(url,where="div",what=None,extend=True,by=By.TAG_NAME,timeout=10,retry=3,login_url=None,username=None,password=None,username_field="username",password_field="password",submit_field="submit",username_by=By.NAME,password_by=By.NAME,submit_by=By.NAME)'
             print(xample)
         content_type, content = fetch_all(url, parser="html.parser")
         texts=extract_text_from_content(content,content_type=content_type,where=where,what=what,extend=extend, **kws)
-        if isinstance(texts, pd.core.frame.DataFrame):
-            # condition=[texts.empty, attempt != retry - 1]
+        if isinstance(texts, pd.core.frame.DataFrame): 
             if not texts.empty:
                 break
-        else:
-            # condition=[not texts, attempt != retry - 1]
+        else: 
             if texts:
                 break
-        # if all(condition):
-        #     texts = fetch(url=url, where=where, what=what, extend=extend, retry=retry-1, **kws)
-        sleep(random.uniform(0.5, 1.5))
+            sleep(random.uniform(0.5, 1.5))
     if isinstance(texts,pd.core.frame.DataFrame):
         condition_=[texts.empty, booster]
     else:
@@ -799,4 +765,68 @@ def find_all(url, dir_save=None):
         else:
             df.to_csv(dir_save)
         print(f"file has been saved at\n{dir_save}")
-    return df
+    return df
+
+
+def flist(fpath, filter="all"):
+    all_files = [os.path.join(fpath, f) for f in os.listdir(fpath) if os.path.isfile(os.path.join(fpath, f))]
+    if isinstance(filter, list):
+        filt_files=[]
+        for filter_ in filter:
+            filt_files.extend(flist(fpath, filter_))
+        return filt_files
+    else:
+        if 'all' in filter.lower():
+            return all_files
+        else:
+            filt_files = [f for f in all_files if istype(f, filter)]
+        return filt_files
+
+def istype(fpath, filter='img'):
+    """
+    Filters file paths based on the specified filter.
+    Args:
+        fpath (str): Path to the file.
+        filter (str): filter of file to filter. Default is 'img' for images. Other options include 'doc' for documents,
+                    'zip' for ZIP archives, and 'other' for other types of files.
+    Returns:
+        bool: True if the file matches the filter, False otherwise.
+    """
+    if 'img' in filter.lower():
+        return is_image(fpath)
+    elif 'doc' in filter.lower():
+        return is_document(fpath)
+    elif 'zip' in filter.lower():
+        return is_zip(fpath)
+    else:
+        return False
+
+def is_image(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type and mime_type.startswith('image'):
+        return True
+    else:
+        return False
+
+def is_document(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type and (
+        mime_type.startswith('text/') or
+        mime_type == 'application/pdf' or
+        mime_type == 'application/msword' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' or
+        mime_type == 'application/vnd.ms-excel' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' or
+        mime_type == 'application/vnd.ms-powerpoint' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+    ):
+        return True
+    else:
+        return False
+
+def is_zip(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type == 'application/zip':
+        return True
+    else:
+        return False

{py2ls-0.1.4.2.dist-info → py2ls-0.1.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.4.2
+Version: 0.1.4.4
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

{py2ls-0.1.4.2.dist-info → py2ls-0.1.4.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
-py2ls/.git/COMMIT_EDITMSG,sha256=pQUXyLHaZO274VHn0hpa4QarY5RJacPcvzWdQH-QYcM,71
-py2ls/.git/FETCH_HEAD,sha256=dEddBzNY8P7TDi5FxO4cpwtS-_PitMWUxom-CfUr5dw,100
+py2ls/.git/COMMIT_EDITMSG,sha256=E6ggbgaSjmHbt33GuQe6KX5ipK0Dp29QiuxxpBkqm60,131
+py2ls/.git/FETCH_HEAD,sha256=Cdzu13JkjE_s_0YvmkAIac4l2WuokfA-YimhsDkYo1Q,100
 py2ls/.git/HEAD,sha256=KNJb-Cr0wOK3L1CVmyvrhZ4-YLljCl6MYD2tTdsrboA,21
 py2ls/.git/config,sha256=XswTg1Ts7_7IBDlKHh4OF_0Tq7v4wW7BXb6xSVInSec,345
 py2ls/.git/description,sha256=ZzMxc0Ca26m45Twn1DDnOHqin5VHEZ9uOTBrScIXSjE,16
@@ -16,12 +16,12 @@ py2ls/.git/hooks/pre-receive.sample,sha256=pMPSuce7P9jRRBwxvU7nGlldZrRPz0ndsxAlI
 py2ls/.git/hooks/prepare-commit-msg.sample,sha256=6d3KpBif3dJe2X_Ix4nsp7bKFjkLI5KuMnbwyOGqRhk,1492
 py2ls/.git/hooks/push-to-checkout.sample,sha256=pT0HQXmLKHxt16-mSu5HPzBeZdP0lGO7nXQI7DsSv18,2783
 py2ls/.git/hooks/update.sample,sha256=jV8vqD4QPPCLV-qmdSHfkZT0XL28s32lKtWGCXoU0QY,3650
-py2ls/.git/index,sha256=hMDaY5LEl9oBKXChb9Y1W0VHPaAYzXErgaSun_5Dfvw,1338
+py2ls/.git/index,sha256=Bxygoi0rCfEqm0Lnpaba5ZsAFg_NS8TYUbR2o4uwnnI,1338
 py2ls/.git/info/exclude,sha256=ZnH-g7egfIky7okWTR8nk7IxgFjri5jcXAbuClo7DsE,240
-py2ls/.git/logs/HEAD,sha256=v2lK-ld92AZRnRMM8iB5eQRDBT6SISXYmwjMfGvbepk,1297
-py2ls/.git/logs/refs/heads/main,sha256=v2lK-ld92AZRnRMM8iB5eQRDBT6SISXYmwjMfGvbepk,1297
-py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=0J6k2shFsGg9lj6WgUFDQCxo8HMVyCod44BhjrUUk5E,1782
-py2ls/.git/logs/refs/remotes/origin/main,sha256=1opXMlnx9w8Tj0G9_RkMjK-CXZv553s67VfcFS3G36U,1216
+py2ls/.git/logs/HEAD,sha256=Wiu2pJYW-hD-oZ452dEJyYjYf9wfjUjvJONH7wAi8Ug,1638
+py2ls/.git/logs/refs/heads/main,sha256=Wiu2pJYW-hD-oZ452dEJyYjYf9wfjUjvJONH7wAi8Ug,1638
+py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=032ZeLPOt_lRcwk8gTcVOzQd81zTAkkTk9P1PtPr6Aw,4212
+py2ls/.git/logs/refs/remotes/origin/main,sha256=YU2rplaPYPyRcfSuRQ65bFZ8oQwKogJrBlSXYhD9Pa4,1520
 py2ls/.git/objects/0b/409e1bc918277010f5679b402d1d1dda53e15c,sha256=y5S1XaGxJz1NXi-SPWjPC_NKIqqSbZv9oOg74MzBihY,156
 py2ls/.git/objects/14/449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed,sha256=PomZFmCUCQM1ii0wH-OJGSHLQCTqRtIwE5w3C0TtzSY,171
 py2ls/.git/objects/15/a8e468aacfcb440e090020f36d0b985d45da23,sha256=xiRunMcN5I_B2hHgBUFupR-F0b8H_CQTmmAZG9XkZik,3215
@@ -30,14 +30,17 @@ py2ls/.git/objects/24/6b368b986f758630c46dc02b7fa512b53422f7,sha256=sw7ERFCFu7m6
 py2ls/.git/objects/25/b796accd261b9135fd32a2c00785f68edf6c46,sha256=4ic5vOwEdfbGL8oARSVEeAnSoDs14-gggGZEL-61nYE,564
 py2ls/.git/objects/30/a2f8da47ee947811dc8d993f5a06a45de374f4,sha256=u5W33_qNtTs1-U8Fardx-zB_udqKvuCm5kiw1mQGdsU,3218
 py2ls/.git/objects/36/b4a1b7403abc6c360f8fe2cb656ab945254971,sha256=X18sHo17gQTxusNptoILoWxSfU0ulrsZvbETlBtV5aY,2327
+py2ls/.git/objects/36/e56a361f526eafa59c5235a5c990bf288b5f9c,sha256=7L1L-iqVvuufrlfEE2myD0-QnAPueFMySKetu08W-Pc,34216
 py2ls/.git/objects/36/ef43e50009e59db11812c258846d9e38718173,sha256=0nwCwQSuQAdGyD2NfEK-_L12ydE9nGVKBXOfFq_Lndc,169
 py2ls/.git/objects/3b/bd972aa7ad680858f8dfbd0f7fcd97756f0d6f,sha256=MQWEzML3wbb4GixiHDCHrxgbXLQitrYDstT1plhmQSU,169
 py2ls/.git/objects/3c/bbe5f4173d165127b9ad96119f1ec24c306ffc,sha256=S1BXemROYtzRaj5WXLPYnTmPTBQDKovMEN0GRLul-I4,33489
 py2ls/.git/objects/3f/d6561300938afbb3d11976cf9c8f29549280d9,sha256=91oqbTWfUE1d_hT_1ptYmRUb5pOQ1X4oxQxpF6NXjKU,8501
+py2ls/.git/objects/43/dbd49b2ee367c5434dd545e3b5795434f2ef0b,sha256=DAzt0dWp2KsuuImCKp7N9ia7KaCDNqwB-tYIx3Wf_c0,565
 py2ls/.git/objects/48/a88fc5806305d0bb0755ee6801161b79696972,sha256=f3JStE39k_hPGE-WRwqZtDTjQkfOmBVb_6-ELBbScjI,203
 py2ls/.git/objects/58/20a729045d4dc7e37ccaf8aa8eec126850afe2,sha256=3Pf6NS8OTK4EdHZGVeJ421BtK7w4WJncQDBauZI_wW4,34
 py2ls/.git/objects/60/f273eb1c412d916fa3f11318a7da7a9911b52a,sha256=aJD9iF_LmYSrqDepXFBZKN1yMYbQczVkN_wnrDosBdI,5620
 py2ls/.git/objects/61/570cec8c061abe74121f27f5face6c69b98f99,sha256=IQZi5MkbRu3ToRUPsRcXuh1Xa3pkAz_HDRCVhNL89ds,5753
+py2ls/.git/objects/64/27a4edff08f93d98f511418423f09f2ab90bcd,sha256=RyNngwk9fvdvvvywmNfllnim718fWNjVauH9U2y8Q2s,258
 py2ls/.git/objects/69/13c452ca319f7cbf6a0836dc10a5bb033c84e4,sha256=NYLQQZTfd0htZst42ALS2dmryv1q_l1N19ZfHEbz_38,3193
 py2ls/.git/objects/6b/7fde264d93a7a0986d394c46c7650d0ce2ab92,sha256=iIl0-RF0wd6BSEjzczgUyApxc899PbdTl04JbDn6_-Q,166
 py2ls/.git/objects/78/3d4167bc95c9d2175e0df03ef1c1c880ba75ab,sha256=SK2QDjDBiDhVMG1I5p19g4RbEm2Rax7mYnxawmVZYxs,15523
@@ -52,7 +55,10 @@ py2ls/.git/objects/a7/3e13eafee65c5b8d73ad2d3ea46d0eee82f0d3,sha256=iv3uTzna5XBz
 py2ls/.git/objects/b0/56be4be89ba6b76949dd641df45bb7036050c8,sha256=8Y7z30eNceBd5QIx09QfMp5cYBbrgUllmats0kvJEJ4,132
 py2ls/.git/objects/b0/9cd7856d58590578ee1a4f3ad45d1310a97f87,sha256=82dx4hIdMpdcB64e5PU1s2gZFVkTvrj1cPwwJ_kasNU,4444
 py2ls/.git/objects/b2/18e6a0f0f1c4df8cdefa9852058348abc713b7,sha256=hOQfdyzDZctjoge0-pAcEDel5XHVPNfOtrMNyFPUOIE,564
+py2ls/.git/objects/bb/934eb33bc1a8b85630bf680caffd99560c1b8f,sha256=ggehjexUsWlskHJvHxW7u6U0otB0OCItmIZdT9O-3OU,9670
 py2ls/.git/objects/c4/cba65f1163661999ee4b8ed23342b63bc1300c,sha256=rwSdKt-C98nUQ_B-7imY4fYRYmn29MQc4SIu9wruHeo,566
+py2ls/.git/objects/c6/7f17e5707313600efcb85e9a3fedea35dba591,sha256=TL7rDIWiaWlk8iIwqPst7St5Xr2otPs-vp17GPlET7o,565
+py2ls/.git/objects/cf/0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d,sha256=T_nV0GrgpVu3mOJ4fYcCW98oCunzgqy0DnSX0luy04Q,183
 py2ls/.git/objects/d9/005f2cc7fc4e65f14ed5518276007c08cf2fd0,sha256=IJIoz93V7pf9yx43U1JdN8gBq_LWtw8A9Z2YMPnq_B0,1450
 py2ls/.git/objects/d9/c2403fd166ce791b4e9d0c6792ed8342c71fcd,sha256=uD7BsKdrmN-9FStTpwsRWh-XxVXeDsV4dGjFkaMIIs8,170
 py2ls/.git/objects/db/ffa8ea7bda721d0cee7b9e4ce5b2ef927733ff,sha256=GhDkvP6JYV26qVg5ETPys1ZEnGlsct9hiXCc24Ky4Xg,565
@@ -62,9 +68,9 @@ py2ls/.git/objects/e9/391ffe371f1cc43b42ef09b705d9c767c2e14f,sha256=RWTy2n8L2XxZ
 py2ls/.git/objects/f4/b64d3107b39e3ad6f540c6607004ea34e6c024,sha256=0egAtqc0x8hc7U1z91tIjcRhSd_BT2a_gxZxo_7NTJA,564
 py2ls/.git/objects/f7/c98ba5c2f903e603b1f5e63d49fbc8a43815cc,sha256=tYbi3A7irrIPB_11bwItuof0Vc9a0MDuLFMNAzRsG3A,33467
 py2ls/.git/objects/fc/292e793ecfd42240ac43be407023bd731fa9e7,sha256=hGIYoxKWNT3IPwk3DE4l3FLBbUYF-kXcHcx7KrH9uS0,1971
-py2ls/.git/refs/heads/main,sha256=b_JnHFFXWnMz1M-3Lc6pKdZvxu7ZGWRaSTn-1i008Ro,41
+py2ls/.git/refs/heads/main,sha256=XWF5fg25wz0SqFKCM1pFdyA5I_3pkrwsU8Ql6LH7X0E,41
 py2ls/.git/refs/remotes/origin/HEAD,sha256=K7aiSqD8bEhBAPXVGim7rYQc0sdV9dk_qiBOXbtOsrQ,30
-py2ls/.git/refs/remotes/origin/main,sha256=b_JnHFFXWnMz1M-3Lc6pKdZvxu7ZGWRaSTn-1i008Ro,41
+py2ls/.git/refs/remotes/origin/main,sha256=XWF5fg25wz0SqFKCM1pFdyA5I_3pkrwsU8Ql6LH7X0E,41
 py2ls/.gitattributes,sha256=Gh2-F2vCM7SZ01pX23UT8pQcmauXWfF3gwyRSb6ZAFs,66
 py2ls/.gitignore,sha256=y7GvbD_zZkjPVVIue8AyiuFkDMuUbvMaV65Lgu89To8,2763
 py2ls/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
@@ -75,12 +81,12 @@ py2ls/correlators.py,sha256=RbOaJIPLCHJtUm5SFi_4dCJ7VFUPWR0PErfK3K26ad4,18243
 py2ls/dbhandler.py,sha256=i9dNrpHyx0oIaFieHI4X4tsrCdN-aFxudPTDOgy9Ppo,3574
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
 py2ls/internet_finder.py,sha256=I-jPepbBhpDaOAsD2MqbKMe1CBN8w1PYo0CjNFkaeeU,19149
-py2ls/ips.py,sha256=mrn1V-7p7Xpv-9p0clP2HzY3Jo1NN5pd-nhbycE-y0U,119243
-py2ls/netfinder.py,sha256=IliFR4vsiG-07a0JSCUYtXlxIA9AXiK8QMdbBToFv1M,30693
+py2ls/ips.py,sha256=MH_TRP_lz9AuWMLjPNgTTxNFHcvLLN2XKGQcG9txjP0,121806
+py2ls/netfinder.py,sha256=EbNH-QMFNYsu_eJ68zqEEJRJ5uAYOCrtkqfoC9ZJpo4,31207
 py2ls/sleep_events_detectors.py,sha256=36MCuRrpurn0Uvzpo3p3b3_JlVsRNHSWCXbJxCGM3mg,51546
 py2ls/translator.py,sha256=QfDUO0-pXHGMBFZBefiBHzOrC93-__N5sUQY_VP4wes,29734
 py2ls/version.py,sha256=CactNZqrHHYTPrkHKccy2WKXmaiUdtTgPqSjFyVXnJk,18
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.1.4.2.dist-info/METADATA,sha256=E1ANItbpXa2-LfGfDj0X_AzUxinK_U_0L_zXfd2eREs,11877
-py2ls-0.1.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-py2ls-0.1.4.2.dist-info/RECORD,,
+py2ls-0.1.4.4.dist-info/METADATA,sha256=sar2UcFB1uR1NAUfvx0pxCP-VO_KyT8n_fS6Yu4K0r4,11877
+py2ls-0.1.4.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+py2ls-0.1.4.4.dist-info/RECORD,,