py2ls 0.1.4.2__tar.gz → 0.1.4.4__tar.gz

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.4.2
+Version: 0.1.4.4
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

py2ls-0.1.4.4/py2ls/.git/COMMIT_EDITMSG

@@ -0,0 +1,4 @@
+listdir type
+
+listdir type 大类别和精确类别,
+e.g.,  kind='img', means filtter all kinds of images, .tiff, tif, jpg, png...

py2ls-0.1.4.4/py2ls/.git/FETCH_HEAD

@@ -0,0 +1 @@
+6427a4edff08f93d98f511418423f09f2ab90bcd		branch 'main' of https://github.com/Jianfengliu0413/py2ls

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/py2ls/.git/logs/HEAD

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/py2ls/.git/logs/refs/heads/main

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553462 +0200	commit: new feature: display_thumbnail_figure
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555179 +0200	commit: listdir type

py2ls-0.1.4.4/py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -0,0 +1,26 @@
+0000000000000000000000000000000000000000 b056be4be89ba6b76949dd641df45bb7036050c8 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370200 +0200	remote set-head
+d9c2403fd166ce791b4e9d0c6792ed8342c71fcd d9c2403fd166ce791b4e9d0c6792ed8342c71fcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370431 +0200	remote set-head
+14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370669 +0200	remote set-head
+36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718371330 +0200	remote set-head
+36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718389149 +0200	remote set-head
+36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718390975 +0200	remote set-head
+36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392810 +0200	remote set-head
+0b409e1bc918277010f5679b402d1d1dda53e15c 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392852 +0200	remote set-head
+6b7fde264d93a7a0986d394c46c7650d0ce2ab92 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718393737 +0200	remote set-head
+3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718491090 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718526960 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718529062 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718531693 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718533521 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718535485 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718537443 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718539464 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718541421 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718543586 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718545544 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718547501 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718549457 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718551415 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553371 +0200	remote set-head
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718553465 +0200	remote set-head
+6427a4edff08f93d98f511418423f09f2ab90bcd 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718555183 +0200	remote set-head

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/py2ls/.git/logs/refs/remotes/origin/main

@@ -6,3 +6,5 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9e
 0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393736 +0200	update by push
 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491090 +0200	update by push
 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526959 +0200	update by push
+48a88fc5806305d0bb0755ee6801161b79696972 cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d Jianfeng <Jianfeng.Liu0413@gmail.com> 1718553464 +0200	update by push
+cf0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d 6427a4edff08f93d98f511418423f09f2ab90bcd Jianfeng <Jianfeng.Liu0413@gmail.com> 1718555183 +0200	update by push

py2ls-0.1.4.4/py2ls/.git/objects/cf/0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d

@@ -0,0 +1 @@
+x
�NIj1�Y��{����4&�C�`�Qk,�4f�!������K-PU�Zk�,��F�����n&�MJd�u$7=eV�1�{ظu0>x�g�h5R��1[�RZ�d�a��u��Z���ʎF��RC��~�t�ic|�B�7;�] �B����O���~�~�klc���2�!^Q�

py2ls-0.1.4.4/py2ls/.git/refs/heads/main

@@ -0,0 +1 @@
+6427a4edff08f93d98f511418423f09f2ab90bcd

py2ls-0.1.4.4/py2ls/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+6427a4edff08f93d98f511418423f09f2ab90bcd

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/py2ls/ips.py

@@ -36,6 +36,7 @@ import time
 from box import Box, BoxList
 from numerizer import numerize
 from tqdm import tqdm
+import mimetypes
 
 def str2num(s, *args):
     delimiter = None
@@ -778,14 +779,15 @@ def dir_name(fpath):
     return dirname(fpath)
 def basename(fpath):
     return os.path.basename(fpath)
+
 def listdir(
     rootdir,
     kind="folder",
     sort_by="name",
     ascending=True,
     contains=None,
-    orient = "list",
-    output='df'
+    orient="list",
+    output="df"
 ):
     def sort_kind(df, by="name", ascending=True):
         if df[by].dtype == 'object':  # Check if the column contains string values
@@ -801,6 +803,69 @@ def listdir(
         sorted_df = df.iloc[sorted_index].reset_index(drop=True)
         return sorted_df
 
+    def flist(fpath, filter="all"):
+        all_files = [os.path.join(fpath, f) for f in os.listdir(fpath) if os.path.isfile(os.path.join(fpath, f))]
+        if isinstance(filter, list):
+            filt_files = []
+            for filter_ in filter:
+                filt_files.extend(flist(fpath, filter_))
+            return filt_files
+        else:
+            if 'all' in filter.lower():
+                return all_files
+            else:
+                filt_files = [f for f in all_files if istype(f, filter)]
+                return filt_files
+
+    def istype(fpath, filter='img'):
+        """
+        Filters file paths based on the specified filter.
+        Args:
+            fpath (str): Path to the file.
+            filter (str): Filter of file to filter. Default is 'img' for images. Other options include 'doc' for documents,
+                        'zip' for ZIP archives, and 'other' for other types of files.
+        Returns:
+            bool: True if the file matches the filter, False otherwise.
+        """
+        if 'img' in filter.lower():
+            return is_image(fpath)
+        elif 'doc' in filter.lower():
+            return is_document(fpath)
+        elif 'zip' in filter.lower():
+            return is_zip(fpath)
+        else:
+            return False
+
+    def is_image(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type and mime_type.startswith('image'):
+            return True
+        else:
+            return False
+
+    def is_document(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type and (
+            mime_type.startswith('text/') or
+            mime_type == 'application/pdf' or
+            mime_type == 'application/msword' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' or
+            mime_type == 'application/vnd.ms-excel' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' or
+            mime_type == 'application/vnd.ms-powerpoint' or
+            mime_type == 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+        ):
+            return True
+        else:
+            return False
+
+    def is_zip(fpath):
+        mime_type, _ = mimetypes.guess_type(fpath)
+        if mime_type == 'application/zip':
+            return True
+        else:
+            return False
+
     if not kind.startswith("."):
         kind = "." + kind
 
@@ -814,7 +879,7 @@ def listdir(
             "path": [],
             "created_time": [],
             "modified_time": [],
-            "last_open_time":[],
+            "last_open_time": [],
             "size": [],
             "fname": [],
             "fpath": [],
@@ -828,16 +893,20 @@ def listdir(
             is_file = kind.lower() in file_extension.lower() and (
                 os.path.isfile(item_path)
             )
-            if not is_folder and not is_file:
-                continue
+            if kind in ['.doc','.img','.zip']: #选择大的类别
+                if kind != ".folder" and not istype(item_path, kind):
+                    continue
+            else: #精确到文件的后缀
+                if not is_folder and not is_file:
+                    continue
             f["name"].append(filename)
             f["length"].append(len(filename))
             f["path"].append(os.path.join(os.path.dirname(item_path), item))
-            fpath=os.path.join(os.path.dirname(item_path), item)
+            fpath = os.path.join(os.path.dirname(item_path), item)
             f["size"].append(round(os.path.getsize(fpath) / 1024 / 1024, 3))
-            f["created_time"].append(pd.to_datetime(os.path.getctime(item_path),unit='s'))
-            f["modified_time"].append(pd.to_datetime(os.path.getmtime(item_path),unit='s'))
-            f['last_open_time'].append(pd.to_datetime(os.path.getatime(item_path),unit='s'))
+            f["created_time"].append(pd.to_datetime(os.path.getctime(item_path), unit='s'))
+            f["modified_time"].append(pd.to_datetime(os.path.getmtime(item_path), unit='s'))
+            f['last_open_time'].append(pd.to_datetime(os.path.getatime(item_path), unit='s'))
             f["fname"].append(filename)  # will be removed
             f["fpath"].append(fpath)  # will be removed
             i += 1
@@ -850,32 +919,35 @@ def listdir(
             'The directory "{}" does NOT exist. Please check the directory "rootdir".'.format(
                 rootdir
             )
-        ) 
+        )
+
     f = pd.DataFrame(f)
+
     if contains is not None:
-        f = f[f["name"].str.contains(contains,case=False)]
+        f = f[f["name"].str.contains(contains, case=False)]
+
     if "nam" in sort_by.lower():
-        # f.sort_values(by="name", ascending=ascending, ignore_index=True, inplace=True)
-        f=sort_kind(f, by="name", ascending=ascending)
+        f = sort_kind(f, by="name", ascending=ascending)
     elif "crea" in sort_by.lower():
-        f=sort_kind(f, by="created_time", ascending=ascending)
+        f = sort_kind(f, by="created_time", ascending=ascending)
     elif "modi" in sort_by.lower():
-        f=sort_kind(f, by="modified_time", ascending=ascending)
+        f = sort_kind(f, by="modified_time", ascending=ascending)
     elif "s" in sort_by.lower() and "z" in sort_by.lower():
-        f=sort_kind(f, by="size", ascending=ascending)
+        f = sort_kind(f, by="size", ascending=ascending)
+
     if 'df' in output:
         return f
     else:
-        if 'l' in orient.lower(): # list # default
+        if 'l' in orient.lower():  # list # default
             res_output = Box(f.to_dict(orient="list"))
             return res_output
-        if 'd' in orient.lower(): # dict
+        if 'd' in orient.lower():  # dict
             return Box(f.to_dict(orient="dict"))
-        if 'r' in orient.lower(): # records
+        if 'r' in orient.lower():  # records
             return Box(f.to_dict(orient="records"))
-        if 'in' in orient.lower(): # records
+        if 'in' in orient.lower():  # records
             return Box(f.to_dict(orient="index"))
-        if 'se' in orient.lower(): # records
+        if 'se' in orient.lower():  # records
             return Box(f.to_dict(orient="series"))
 
 # Example usage:

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/py2ls/netfinder.py

@@ -20,7 +20,9 @@ from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 import pprint
+import mimetypes
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -42,61 +44,22 @@ def user_agent(browsers=["chrome", "edge", "firefox", "safari"], platforms=["pc"
     output_ua = ua.random
     if verbose:
         print(output_ua)
-    return output_ua
-# def extract_text_from_content(content,where,what,extend=False):
-#     if extend:
-#         texts = ""
-
-#         def extract_text(element):
-#             nonlocal texts
-#             if isinstance(element, str) and element.strip():
-#                 texts += element.strip()
-#             elif hasattr(element, "children"):
-#                 for child in element.children:
-#                     extract_text(child)
-
-#         result_set = (
-#             content.find_all(where, class_=what)
-#             if what
-#             else content.find_all(where)
-#         )
-#         for tag in result_set:
-#             extract_text(tag)
-
-#         text = [tx for tx in texts.split("\n") if tx]
-#         return text
-#     else:
-#         result_set = (
-#             content.find_all(where, class_=what)
-#             if what
-#             else content.find_all(where)
-#         )
-#         texts_ = " ".join(tag.get_text() + "\n" for tag in result_set)
-#         texts = [tx.strip() for tx in texts_.split("\n") if tx]
-#         return texts
-# def extract_text_from_content(content, where, what=None, extend=True):
-#     if extend:
-#         def extract_text(element):
-#             texts = ""
-#             if isinstance(element, str) and element.strip():
-#                 texts += element.strip()
-#             elif hasattr(element, "children"):
-#                 for child in element.children:
-#                     texts += extract_text(child)
-#             return texts
-
-#         result_set = content.find_all(where, class_=what) if what else content.find_all(where)
-#         texts = ""
-#         for tag in result_set:
-#             texts += extract_text(tag) + "\n"
-#         text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
-#         return text_list
-#     else:
-#         result_set = content.find_all(where, class_=what) if what else content.find_all(where)
-#         texts_ = " ".join(tag.get_text() for tag in result_set)
-#         texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
-#         return texts
+    return output_ua 
 def extract_text_from_content(content, content_type="text/html", where=None, what=None, extend=True, **kwargs):
+    """
+    Extracts text from the given content based on the specified content type and search criteria.
+
+    Parameters:
+    - content (str/BeautifulSoup): The content to extract text from.
+    - content_type (str): The type of content, e.g., "text/html" or "application/json".
+    - where (str/list): The HTML tag or list of tags to search for.
+    - what (str): The class name to filter the tags (optional).
+    - extend (bool): Whether to recursively extract text from child elements.
+    - **kwargs: Additional keyword arguments for the search (e.g., id, attributes).
+
+    Returns:
+    - list: A list of extracted text segments.
+    """
     if content is None:
         logger.error("Content is None, cannot extract text.")
         return []
@@ -109,35 +72,41 @@ def extract_text_from_content(content, content_type="text/html", where=None, wha
         where = None
         return extract_text_from_json(content, where)
     elif 'text' in content_type:
-        if extend:
-            def extract_text(element):
-                texts = ""
-                if isinstance(element, str) and element.strip():
-                    texts += element.strip()
-                elif hasattr(element, "children"):
-                    for child in element.children:
-                        texts += extract_text(child)
-                return texts
-
-            search_kwargs = {**kwargs}
-            if what:
-                search_kwargs["class_"] = what
-
-            result_set = content.find_all(where, **search_kwargs)
-            texts = ""
-            for tag in result_set:
-                texts += extract_text(tag) + "\n"
-            text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
-            return text_list
+        if isinstance(where, list):
+            res=[]
+            for where_ in where:
+                res.extend(extract_text_from_content(content, content_type="text/html", where=where_, what=what, extend=extend, **kwargs))
+            return res
         else:
-            search_kwargs = {**kwargs}
-            if what:
-                search_kwargs["class_"] = what
+            if extend:
+                def extract_text(element):
+                    texts = ""
+                    if isinstance(element, str) and element.strip():
+                        texts += element.strip()
+                    elif hasattr(element, "children"):
+                        for child in element.children:
+                            texts += extract_text(child)
+                    return texts
+
+                search_kwargs = {**kwargs}
+                if what:
+                    search_kwargs["class_"] = what
+
+                result_set = content.find_all(where, **search_kwargs)
+                texts = ""
+                for tag in result_set:
+                    texts += extract_text(tag) + "\n"
+                text_list = [tx.strip() for tx in texts.split("\n") if tx.strip()]
+                return text_list
+            else:
+                search_kwargs = {**kwargs}
+                if what:
+                    search_kwargs["class_"] = what
 
-            result_set = content.find_all(where, **search_kwargs)
-            texts_ = " ".join(tag.get_text() for tag in result_set)
-            texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
-            return texts
+                result_set = content.find_all(where, **search_kwargs)
+                texts_ = " ".join(tag.get_text() for tag in result_set)
+                texts = [tx.strip() for tx in texts_.split("\n") if tx.strip()]
+                return texts
 
 def extract_text_from_json(content, key=None):
     if key:
@@ -366,7 +335,7 @@ def pdf_detector(url, contains=None, dir_save=None,booster=False):
         print(f'{len(fnames)} files are downloaded:\n{fnames}\n to local: \n{dir_save}')
 
 
-def find_img(url, dir_save="images"):
+def find_img(url, dir_save="images", verbose=True):
     """
     Save images referenced in HTML content locally.
     Args:
@@ -381,7 +350,6 @@ def find_img(url, dir_save="images"):
     if "html" in content_type.lower():
         # Create the directory if it doesn't exist
         os.makedirs(dir_save, exist_ok=True)
-
         # Parse HTML content if it's not already a BeautifulSoup object
         if isinstance(content, str):
             content = BeautifulSoup(content, "html.parser")
@@ -390,13 +358,9 @@ def find_img(url, dir_save="images"):
         images = content.find_all("img", src=True)
         for i, image in enumerate(images):
             try:
-                # Get the image URL
                 image_url = image["src"]
-
                 if image_url.startswith("data:image"):
-                    # Extract the image data from the data URI
                     mime_type, base64_data = image_url.split(",", 1)
-                    # Determine the file extension from the MIME type
                     if ":" in mime_type:
                         # image_extension = mime_type.split(":")[1].split(";")[0]
                         image_extension = (
@@ -406,44 +370,68 @@ def find_img(url, dir_save="images"):
                         image_extension = (
                             "png"  # Default to PNG if extension is not specified
                         )
-                    # if 'svg+xml' in image_extension:
-                    #     image_extension='svg'
                     image_data = base64.b64decode(base64_data)
-                    # Save the image data to a file
                     image_filename = os.path.join(
                         dir_save, f"image_{i}.{image_extension}"
                     )
                     with open(image_filename, "wb") as image_file:
                         image_file.write(image_data)
-
-                    # Update the src attribute of the image tag to point to the local file
                     image["src"] = image_filename
+                    if verbose:
+                        plt.imshow(image_data)
                 else:
                     # Construct the absolute image URL
                     absolute_image_url = urljoin(url, image_url)
-
                     # Parse the image URL to extract the file extension
                     parsed_url = urlparse(absolute_image_url)
                     image_extension = os.path.splitext(parsed_url.path)[1]
-
                     # Download the image
                     image_response = requests.get(absolute_image_url,proxies=proxies_glob)
-
                     # Save the image to a file
                     image_filename = os.path.join(
                         dir_save, f"image_{i}{image_extension}"
                     )
                     with open(image_filename, "wb") as image_file:
                         image_file.write(image_response.content)
-
                     # Update the src attribute of the image tag to point to the local file
                     image["src"] = image_filename
             except (requests.RequestException, KeyError) as e:
                 print(f"Failed to process image {image_url}: {e}")
         print(f"images were saved at\n{dir_save}")
-    # Return the HTML content with updated image URLs
+        if verbose:
+            display_thumbnail_figure(flist(dir_save,filter='img'),dpi=200)
     return content
 
+def display_thumbnail_figure(dir_img_list,figsize=(10,10),dpi=100):
+    import matplotlib.pyplot as plt
+    from PIL import Image
+    """
+    Display a thumbnail figure of all images in the specified directory.
+    Args:
+        dir_img_list (list): List of the Directory containing the images.
+    """
+    num_images = len(dir_img_list)
+
+    if num_images == 0:
+        print("No images found to display.")
+        return
+
+    # Determine grid size
+    grid_size = int(num_images ** 0.5) + 1
+
+    fig, axs = plt.subplots(grid_size, grid_size, figsize=figsize,dpi=dpi)
+
+    for ax, image_file in zip(axs.flatten(), dir_img_list):
+        img = Image.open(image_file)
+        ax.imshow(img)
+        ax.axis('off')  # Hide axes
+
+    # Hide remaining subplots
+    for ax in axs.flatten()[num_images:]:
+        ax.axis('off')
+
+    plt.tight_layout()
+    plt.show()
 
 def content_div_class(content, div="div", div_class="highlight"):
     texts = [div.text for div in content.find_all(div, class_=div_class)]
@@ -467,7 +455,7 @@ def fetch_selenium(
     username_by=By.NAME,
     password_by=By.NAME,
     submit_by=By.NAME,
-    cap_mode='eager', # eager or none
+    # capability='eager', # eager or none
     proxy=None,  # Add proxy parameter
     javascript=True,  # Add JavaScript option
     disable_images=False,  # Add option to disable images
@@ -479,21 +467,16 @@ def fetch_selenium(
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--disable-dev-shm-usage")
     chrome_options.add_argument(f"user-agent={user_agent()}")
-    
     if proxy:
         chrome_options.add_argument(f'--proxy-server={proxy}')
-    
     if disable_images:
         prefs = {"profile.managed_default_content_settings.images": 2}
         chrome_options.add_experimental_option("prefs", prefs)
-    
-    caps = DesiredCapabilities().CHROME
-    caps["pageLoadStrategy"] = "eager"  # You can set this to "none" if needed
-
+    # chrome_options.page_load_strategy = capability
     service = Service(ChromeDriverManager().install())
     for attempt in range(retry):
         try:
-            driver = webdriver.Chrome(service=service, options=chrome_options, desired_capabilities=caps)
+            driver = webdriver.Chrome(service=service, options=chrome_options)
             
             if not javascript:
                 driver.execute_cdp_cmd("Emulation.setScriptExecutionDisabled", {"value": True})
@@ -540,36 +523,19 @@ def fetch_selenium(
 
 
 def fetch(url, where="div", what=None, extend=True, booster=False,retry=2,verbose=False, **kws):
-    # for attempt in range(retry):
-    #     if verbose and attempt==0:
-    #         xample = 'fetch(url,where="div",what=None,extend=True,by=By.TAG_NAME,timeout=10,retry=3,login_url=None,username=None,password=None,username_field="username",password_field="password",submit_field="submit",username_by=By.NAME,password_by=By.NAME,submit_by=By.NAME)'
-    #         print(xample)
-    #     content_type, content = fetch_all(url, parser="html.parser")
-    #     texts=extract_text_from_content(content,content_type=content_type,where=where,what=what,extend=extend, **kws)
-    #     if isinstance(texts,pd.core.frame.DataFrame):
-    #         condition=[texts.empty, attempt != retry - 1]
-    #     else:
-    #         condition=[not texts, attempt != retry - 1]
-    #     if all(condition):
-    #         texts = fetch(url=url, where=where, what=what, extend=extend, retry=retry-1, **kws)
-    #     sleep(random.uniform(0.5, 1.5))
     for attempt in range(retry):
         if verbose and attempt==0:
             xample = 'fetch(url,where="div",what=None,extend=True,by=By.TAG_NAME,timeout=10,retry=3,login_url=None,username=None,password=None,username_field="username",password_field="password",submit_field="submit",username_by=By.NAME,password_by=By.NAME,submit_by=By.NAME)'
             print(xample)
         content_type, content = fetch_all(url, parser="html.parser")
         texts=extract_text_from_content(content,content_type=content_type,where=where,what=what,extend=extend, **kws)
-        if isinstance(texts, pd.core.frame.DataFrame):
-            # condition=[texts.empty, attempt != retry - 1]
+        if isinstance(texts, pd.core.frame.DataFrame): 
             if not texts.empty:
                 break
-        else:
-            # condition=[not texts, attempt != retry - 1]
+        else: 
             if texts:
                 break
-        # if all(condition):
-        #     texts = fetch(url=url, where=where, what=what, extend=extend, retry=retry-1, **kws)
-        sleep(random.uniform(0.5, 1.5))
+            sleep(random.uniform(0.5, 1.5))
     if isinstance(texts,pd.core.frame.DataFrame):
         condition_=[texts.empty, booster]
     else:
@@ -799,4 +765,68 @@ def find_all(url, dir_save=None):
         else:
             df.to_csv(dir_save)
         print(f"file has been saved at\n{dir_save}")
-    return df
+    return df
+
+
+def flist(fpath, filter="all"):
+    all_files = [os.path.join(fpath, f) for f in os.listdir(fpath) if os.path.isfile(os.path.join(fpath, f))]
+    if isinstance(filter, list):
+        filt_files=[]
+        for filter_ in filter:
+            filt_files.extend(flist(fpath, filter_))
+        return filt_files
+    else:
+        if 'all' in filter.lower():
+            return all_files
+        else:
+            filt_files = [f for f in all_files if istype(f, filter)]
+        return filt_files
+
+def istype(fpath, filter='img'):
+    """
+    Filters file paths based on the specified filter.
+    Args:
+        fpath (str): Path to the file.
+        filter (str): filter of file to filter. Default is 'img' for images. Other options include 'doc' for documents,
+                    'zip' for ZIP archives, and 'other' for other types of files.
+    Returns:
+        bool: True if the file matches the filter, False otherwise.
+    """
+    if 'img' in filter.lower():
+        return is_image(fpath)
+    elif 'doc' in filter.lower():
+        return is_document(fpath)
+    elif 'zip' in filter.lower():
+        return is_zip(fpath)
+    else:
+        return False
+
+def is_image(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type and mime_type.startswith('image'):
+        return True
+    else:
+        return False
+
+def is_document(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type and (
+        mime_type.startswith('text/') or
+        mime_type == 'application/pdf' or
+        mime_type == 'application/msword' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' or
+        mime_type == 'application/vnd.ms-excel' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' or
+        mime_type == 'application/vnd.ms-powerpoint' or
+        mime_type == 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+    ):
+        return True
+    else:
+        return False
+
+def is_zip(fpath):
+    mime_type, _ = mimetypes.guess_type(fpath)
+    if mime_type == 'application/zip':
+        return True
+    else:
+        return False

{py2ls-0.1.4.2 → py2ls-0.1.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "py2ls"
-version = "0.1.4.2"
+version = "0.1.4.4"
 description = "py(thon)2(too)ls"
 authors = ["Jianfeng <Jianfeng.Liu0413@gmail.com>"]
 readme = "README.md"

py2ls-0.1.4.2/py2ls/.git/COMMIT_EDITMSG

@@ -1,3 +0,0 @@
-Update netfinder.py
-
-new features: iframe, disable_image, capabilities

py2ls-0.1.4.2/py2ls/.git/FETCH_HEAD

@@ -1 +0,0 @@
-48a88fc5806305d0bb0755ee6801161b79696972		branch 'main' of https://github.com/Jianfengliu0413/py2ls

py2ls-0.1.4.2/py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -1,11 +0,0 @@
-0000000000000000000000000000000000000000 b056be4be89ba6b76949dd641df45bb7036050c8 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370200 +0200	remote set-head
-d9c2403fd166ce791b4e9d0c6792ed8342c71fcd d9c2403fd166ce791b4e9d0c6792ed8342c71fcd Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370431 +0200	remote set-head
-14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718370669 +0200	remote set-head
-36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718371330 +0200	remote set-head
-36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718389149 +0200	remote set-head
-36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718390975 +0200	remote set-head
-36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392810 +0200	remote set-head
-0b409e1bc918277010f5679b402d1d1dda53e15c 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392852 +0200	remote set-head
-6b7fde264d93a7a0986d394c46c7650d0ce2ab92 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718393737 +0200	remote set-head
-3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718491090 +0200	remote set-head
-48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718526960 +0200	remote set-head

py2ls-0.1.4.2/py2ls/.git/refs/heads/main

@@ -1 +0,0 @@
-48a88fc5806305d0bb0755ee6801161b79696972

py2ls-0.1.4.2/py2ls/.git/refs/remotes/origin/main

@@ -1 +0,0 @@
-48a88fc5806305d0bb0755ee6801161b79696972