py2ls 0.1.4.1__tar.gz → 0.1.4.2__tar.gz

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.4.1
+Version: 0.1.4.2
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

py2ls-0.1.4.2/py2ls/.git/COMMIT_EDITMSG

@@ -0,0 +1,3 @@
+Update netfinder.py
+
+new features: iframe, disable_image, capabilities

py2ls-0.1.4.2/py2ls/.git/FETCH_HEAD

@@ -0,0 +1 @@
+48a88fc5806305d0bb0755ee6801161b79696972		branch 'main' of https://github.com/Jianfengliu0413/py2ls

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/.git/logs/HEAD

@@ -3,3 +3,6 @@ b056be4be89ba6b76949dd641df45bb7036050c8 d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed Jianfeng <Jianfeng.Liu0413@gmail.com> 1718370666 +0200	commit: Update README.md
 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed 36ef43e50009e59db11812c258846d9e38718173 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718371326 +0200	commit: Update README.md
 36ef43e50009e59db11812c258846d9e38718173 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng <Jianfeng.Liu0413@gmail.com> 1718392848 +0200	commit: readme
+0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
+6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
+3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/.git/logs/refs/heads/main

@@ -3,3 +3,6 @@ b056be4be89ba6b76949dd641df45bb7036050c8 d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed Jianfeng <Jianfeng.Liu0413@gmail.com> 1718370666 +0200	commit: Update README.md
 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed 36ef43e50009e59db11812c258846d9e38718173 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718371326 +0200	commit: Update README.md
 36ef43e50009e59db11812c258846d9e38718173 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng <Jianfeng.Liu0413@gmail.com> 1718392848 +0200	commit: readme
+0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393734 +0200	commit: Update ips.py
+6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491087 +0200	commit: Update netfinder.py
+3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526957 +0200	commit: Update netfinder.py

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -6,3 +6,6 @@ d9c2403fd166ce791b4e9d0c6792ed8342c71fcd d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718390975 +0200	remote set-head
 36ef43e50009e59db11812c258846d9e38718173 36ef43e50009e59db11812c258846d9e38718173 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392810 +0200	remote set-head
 0b409e1bc918277010f5679b402d1d1dda53e15c 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718392852 +0200	remote set-head
+6b7fde264d93a7a0986d394c46c7650d0ce2ab92 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718393737 +0200	remote set-head
+3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718491090 +0200	remote set-head
+48a88fc5806305d0bb0755ee6801161b79696972 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1718526960 +0200	remote set-head

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/.git/logs/refs/remotes/origin/main

@@ -3,3 +3,6 @@ b056be4be89ba6b76949dd641df45bb7036050c8 d9c2403fd166ce791b4e9d0c6792ed8342c71fc
 d9c2403fd166ce791b4e9d0c6792ed8342c71fcd 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed Jianfeng <Jianfeng.Liu0413@gmail.com> 1718370668 +0200	update by push
 14449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed 36ef43e50009e59db11812c258846d9e38718173 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718371329 +0200	update by push
 36ef43e50009e59db11812c258846d9e38718173 0b409e1bc918277010f5679b402d1d1dda53e15c Jianfeng <Jianfeng.Liu0413@gmail.com> 1718392851 +0200	update by push
+0b409e1bc918277010f5679b402d1d1dda53e15c 6b7fde264d93a7a0986d394c46c7650d0ce2ab92 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718393736 +0200	update by push
+6b7fde264d93a7a0986d394c46c7650d0ce2ab92 3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f Jianfeng <Jianfeng.Liu0413@gmail.com> 1718491090 +0200	update by push
+3bbd972aa7ad680858f8dfbd0f7fcd97756f0d6f 48a88fc5806305d0bb0755ee6801161b79696972 Jianfeng <Jianfeng.Liu0413@gmail.com> 1718526959 +0200	update by push

py2ls-0.1.4.2/py2ls/.git/objects/48/a88fc5806305d0bb0755ee6801161b79696972

@@ -0,0 +1,2 @@
+x
��=N�0��}
+��ȱ���F��h�]�;��q{�p
��oF�I{)ܥ^�CoD���d"g��
�
�8�͡"���B#hT�41��5�tAr�Qe��H�u����&�j��ʗ?5}�n�y[�6�������j�X/�VJ:nv����<:�J=sEj��#D�o�	���|��z��'č���:l�"oܙN�!�`&

py2ls-0.1.4.2/py2ls/.git/refs/heads/main

@@ -0,0 +1 @@
+48a88fc5806305d0bb0755ee6801161b79696972

py2ls-0.1.4.2/py2ls/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+48a88fc5806305d0bb0755ee6801161b79696972

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/ips.py

@@ -778,36 +778,6 @@ def dir_name(fpath):
     return dirname(fpath)
 def basename(fpath):
     return os.path.basename(fpath)
-
-def finfo(fpath):
-    fname, fmt = os.path.splitext(fpath)
-    dir_par = os.path.dirname(fpath) + '/'
-    data = {
-        "size": round(os.path.getsize(fpath) / 1024 / 1024, 3),
-        "creation_time": time.ctime(os.path.getctime(fpath)),
-        "ctime": time.ctime(os.path.getctime(fpath)),
-        "mod_time": time.ctime(os.path.getmtime(fpath)),
-        "mtime": time.ctime(os.path.getmtime(fpath)),
-        "parent_dir": dir_par,
-        "fname": fname.replace(dir_par, ""),
-        "kind": fmt
-    }
-    extra_info = {}
-    if data["kind"] == ".pdf":
-        extra_info = pdfinfo_from_path(fpath)
-    
-    return FileInfo(
-        size=data["size"],
-        creation_time=data["creation_time"],
-        ctime=data["ctime"],
-        mod_time=data["mod_time"],
-        mtime=data["mtime"],
-        parent_dir=data["parent_dir"],
-        fname=data["fname"],
-        kind=data["kind"],
-        extra_info=extra_info
-    )
-
 def listdir(
     rootdir,
     kind="folder",
@@ -2841,3 +2811,32 @@ class FileInfo:
             "kind": self.kind,
             **{key: getattr(self, key) for key in vars(self) if key not in ["size", "creation_time", "ctime", "mod_time", "mtime", "parent_dir", "fname", "kind"]}
         }
+
+def finfo(fpath):
+    fname, fmt = os.path.splitext(fpath)
+    dir_par = os.path.dirname(fpath) + '/'
+    data = {
+        "size": round(os.path.getsize(fpath) / 1024 / 1024, 3),
+        "creation_time": time.ctime(os.path.getctime(fpath)),
+        "ctime": time.ctime(os.path.getctime(fpath)),
+        "mod_time": time.ctime(os.path.getmtime(fpath)),
+        "mtime": time.ctime(os.path.getmtime(fpath)),
+        "parent_dir": dir_par,
+        "fname": fname.replace(dir_par, ""),
+        "kind": fmt
+    }
+    extra_info = {}
+    if data["kind"] == ".pdf":
+        extra_info = pdfinfo_from_path(fpath)
+    
+    return FileInfo(
+        size=data["size"],
+        creation_time=data["creation_time"],
+        ctime=data["ctime"],
+        mod_time=data["mod_time"],
+        mtime=data["mtime"],
+        parent_dir=data["parent_dir"],
+        fname=data["fname"],
+        kind=data["kind"],
+        extra_info=extra_info
+    )

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/py2ls/netfinder.py

@@ -20,7 +20,7 @@ from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
-
+import pprint
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -344,7 +344,7 @@ def pdf_detector(url, contains=None, dir_save=None,booster=False):
             pdf_links = filter_links(links=links_all, contains=["pdf"])
 
     if pdf_links:
-        print(f"pdf detected\n{pdf_links}")
+        pprint.pp(f"pdf detected\n{pdf_links}")
     else:
         print('no pdf file')
     if dir_save:
@@ -450,7 +450,6 @@ def content_div_class(content, div="div", div_class="highlight"):
     return texts
 
 
-
 def fetch_selenium(
     url,
     where="div",
@@ -468,7 +467,11 @@ def fetch_selenium(
     username_by=By.NAME,
     password_by=By.NAME,
     submit_by=By.NAME,
+    cap_mode='eager', # eager or none
     proxy=None,  # Add proxy parameter
+    javascript=True,  # Add JavaScript option
+    disable_images=False,  # Add option to disable images
+    iframe_name=None,  # Add option to handle iframe
     **kwargs
 ):
     chrome_options = Options()
@@ -476,13 +479,25 @@ def fetch_selenium(
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--disable-dev-shm-usage")
     chrome_options.add_argument(f"user-agent={user_agent()}")
+    
     if proxy:
         chrome_options.add_argument(f'--proxy-server={proxy}')
-
-    service = Service(ChromeDriverManager().install()) 
+    
+    if disable_images:
+        prefs = {"profile.managed_default_content_settings.images": 2}
+        chrome_options.add_experimental_option("prefs", prefs)
+    
+    caps = DesiredCapabilities().CHROME
+    caps["pageLoadStrategy"] = "eager"  # You can set this to "none" if needed
+
+    service = Service(ChromeDriverManager().install())
     for attempt in range(retry):
         try:
-            driver = webdriver.Chrome(service=service, options=chrome_options)
+            driver = webdriver.Chrome(service=service, options=chrome_options, desired_capabilities=caps)
+            
+            if not javascript:
+                driver.execute_cdp_cmd("Emulation.setScriptExecutionDisabled", {"value": True})
+
             if login_url:
                 driver.get(login_url)
                 WebDriverWait(driver, timeout).until(
@@ -496,6 +511,13 @@ def fetch_selenium(
                 ).click()
 
             driver.get(url)
+            
+            if iframe_name:
+                iframe = WebDriverWait(driver, timeout).until(
+                    EC.presence_of_element_located((By.NAME, iframe_name))
+                )
+                driver.switch_to.frame(iframe)
+
             WebDriverWait(driver, timeout).until(
                 EC.presence_of_element_located((by, where))
             )
@@ -503,7 +525,7 @@ def fetch_selenium(
             driver.quit()
 
             content = BeautifulSoup(page_source, "html.parser")
-            texts=extract_text_from_content(content, where=where,what=what,extend=extend,**kwargs)
+            texts = extract_text_from_content(content, where=where, what=what, extend=extend, **kwargs)
             return texts
         except Exception as e:
             # logger.error(f"Attempt {attempt + 1} failed with error ")

{py2ls-0.1.4.1 → py2ls-0.1.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "py2ls"
-version = "0.1.4.1"
+version = "0.1.4.2"
 description = "py(thon)2(too)ls"
 authors = ["Jianfeng <Jianfeng.Liu0413@gmail.com>"]
 readme = "README.md"

py2ls-0.1.4.1/py2ls/.git/COMMIT_EDITMSG

@@ -1 +0,0 @@
-readme

py2ls-0.1.4.1/py2ls/.git/FETCH_HEAD

@@ -1 +0,0 @@
-0b409e1bc918277010f5679b402d1d1dda53e15c		branch 'main' of https://github.com/Jianfengliu0413/py2ls

py2ls-0.1.4.1/py2ls/.git/refs/heads/main

@@ -1 +0,0 @@
-0b409e1bc918277010f5679b402d1d1dda53e15c

py2ls-0.1.4.1/py2ls/.git/refs/remotes/origin/main

@@ -1 +0,0 @@
-0b409e1bc918277010f5679b402d1d1dda53e15c