py2ls 0.1.10.0__tar.gz → 0.1.10.1__tar.gz

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.1.10.0
+Version: 0.1.10.1
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com

py2ls-0.1.10.1/py2ls/.git/COMMIT_EDITMSG

@@ -0,0 +1 @@
+reg_update

py2ls-0.1.10.1/py2ls/.git/FETCH_HEAD

@@ -0,0 +1 @@
+86e288b46f8fe179907e4413f665aeb5053fddb1		branch 'main' of https://github.com/Jianfengliu0413/py2ls

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/.git/logs/HEAD

@@ -19,3 +19,4 @@ d84688b54c0040a30976b3a6540bc47adf7ce680 32fd627b62fad7cf3b2f9e34ab9777126a0987a
 1c3f92adda34344bcbbbf9d409c79855ae2aaea8 bf67907e337021ebff434e02b19b30a741c144af Jianfeng <Jianfeng.Liu0413@gmail.com> 1720948339 +0200	commit: update
 bf67907e337021ebff434e02b19b30a741c144af a15389729850729fc7bd78a54f26fce77f30be12 Jianfeng <Jianfeng.Liu0413@gmail.com> 1721137352 +0200	commit: update ips
 a15389729850729fc7bd78a54f26fce77f30be12 6dc2cdf4a84e538e5d4777486aeff87e42f41799 Jianfeng <Jianfeng.Liu0413@gmail.com> 1723527985 +0200	commit: update
+6dc2cdf4a84e538e5d4777486aeff87e42f41799 86e288b46f8fe179907e4413f665aeb5053fddb1 Jianfeng <Jianfeng.Liu0413@gmail.com> 1725537214 +0200	commit: reg_update

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/.git/logs/refs/heads/main

@@ -19,3 +19,4 @@ d84688b54c0040a30976b3a6540bc47adf7ce680 32fd627b62fad7cf3b2f9e34ab9777126a0987a
 1c3f92adda34344bcbbbf9d409c79855ae2aaea8 bf67907e337021ebff434e02b19b30a741c144af Jianfeng <Jianfeng.Liu0413@gmail.com> 1720948339 +0200	commit: update
 bf67907e337021ebff434e02b19b30a741c144af a15389729850729fc7bd78a54f26fce77f30be12 Jianfeng <Jianfeng.Liu0413@gmail.com> 1721137352 +0200	commit: update ips
 a15389729850729fc7bd78a54f26fce77f30be12 6dc2cdf4a84e538e5d4777486aeff87e42f41799 Jianfeng <Jianfeng.Liu0413@gmail.com> 1723527985 +0200	commit: update
+6dc2cdf4a84e538e5d4777486aeff87e42f41799 86e288b46f8fe179907e4413f665aeb5053fddb1 Jianfeng <Jianfeng.Liu0413@gmail.com> 1725537214 +0200	commit: reg_update

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/.git/logs/refs/remotes/origin/HEAD

@@ -137,3 +137,4 @@ a15389729850729fc7bd78a54f26fce77f30be12 a15389729850729fc7bd78a54f26fce77f30be1
 a15389729850729fc7bd78a54f26fce77f30be12 a15389729850729fc7bd78a54f26fce77f30be12 Jianfeng Liu <macjianfeng@JFLMBP.cin.medizin.uni-tuebingen.de> 1721326072 +0200	remote set-head
 a15389729850729fc7bd78a54f26fce77f30be12 a15389729850729fc7bd78a54f26fce77f30be12 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1723527981 +0200	remote set-head
 6dc2cdf4a84e538e5d4777486aeff87e42f41799 6dc2cdf4a84e538e5d4777486aeff87e42f41799 Jianfeng Liu <macjianfeng@jflmbp.speedport.ip> 1723527990 +0200	remote set-head
+86e288b46f8fe179907e4413f665aeb5053fddb1 86e288b46f8fe179907e4413f665aeb5053fddb1 Jianfeng Liu <macjianfeng@JFLMBP.cin.medizin.uni-tuebingen.de> 1725537218 +0200	remote set-head

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/.git/logs/refs/remotes/origin/main

@@ -18,3 +18,4 @@ d84688b54c0040a30976b3a6540bc47adf7ce680 32fd627b62fad7cf3b2f9e34ab9777126a0987a
 1c3f92adda34344bcbbbf9d409c79855ae2aaea8 bf67907e337021ebff434e02b19b30a741c144af Jianfeng <Jianfeng.Liu0413@gmail.com> 1720948342 +0200	update by push
 bf67907e337021ebff434e02b19b30a741c144af a15389729850729fc7bd78a54f26fce77f30be12 Jianfeng <Jianfeng.Liu0413@gmail.com> 1721137354 +0200	update by push
 a15389729850729fc7bd78a54f26fce77f30be12 6dc2cdf4a84e538e5d4777486aeff87e42f41799 Jianfeng <Jianfeng.Liu0413@gmail.com> 1723527989 +0200	update by push
+6dc2cdf4a84e538e5d4777486aeff87e42f41799 86e288b46f8fe179907e4413f665aeb5053fddb1 Jianfeng <Jianfeng.Liu0413@gmail.com> 1725537218 +0200	update by push

py2ls-0.1.10.1/py2ls/.git/refs/heads/main

@@ -0,0 +1 @@
+86e288b46f8fe179907e4413f665aeb5053fddb1

py2ls-0.1.10.1/py2ls/.git/refs/remotes/origin/main

@@ -0,0 +1 @@
+86e288b46f8fe179907e4413f665aeb5053fddb1

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/batman.py

@@ -7,20 +7,19 @@ import schedule
 import time
 from datetime import datetime
 import os
+from pprint import pp
+import json
 
 
-signature_styles = [
-    "",  # none
-    (
-        "------------------------------------------------------------------------------------------------\n"
-        "Jianfeng Liu, PhD\n"
-        "Institute of Medical Psychology and Behavioural Neurobiology\n"
-        "University of Tübingen\n"
-        "Otfried-Müller-Str. 25\n"
-        "72076 Tübingen\n\n"
-    ),
-    f"Pappelweg 8\n72076 Tübingen\n\n",
-]
+def extract_kv(
+    dir_data="/Users/macjianfeng/Dropbox/github/python/py2ls/confidential_data/gmail_login.json",
+    idx=0,
+):  # select the index
+    with open(dir_data, "r") as file:
+        email_login = json.load(file)
+    for i, (user, pd) in enumerate(email_login.items()):
+        if i == idx:
+            return user, pd
 
 
 def email_to(**kwargs):
@@ -40,7 +39,10 @@ def email_to(**kwargs):
     who, what, subject = None, None, None
     attachments = False
     pause_sec = False  # default 10 seconds pause
-    signature = signature_styles[0]
+
+    signature_styles = extract_kv(idx=1)[1]  # signature list
+    signature = signature_styles[0]  # default signature,None
+    verbose = True
 
     # params config
     for k, v in kwargs.items():
@@ -67,15 +69,14 @@ def email_to(**kwargs):
         if any(["sig" in k.lower()]):  # 'attachments', 'file'
             signature = v  # optional
             if not isinstance(v, str):
+                print(signature_styles)
                 signature = signature_styles[v]
-
-    # config sender
-    email_address = "andyandhope@gmail.com"
-    email_password = "myff ltls sfym wehe"  # this is fake info
+        if any(["verb" in k.lower(), "show" in k.lower()]):  # 'verbose', 'show'
+            verbose = v  # verbose
 
     # Create email message
     message = MIMEMultipart()
-    message["From"] = email_address
+    message["From"] = extract_kv()[0]
     message["To"] = who
     message["Subject"] = subject
 
@@ -97,10 +98,9 @@ def email_to(**kwargs):
                     part.set_payload(attachment.read())
                 encoders.encode_base64(part)
 
-                fname = os.path.basename(file)
-                print(fname)
                 part.add_header(
-                    "Content-Disposition", f'attachment; filename= "{fname}"'
+                    "Content-Disposition",
+                    f'attachment; filename= "{os.path.basename(file)}"',
                 )
                 message.attach(part)
             except FileNotFoundError:
@@ -109,13 +109,21 @@ def email_to(**kwargs):
                 print(f"Error attaching file {file}: {e}")
     if pause_sec:
         time.sleep(pause_sec)
+
     # Send the email
     try:
         with smtplib.SMTP("smtp.gmail.com", 587) as server:
             server.starttls()
-            server.login(email_address, email_password)
-            server.sendmail(email_address, who, message.as_string())
-        print(f"Email successfully sent to {who}")
+            server.login(extract_kv()[0], extract_kv()[1])
+            server.sendmail(extract_kv()[0], who, message.as_string())
+        if verbose:
+            if attachments:
+                print(
+                    f'\nEmail successfully sent to:\nto:"{who}"\nsubject:{subject}\nattached:'
+                )
+                pp(attachments)
+            else:
+                print(f'\nEmail successfully sent to:\nto:"{who}"\nsubject:{subject}')
     except Exception as e:
         print(f"Failed to send email: {e}")
 
@@ -158,26 +166,33 @@ def example_job():
 
 
 if __name__ == "__main__":
-    # Schedule the job to run every minute
-    schedule.every(1).minutes.do(example_job)
-
-    while True:
-        schedule.run_pending()
-        time.sleep(1)
-
-    # example2:
-    email_to(
-        who="Jianfeng.Liu@medizin.uni-tuebingen.de",
-        subj="test",
-        wha="this is the body",
-        signat="\n\n Best, Jianfeng\n",
-        att="/Users/macjianfeng/Dropbox/Downloads/_*_doc-xlsx/20240822-preprints_full_20190101_20201031-2.xlsx",
-    )
-    # example3:
-    email_to(
-        who="Jianfeng.Liu@medizin.uni-tuebingen.de",
-        subj="test",
-        wha="this is the test",
-        signat=2,
-        att="/Users/macjianfeng/Dropbox/Downloads/_*_doc-xlsx/20240822-preprints_full_20190101_20201031-2.xlsx",
-    )
+    pass
+    # # Schedule the job to run every minute
+    # schedule.every(1).minutes.do(example_job)
+
+    # while True:
+    #     schedule.run_pending()
+    #     time.sleep(1)
+
+    # # example2:
+    # email_to(
+    #     who="Jianfeng.Liu@medizin.uni-tuebingen.de",
+    #     subj="test",
+    #     wha="this is the body",
+    #     signat="\n\n Best, Jianfeng\n",
+    #     att="/Users/macjianfeng/Dropbox/Downloads/_*_doc-xlsx/20240822-preprints_full_20190101_20201031-2.xlsx",
+    # )
+
+    # # example3:
+    # email_to(
+    #     who="Jianfeng.Liu@medizin.uni-tuebingen.de",
+    #     subj="test",
+    #     wha="this is the test",
+    #     signat=2,
+    #     # att="/Users/macjianfeng/Dropbox/Downloads/_*_doc-xlsx/20240822-preprints_full_20190101_20201031-2.xlsx",
+    # )
+
+    # # example_4
+    # print(extract_kv(idx=0))
+    # print(extract_kv(idx=0)[0])
+    # print(extract_kv(idx=1))

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/ips.py

@@ -295,7 +295,7 @@ def is_text(s):
     return has_alpha and has_non_alpha
 
 
-def strcmp(search_term, candidates, ignore_case=True, verbose=True, scorer="WR"):
+def strcmp(search_term, candidates, ignore_case=True, verbose=False, scorer="WR"):
     """
     Compares a search term with a list of candidate strings and finds the best match based on similarity score.
 

{py2ls-0.1.10.0 → py2ls-0.1.10.1}/py2ls/netfinder.py

@@ -63,6 +63,36 @@ def user_agent(
     return output_ua
 
 
+def get_tags(content, ascending=True):
+    tag_names = set()
+
+    # Iterate through all tags in the parsed HTML
+    for tag in content.find_all(True):  # `True` finds all tags
+        tag_names.add(tag.name)  # Add the tag name to the set
+
+    # Convert set to a sorted list for easier reading (optional)
+    if ascending is None:
+        return tag_names
+    else:
+        if ascending:
+            return sorted(tag_names)
+        else:
+            return tag_names
+
+
+def get_attr(content, where, attr):
+    all_tags = get_tags(content)
+    if all([where, attr]):
+        if where in all_tags:
+            element_ = content.find_all(where)
+            return [i[attr] for i in element_]
+        else:
+            print(
+                f"cannot find attr {attr} in tag_name{where}\n or possibly cannot find the tag_names:"
+            )
+            pp(all_tags)
+
+
 def extract_text_from_content(
     content, content_type="text/html", where=None, what=None, extend=True, **kwargs
 ):
@@ -128,7 +158,9 @@ def extract_text_from_content(
                 result_set = content.find_all(where, **search_kwargs)
             else:
                 result_set = content.find_all(where, attrs=dict(**search_kwargs))
-
+            if "get" in kwargs:
+                attr = kwargs["get"]
+                return get_attr(content, where, attr)
             if not result_set:
                 print("Failed: check the 'attrs' setting:  attrs={'id':'xample'}")
             if extend:
@@ -216,6 +248,60 @@ def get_cookies(url, login={"username": "your_username", "password": "your_passw
     return cookies_dict
 
 
+### 更加平滑地移动鼠标, 这样更容易反爬
+def scroll_smth_steps(driver, scroll_pause=0.5, min_step=200, max_step=600):
+    """Smoothly scrolls down the page to trigger lazy loading."""
+    current_scroll_position = 0
+    end_of_page = driver.execute_script("return document.body.scrollHeight")
+
+    while current_scroll_position < end_of_page:
+        step = random.randint(min_step, max_step)
+        driver.execute_script(f"window.scrollBy(0, {step});")
+        time.sleep(scroll_pause)
+
+        # Update the current scroll position
+        current_scroll_position += step
+        end_of_page = driver.execute_script("return document.body.scrollHeight")
+
+
+def scroll_inf2end(driver, scroll_pause=1):
+    """Continuously scrolls until the end of the page is reached."""
+    last_height = driver.execute_script("return document.body.scrollHeight")
+
+    while True:
+        # Scroll to the bottom
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+        time.sleep(scroll_pause)
+
+        # Get the new height after scrolling
+        new_height = driver.execute_script("return document.body.scrollHeight")
+        if new_height == last_height:
+            break  # Exit if no new content is loaded
+        last_height = new_height
+
+
+def corr_by_kind(wait_until_kind):
+    """
+    Map the 'wait_until_kind' string to the appropriate Selenium By strategy.
+    """
+    if "tag" in wait_until_kind:
+        return By.TAG_NAME
+    elif "css" in wait_until_kind:
+        return By.CSS_SELECTOR
+    elif "id" in wait_until_kind:
+        return By.ID
+    elif "name" in wait_until_kind:
+        return By.NAME
+    elif "class" in wait_until_kind:
+        return By.CLASS_NAME
+    elif "path" in wait_until_kind:
+        return By.XPATH
+    elif "link" in wait_until_kind or "text" in wait_until_kind:
+        return By.LINK_TEXT
+    else:
+        raise ValueError(f"Unsupported wait_until_kind: {wait_until_kind}")
+
+
 def fetch_all(
     url,
     parser="lxml",
@@ -224,6 +310,8 @@ def fetch_all(
     timeout=10,
     retry=2,
     wait=0,
+    wait_until=None,
+    wait_until_kind=None,
     scroll_try=3,
     login_url=None,
     username=None,
@@ -308,7 +396,10 @@ def fetch_all(
                 prefs = {"profile.managed_default_content_settings.images": 2}
                 chrome_options.add_experimental_option("prefs", prefs)
             # chrome_options.page_load_strategy = capability
+
             service = Service(ChromeDriverManager().install())
+            # driver_path='/Users/macjianfeng/.wdm/drivers/chromedriver/mac64/127.0.6533.119/chromedriver-mac-arm64/chromedriver'
+            # service=Service(executable_path=driver_path)
 
             driver_ = webdriver.Chrome(service=service, options=chrome_options)
 
@@ -323,6 +414,11 @@ def fetch_all(
                 wait_ = 0
             driver_.implicitly_wait(wait_)
 
+            if wait_until is not None and wait_until_kind is not None:
+                strategy = corr_by_kind(wait_until_kind)
+                WebDriverWait(driver_, timeout).until(
+                    EC.presence_of_element_located((strategy, wait_until))
+                )
             if login_url and login_dict:
                 cookies = get_cookies(url=login_url, login=login_dict)
                 driver_.get(url)
@@ -358,6 +454,9 @@ def fetch_all(
             #     EC.presence_of_element_located((by, where))
             # )
 
+            # # scroll down the page by a certain number of pixels
+            scroll_smth_steps(driver_)
+
             # 设置轮询
             for attempt in range(scroll_try):
                 page_source = driver_.page_source