jacksung-dev 0.0.4.15__py3-none-any.whl

jacksung/utils/login.py

@@ -0,0 +1,149 @@
+import time
+import os
+import requests
+import subprocess
+# 从selenium导入webdriver
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+import platform
+from selenium.common.exceptions import NoSuchElementException
+import traceback, sys
+import argparse
+import yaml
+
+
+class ecnu_login:
+    def __init__(self, driver_path=None, tmp_path=None):
+        options = webdriver.ChromeOptions()
+
+        options.add_argument("--no-sandbox")
+        options.add_argument('--disable-dev-shm-usage')
+        options.add_argument('--user-agent=Mozilla/5.0 HAHA')
+        options.add_experimental_option("detach", True)
+        options.add_argument('--headless')  # 浏览器隐式启动
+        options.add_argument("--window-size=1920,1080")
+        self.url = "https://login.ecnu.edu.cn/"
+        self.driver = None
+        if tmp_path:
+            options.add_argument("crash-dumps-dir=" + tmp_path)
+        self.options = options
+        self.driver_path = driver_path
+
+    def get_drive(self):
+        print('driver is going to init!')
+        self.driver = webdriver.Chrome(service=Service(self.driver_path) if self.driver_path else None,
+                                       options=self.options)
+        print('driver is inited!')
+        self.driver.get(self.url)
+        time.sleep(0.5)
+        self.driver.refresh()
+        time.sleep(2)
+
+    #     todo:准备打包
+    def refresh_drive(self):
+        self.driver.get(self.url)
+        self.driver.refresh()
+        time.sleep(3)
+
+    def close_driver(self):
+        self.driver.close()
+        self.driver.quit()
+
+    def login_check(self, username, password):
+        print('checking net......')
+        if self.print_ip():
+            return True
+        else:
+            return self.login(username, password)
+
+    def print_ip(self):
+        try:
+            time.sleep(4)
+            ipv4 = self.driver.find_element(By.ID, 'ipv4')
+            if ipv4:
+                print('当前IP：', ipv4.text)
+                return True
+        except Exception as e:
+            return False
+
+    def login(self, username, password):
+        if self.print_ip():
+            print('目前已登录！')
+            return True
+        driver = self.driver
+        try:
+            driver.refresh()
+            time.sleep(2)
+            username_ele = driver.find_element(By.ID, "username")
+            password_ele = driver.find_element(By.ID, "password")
+            username_ele.send_keys(username)
+            password_ele.send_keys(password)
+            print(rf'当前登录账户:{username}')
+            # driver.find_element(By.ID, 'login-account').click()
+            button = driver.find_element(By.ID, 'login-account')
+            driver.execute_script("arguments[0].click();", button)
+            time.sleep(2)
+            try:
+                ipv4 = driver.find_element(By.ID, 'ipv4')
+                print('登录成功')
+                print('当前IP：', ipv4.text)
+                return True
+            except NoSuchElementException as e:
+                print(driver.find_element(By.XPATH,
+                                          "/html/body/div[2]/div[@class='component dialog confirm active']/div[@class='content']/div[@class='section']")
+                      .text)
+                return False
+        except Exception as e:
+            print('登录失败！')
+            traceback.print_exc()
+            driver.save_screenshot('err.png')
+            return False
+
+    def logout(self):
+        driver = self.driver
+        try:
+            driver.find_element(By.ID, 'logout').click()
+            time.sleep(3)
+            driver.find_element(By.CLASS_NAME, 'btn-confirm').click()
+            print('登出成功')
+            return True
+        except Exception as e:
+            print('登出失败！')
+            traceback.print_exc()
+            return False
+
+
+def main():
+    if platform.system().lower() == 'windows':
+        driver_path = os.path.expanduser("~/chrome/chromedriver.exe")
+    else:
+        driver_path = os.path.expanduser("~/chrome/chromedriver")
+
+    login = ecnu_login(driver_path=driver_path, tmp_path=os.path.expanduser("~/chrome/tmp"))
+    login.get_drive()
+    parser = argparse.ArgumentParser(
+        prog='login',  # 程序名
+        description='login or logout',  # 描述
+        epilog='Copyright(r), 2023'  # 说明信息
+    )
+
+    parser.add_argument('-t', default='login_check')
+    parser.add_argument('-u', default='')
+    parser.add_argument('-p', default='')
+    args = parser.parse_args()
+    config_file = os.path.expanduser("~/.ecnu_login")
+    if os.path.exists(config_file):
+        opt = vars(args)
+        yaml_args = yaml.load(open(config_file), Loader=yaml.FullLoader)
+        if args.u != '' and 'u' in yaml_args:
+            yaml_args.pop('u')
+        if args.p != '' and 'p' in yaml_args:
+            yaml_args.pop('p')
+        opt.update(yaml_args)
+    if args.t == 'login_check':
+        login.login_check(args.u, args.p)
+    elif args.t == 'login':
+        login.login(args.u, args.p)
+    elif args.t == 'logout':
+        login.logout()

jacksung/utils/mean_std.py

@@ -0,0 +1,66 @@
+import numpy as np
+
+
+def mean_std_part2all(num_list, mean_list, std_list):
+    # [(n,mean,std)]
+    group_num = len(num_list)
+    dim = mean_list[0].shape
+    data_num = sum(num_list)
+
+    global_mean = np.zeros(dim, dtype=np.float64)
+    global_var = np.zeros(dim, dtype=np.float64)
+    for i in range(group_num):
+        global_mean += (num_list[i] / data_num * mean_list[i])
+    for i in range(group_num):
+        global_var += (num_list[i] / data_num * (std_list[i] ** 2 + (global_mean - mean_list[i]) ** 2))
+    return global_mean, np.sqrt(global_var)
+
+
+def cal_mean_std_one_loop(s, ss, count):
+    mean_pixel = s / count
+    mean_level = mean_pixel.mean((-2, -1))
+
+    var_pixel = ss / count - np.square(mean_pixel)
+    ss_level = ss.mean((-2, -1))
+    var_level = ss_level / count - np.square(mean_level)
+
+    std_level = np.sqrt(var_level)
+    std_pixel = np.sqrt(var_pixel)
+
+    return mean_pixel, std_pixel, mean_level, std_level
+
+
+if __name__ == '__main__':
+    d1 = np.random.rand(2, 3, 2, 2)
+    d2 = np.random.rand(3, 3, 2, 2)
+    d = np.concatenate([d1, d2])
+    s1 = d1.sum((0,))
+    ss1 = np.array([i ** 2 for i in d1]).sum((0,))
+
+    s2 = d2.sum((0,))
+    ss2 = np.array([i ** 2 for i in d2]).sum((0,))
+
+    mean_pixel1, std_pixel1, mean_level1, std_level1 = cal_mean_std_one_loop(s1, ss1, 2)
+    mean_pixel2, std_pixel2, mean_level2, std_level2 = cal_mean_std_one_loop(s2, ss2, 3)
+    # print('*' * 40)
+    # print(mean_pixel1, end='\n\n')
+    # print(std_pixel1, end='\n\n')
+    # print(mean_level1, end='\n\n')
+    print(std_level1)
+    # print('*' * 40)
+    # print(mean_pixel2, end='\n\n')
+    # print(std_pixel2, end='\n\n')
+    # print(mean_level2, end='\n\n')
+    print(std_level2)
+
+    mean_pixel, std_pixel = mean_std_part2all([2, 3], [mean_pixel1, mean_pixel2], [std_pixel1, std_pixel2])
+    mean_level, std_level = mean_std_part2all([2, 3], [mean_level1, mean_level2], [std_level1, std_level2])
+    # print('*' * 40)
+    # print(mean_pixel, end='\n\n')
+    # print(std_pixel, end='\n\n')
+    # print(mean_level, end='\n\n')
+    print(std_level)
+    print('*' * 40)
+    print(d1.std((0, -2, -1)))
+    print(d2.std((0, -2, -1)))
+    print(d.std((0, -2, -1)))

jacksung/utils/multi_task.py

@@ -0,0 +1,129 @@
+import concurrent.futures
+import threading
+import multiprocessing
+import time
+from tqdm import tqdm
+
+type_thread = 'type_thread'
+type_process = 'type_process'
+
+
+class ThreadingLock:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.owner = None
+
+    def acquire(self, blocking=True):
+        acquired = self.lock.acquire(blocking)
+        if acquired:
+            self.owner = threading.current_thread().name
+        return acquired
+
+    def release(self):
+        self.lock.release()
+        self.owner = None
+
+    def get_owner(self):
+        return self.owner
+
+
+def init_process(pl):
+    """进程池初始化函数，只设置进程锁"""
+    global p_lock
+    p_lock = pl
+
+
+def init_thread(tl):
+    """线程池初始化函数，只设置线程锁"""
+    global t_lock
+    t_lock = tl
+
+
+class MultiTasks:
+    def __init__(self, threads=10, pool=type_thread, desc="Mult. Pro."):
+        self.threads = threads
+        self.task_list = {}
+        self.features = {}
+        self.results = {}
+        self.submitted = []
+        self.pool_type = pool
+
+        if pool == type_thread:
+            # 线程池不需要在初始化时传递锁
+            self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.threads)
+        elif pool == type_process:
+            # 进程池需要特殊处理锁
+            self.process_mutex = multiprocessing.Manager().Lock()
+            self.executor = concurrent.futures.ProcessPoolExecutor(
+                max_workers=self.threads,
+                initializer=init_process,
+                initargs=(self.process_mutex,)
+            )
+        self.desc = desc
+        self.progress_bar = None
+
+    def add_task(self, k, function, args):
+        self.task_list[k] = (function, args)
+
+    def execute_task_nowait(self, save=False, print_log=False):
+        if print_log:
+            print('Now running tasks:', rf'{len(self.submitted)}/{len(self.task_list)}')
+        for k, f_and_a in self.task_list.items():
+            if k not in self.submitted:
+                r = self.executor.submit(f_and_a[0], *f_and_a[1])
+                if save:
+                    self.features[k] = r
+                self.submitted.append(k)
+
+    def wrap_fun(self, fun, args):
+        """包装函数，处理进度条更新"""
+        result = fun(*args)
+        if self.progress_bar:
+            # 使用适当的方式更新进度条
+            if self.pool_type == type_thread:
+                # 线程环境下直接更新
+                self.progress_bar.update(1)
+            else:
+                # 进程环境下需要特殊处理，这里简化处理
+                raise Exception('进程环境下需要特殊处理，这里简化处理.')
+        return result
+
+    def execute_task(self, print_percent=True, desc=None):
+        if print_percent:
+            self.progress_bar = tqdm(total=len(self.task_list), desc=desc if desc else self.desc)
+
+        # 根据池类型选择不同的执行策略
+        if self.pool_type == type_thread:
+            # 线程池直接执行
+            for k, f_and_a in self.task_list.items():
+                self.features[k] = self.executor.submit(self.wrap_fun, f_and_a[0], f_and_a[1])
+        else:
+            # 进程池需要避免传递不可序列化对象
+            for k, f_and_a in self.task_list.items():
+                # 直接提交函数，不包装进度条更新
+                self.features[k] = self.executor.submit(f_and_a[0], *f_and_a[1])
+
+        # 收集结果并更新进度条
+        for i, (k, feature) in enumerate(self.features.items()):
+            self.results[k] = feature.result()
+            if self.progress_bar and self.pool_type == type_process:
+                # 进程环境下在主线程更新进度条
+                self.progress_bar.update(1)
+
+        if self.progress_bar:
+            self.progress_bar.close()
+
+        return self.results
+
+
+def worker(i):
+    # print(i)
+    time.sleep(0.1)
+    return i * 2
+
+
+if __name__ == '__main__':
+    mt = MultiTasks(10, pool=type_thread)
+    for i in range(100):
+        mt.add_task(i, worker, [i])
+    mt.execute_task()

jacksung/utils/number.py

@@ -0,0 +1,6 @@
+def round2str(number, digits=2):
+    number = round(number, digits)
+    if digits == 0:
+        return str(int(number))
+    else:
+        return f"{number:.{digits}f}"

jacksung/utils/nvidia.py

@@ -0,0 +1,140 @@
+import sys
+import os
+import re
+import subprocess
+import select
+import argparse
+from termcolor import colored
+
+MEMORY_FREE_RATIO = 0.05
+MEMORY_MODERATE_RATIO = 0.9
+GPU_FREE_RATIO = 0.05
+GPU_MODERATE_RATIO = 0.75
+
+
+def colorize(_lines):
+    for i in range(len(_lines)):
+        line = _lines[i]
+        m = re.match(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+\/\s+([0-9]+)MiB.*\s([0-9]+)%", line)
+        if m is not None:
+            used_mem = int(m.group(1))
+            total_mem = int(m.group(2))
+            gpu_util = int(m.group(3)) / 100.0
+            mem_util = used_mem / float(total_mem)
+
+            is_low = is_moderate = is_high = False
+            is_high = gpu_util >= GPU_MODERATE_RATIO or mem_util >= MEMORY_MODERATE_RATIO
+            if not is_high:
+                is_moderate = gpu_util >= GPU_FREE_RATIO or mem_util >= MEMORY_FREE_RATIO
+
+            if not is_high and not is_moderate:
+                is_free = True
+
+            c = 'red' if is_high else ('yellow' if is_moderate else 'green')
+            _lines[i] = colored(_lines[i], c)
+            _lines[i - 1] = colored(_lines[i - 1], c)
+
+    return _lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='nvidia-watch',  # 程序名
+        description='watch gpu',  # 描述
+        epilog='Copyright(r), 2023'  # 说明信息
+    )
+    parser.add_argument('-l', '--command-length', default=20, const=100, type=int, nargs='?')
+    parser.add_argument('-c', '--color', action='store_true')
+    args = parser.parse_args()
+
+    # parse the command length argument
+    command_length = args.command_length
+    color = args.color
+    processes = subprocess.run('nvidia-smi', stdout=subprocess.PIPE)
+    lines = processes.stdout.decode().split("\n")[:-1]
+    lines_to_print = []
+    # Copy the utilization upper part verbatim
+    pid_idx = 0
+    for i in range(len(lines)):
+        if not lines[i].startswith("| Processes:"):
+            # if lines[i].count('MIG M.') > 0 or lines[i].count('N/A') > 0:
+            #     continue
+            lines_to_print.append(lines[i].rstrip())
+        else:
+            pid_idx = i + 3
+            break
+
+    if color:
+        lines_to_print = colorize(lines_to_print)
+
+    for line in lines_to_print:
+        print(line)
+
+    # Parse the PIDs from the lower part
+    gpu_num = []
+    pid = []
+    gpu_mem = []
+    user = []
+    cpu = []
+    mem = []
+    time = []
+    command = []
+    for i in range(pid_idx, len(lines)):
+        if lines[i].startswith("+--") or lines[i].startswith('|==') or "Not Supported" in lines[i]:
+            continue
+        no_running_process = "No running processes found"
+        if no_running_process in lines[i]:
+            print("|  " + no_running_process + " " * (83 - len(no_running_process)) + "  |")
+            print(lines[-1])
+            sys.exit()
+        line = lines[i]
+        line = re.split(r'\s+', line)
+        gpu_num.append(line[1])
+        pid.append(line[4])
+        gpu_mem.append(line[7])
+        user.append("")
+        cpu.append("")
+        mem.append("")
+        time.append("")
+        command.append("")
+
+    # Query the PIDs using ps
+    ps_format = "pid,user,%cpu,%mem,etime,command"
+    processes = subprocess.run(["ps", "-o", ps_format, "-p", ",".join(pid)], stdout=subprocess.PIPE)
+
+    # Parse ps output
+    for line in processes.stdout.decode().split("\n"):
+        if line.strip().startswith('PID') or len(line) == 0:
+            continue
+        parts = re.split(r'\s+', line.strip(), 5)
+        idx = pid.index(parts[0])
+        user[idx] = parts[1]
+        cpu[idx] = parts[2]
+        mem[idx] = parts[3]
+        time[idx] = parts[4] if not "-" in parts[4] else parts[4].split("-")[0] + " days"
+        command[idx] = parts[5][0:100]
+
+    format = ("| %5s %10s %8s %10s %8s %8s %8s %-" + str(command_length) + "." + str(command_length) + "s  |")
+
+    print(format % (
+        "GPU", "PID", "USER", "GPU MEM", "%CPU", "%MEM", "TIME", "COMMAND"
+    ))
+
+    for i in range(len(pid)):
+        # continue
+        print(format % (
+            gpu_num[i],
+            pid[i],
+            user[i],
+            gpu_mem[i],
+            cpu[i],
+            mem[i],
+            time[i],
+            command[i]
+        ))
+
+    print(lines[-1])
+
+
+if __name__ == '__main__':
+    main()

jacksung/utils/time.py

@@ -0,0 +1,87 @@
+import time
+from datetime import datetime
+import pytz
+from jacksung.utils.log import oprint
+import calendar
+
+
+def get_days_in_month(year, month):
+    return calendar.monthrange(year, month)[1]
+
+
+def cal_time(fun_str):
+    # 记录第一个时间戳
+    start_time = time.time()
+    eval(fun_str)
+    # 记录第二个时间戳
+    end_time = time.time()
+    # 计算耗时
+    elapsed_time = end_time - start_time
+    print("Elapsed Time:", elapsed_time, "seconds")
+
+def get_time_str():
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+def cur_timestamp_str():
+    now = datetime.now()
+    year = str(now.year)
+    month = str(now.month).zfill(2)
+    day = str(now.day).zfill(2)
+    hour = str(now.hour).zfill(2)
+    minute = str(now.minute).zfill(2)
+    second = str(now.second).zfill(2)
+    microsecond = str(now.microsecond // 10000).zfill(2)
+    content = "{}-{}{}-{}{}-{}{}".format(year, month, day, hour, minute, second, microsecond)
+    return content
+
+
+class RemainTime:
+    def __init__(self, epoch):
+        self.start_time = time.time()
+        self.epoch = epoch
+        self.now_epoch = 0
+
+    def update(self, log_temp='Spe {:.0f}s, Rem Epoch:{}, Fin in {}', print_log=True, update_step=1):
+        self.now_epoch += update_step
+        epoch_time = time.time() - self.start_time
+        epoch_remaining = self.epoch - self.now_epoch
+        time_remaining = epoch_time * epoch_remaining
+        pytz.timezone('Asia/Shanghai')  # 东八区
+        t = datetime.fromtimestamp(int(time.time()) + time_remaining,
+                                   pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')
+        log = log_temp.format(epoch_time, epoch_remaining, t)
+        if print_log:
+            oprint(log)
+        self.start_time = time.time()
+        return epoch_remaining, t
+
+
+class Stopwatch:
+    def __init__(self):
+        self.start_time = time.time()
+
+    def reset(self, format_type='{:>5.1f}s'):
+        timer_end = time.time()
+        duration = timer_end - self.start_time
+        self.start_time = timer_end
+        return format_type.format(duration)
+
+    def pinch(self, format_type='{:>5.1f}s'):
+        return format_type.format(time.time() - self.start_time)
+
+
+def getHumanSize(in_size):
+    unit = 'B'
+    if in_size >= 1024:
+        in_size /= 1024
+        unit = 'K'
+    if in_size >= 1024:
+        in_size /= 1024
+        unit = 'M'
+    if in_size >= 1024:
+        in_size /= 1024
+        unit = 'G'
+    return f'{round(in_size, 2)} {unit}'
+
+
+if __name__ == '__main__':
+    print(getHumanSize(1023))

jacksung/utils/web.py

@@ -0,0 +1,63 @@
+import os
+import platform
+# 从selenium导入webdriver
+from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException
+from selenium.webdriver.chrome.service import Service
+from jacksung.utils.log import oprint as print
+from selenium.webdriver.common.by import By
+
+
+def make_driver(url=None, is_headless=False, tmp_path=None, download_dir=None, options=webdriver.ChromeOptions()):
+    options.add_argument("--disable-blink-features=AutomationControlled")
+    if tmp_path:
+        options.add_argument("crash-dumps-dir=" + tmp_path)
+    options.add_argument("--no-sandbox")
+    # options.add_argument("--auto-open-devtools-for-tabs")
+    options.add_argument('--disable-dev-shm-usage')
+    options.add_argument("--disable-web-security")  # 禁用Web安全
+    options.add_argument("--allow-running-insecure-content")  # 允许不安全的内容
+    options.add_argument('--user-agent=Mozilla/5.0')
+    options.add_argument('--ignore-ssl-errors=yes')
+    options.add_argument('--allow-insecure-localhost')
+    options.add_argument('--ignore-certificate-errors')
+    options.add_argument(
+        '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36')  # 设置User-Agent
+    options.add_argument("--lang=zh-CN")  # 将语言设置为简体中文，英文为en-US
+    options.add_experimental_option("detach", True)
+    if download_dir:
+        options.add_experimental_option("prefs", {
+            "download.default_directory": download_dir,
+            "download.prompt_for_download": False,
+            "safebrowsing.enabled": True,  # 允许“不安全”文件自动下载
+            "safebrowsing.disable_download_protection": True,  # 禁用“可能有害文件”的拦截
+            "intl.accept_languages": "zh-CN,zh"
+            # "download.directory_upgrade": True,
+        })
+    options.set_capability('pageLoadStrategy', 'none')
+    options.set_capability("unhandledPromptBehavior", "accept")
+    if is_headless:
+        options.add_argument('--headless')  # 浏览器隐式启动
+    # driver_path = os.path.expanduser("~/chrome/chromedriver.exe")
+    print('driver is going to init!')
+    if platform.system().lower() == 'windows':
+        driver_path = None
+        # driver_path = os.path.expanduser("~/chrome/chromedriver.exe")
+    else:
+        driver_path = os.path.expanduser("~/chrome/chromedriver")
+    # driver = uc.Chrome(service=Service(driver_path) if driver_path else None, options=options)
+    driver = webdriver.Chrome(service=Service(driver_path) if driver_path else None, options=options)
+    # driver.maximize_window()
+    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
+        "source": """
+            Object.defineProperty(navigator, 'webdriver', {
+              get: () => undefined
+            })
+        """
+    })
+    driver.implicitly_wait(10)
+    driver.set_page_load_timeout(10)
+    if url is not None:
+        print(f'请求地址：{url}')
+        driver.get(url)
+    return driver