pyxllib 0.3.102__tar.gz → 0.3.104__tar.gz

{pyxllib-0.3.102/pyxllib.egg-info → pyxllib-0.3.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.102
+Version: 0.3.104
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/data/echarts.py

@@ -58,7 +58,8 @@ class XlChart(Chart):
                 kwargs['label']['formatter'] = fmt
 
         self._append_color(color)
-        self._append_legend(name, is_selected=True)
+        # self._append_legend(name, is_selected=True)
+        self._append_legend(name)
 
         self.options.get('series').append(
             {
@@ -192,3 +193,4 @@ def draw_pareto_chart(data, accuracy=0.1, *, title='帕累托累积权重', valu
 if __name__ == '__main__':
     with TicToc(__name__):
         pass
+

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/data/pglib.py

@@ -49,6 +49,10 @@ from pyxllib.data.sqlite import SqlBase
 
 class Connection(psycopg.Connection, SqlBase):
 
+    def __init__(self, *args, **kwargs):
+        psycopg.Connection.__init__(self, *args, **kwargs)
+        SqlBase.__init__(self, *args, **kwargs)
+
     def __1_库(self):
         pass
 
@@ -140,7 +144,7 @@ class Connection(psycopg.Connection, SqlBase):
     def __5_增删改查(self):
         pass
 
-    def insert_row(self, table_name, cols, *, on_conflict='DO NOTHING', commit=True):
+    def insert_row(self, table_name, cols, *, on_conflict='DO NOTHING', commit=False):
         """ 【增】插入新数据
 
         :param dict cols: 用字典表示的要插入的值
@@ -168,11 +172,9 @@ class Connection(psycopg.Connection, SqlBase):
             on_conflict = f'ON CONFLICT {on_conflict}'
         query += f' {on_conflict}'
 
-        self.execute(query, params)
-        if commit:
-            self.commit()
+        self.commit_base(commit, query, params)
 
-    def insert_rows(self, table_name, keys, ls, *, on_conflict='DO NOTHING', commit=True):
+    def insert_rows(self, table_name, keys, ls, *, on_conflict='DO NOTHING', commit=False):
         """ 【增】插入新数据
 
         :param str keys: 要插入的字段名，一个字符串，逗号,隔开属性值
@@ -194,9 +196,7 @@ class Connection(psycopg.Connection, SqlBase):
             on_conflict = f'ON CONFLICT {on_conflict}'
         query += f' {on_conflict}'
 
-        self.execute(query, params)
-        if commit:
-            self.commit()
+        self.commit_base(commit, query, params)
 
 
 """
@@ -392,7 +392,7 @@ class XlprDb(Connection):
         self.insert_row('xlapi', {'input': input, 'output': output,
                                   'elapse_ms': elapse_ms, 'update_time': utc_timestamp(8)},
                         on_conflict=on_conflict)
-
+        self.commit()
         return self.execute('SELECT id FROM xlapi WHERE input=%s', (input,)).fetchone()[0]
 
     def insert_row2xlserver(self, request, xlapi_id=0, **kwargs):
@@ -402,7 +402,7 @@ class XlprDb(Connection):
               'xlapi_id': xlapi_id}
         kw.update(kwargs)
         print(kw)  # 监控谁在用api
-        self.insert_row('xlserver', kw)
+        self.insert_row('xlserver', kw, commit=True)
 
     def __3_host_trace相关可视化(self):
         """ TODO dbview 改名 host_trace """
@@ -442,7 +442,8 @@ class XlprDb(Connection):
 
             if status:
                 self.insert_row('host_trace',
-                                {'host_name': host_name, 'status': status, 'update_time': utc_timestamp(8)})
+                                {'host_name': host_name, 'status': status, 'update_time': utc_timestamp(8)},
+                                commit=True)
             print()
 
     def _get_host_trace_total(self, mode, title, yaxis_name, date_trunc, recent, host_attr):

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/data/sqlite.py

@@ -8,6 +8,7 @@ import json
 import re
 import sqlite3
 import warnings
+from collections import defaultdict
 
 import pandas as pd
 
@@ -17,6 +18,9 @@ warnings.filterwarnings('ignore', message="pandas only support SQLAlchemy connec
 class SqlBase:
     """ Sql语法通用的功能 """
 
+    def __init__(self, *args, **kwargs):
+        self._commit_cache = defaultdict(list)
+
     def __1_库(self):
         pass
 
@@ -84,6 +88,12 @@ class SqlBase:
             cols = ','.join(map(str, cols))
         self.execute(f'CREATE INDEX {index_name} ON {table_name}({cols})')
 
+    def create_index2(self, table_name, cols):
+        """ 创建一个简单的索引，索引名字自动生成 """
+        if not isinstance(cols, str):
+            cols = ','.join(map(str, cols))
+        self.execute(f'CREATE INDEX idx_{table_name}_{cols.replace(",", "_")} ON {table_name}({cols})')
+
     def keep_top_n_rows(self, table_name, num, col_name='id'):
         """ 只保留一小部分数据，常用来做lite、demo数据示例文件
 
@@ -155,6 +165,32 @@ class SqlBase:
     def __5_增删改查(self):
         pass
 
+    def commit_base(self, commit_type, query, params=None):
+        """
+        :param commit_type:
+            -1，先真正缓存在本地
+            False，传统的事务机制，虽然不会更新数据，但每一条依然会连接数据库，其实速度回挺慢的
+            True，传统的事务机制，但每条都作为独立事务，直接更新了
+        """
+        if commit_type == -1:
+            self._commit_cache[query].append(params)
+        elif commit_type is False:
+            self.execute(query, params)
+        elif commit_type is True:
+            self.execute(query, params)
+            self.commit()
+
+    def commit_all(self):
+        if not self._commit_cache:
+            self.commit()
+            return
+
+        for query, params in self._commit_cache.items():
+            cur = self.cursor()
+            cur.executemany(query, params)
+            cur.close()
+            self.commit()
+
     def update_row(self, table_name, cols, where, *, commit=False):
         """ 【改】更新数据
 
@@ -170,9 +206,9 @@ class SqlBase:
         kvs = ','.join([f'{k}=%s' for k in cols.keys()])
         ops = ' AND '.join([f'{k}=%s' for k in where.keys()])
         vals = list(cols.values()) + list(where.values())
-        self.execute(f'UPDATE {table_name} SET {kvs} WHERE {ops}', self.cvt_types(vals))
-        if commit:
-            self.commit()
+        self.commit_base(commit,
+                         f'UPDATE {table_name} SET {kvs} WHERE {ops}',
+                         self.cvt_types(vals))
 
     def select_col(self, table_name, col):
         """ 获得一列数据，常使用的功能，所以做了一个封装

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/ext/JLineViewer.py

@@ -390,9 +390,22 @@ class JLineViewer(QMainWindow):
         if parent is None:
             parent = QStandardItemModel()
             parent.setHorizontalHeaderLabels(['Key', 'Value'])
+
+        # if isinstance(data, dict):
+        #     for key, value in data.items():
+        #         self.dataToModel(key, value, parent)
+
+        # 判断数据类型，并相应处理
         if isinstance(data, dict):
             for key, value in data.items():
                 self.dataToModel(key, value, parent)
+        elif isinstance(data, list):
+            # 处理列表：创建一个无key的父项，将列表元素作为子项添加
+            self.dataToModel("List", data, parent)
+        else:
+            # 处理基本数据类型：创建一个单独的条目
+            self.dataToModel("Value", data, parent)
+
         return parent
 
     def dataToModel(self, key, value, parent):

pyxllib-0.3.104/pyxllib/ext/drissionlib.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Author : 陈坤泽
+# @Email  : 877362867@qq.com
+# @Date   : 2024/04/12
+
+from pyxllib.prog.pupil import check_install_package
+
+check_install_package('DrissionPage')
+
+import re
+from urllib.parse import unquote
+
+from DrissionPage import ChromiumPage
+
+
+def get_latest_not_dev_tab(page=None):
+    """ 开发工具本身也会算一个tab，这个函数返回最新的一个不是开发工具的tab """
+    if page is None:
+        page = ChromiumPage()
+    tabs = page.get_tabs()
+    for tab in tabs:
+        if tab.url.startswith('devtools://'):
+            continue
+        return tab
+
+
+def set_input_text(input_ele, text):
+    """ 因为input输入框可能原本就自带了内容，为了不重复输入，先清空再输入 """
+    input_ele.clear()
+    input_ele.input(text)
+
+
+def get_download_files(page=None):
+    """ 获取下载列表
+
+    :param search_name: 搜索文件名，输入该参数时，只会从上往下找到第一个匹配的文件
+        否则返回一个list结构，存储下载清单里的文件
+    :return:
+
+    todo 默认应该显示是不全的，有时间可以考虑往下滑动继续检索的功能
+    """
+    if page is None:
+        page = ChromiumPage()
+
+    files = []
+    page2 = page.new_tab('chrome://downloads/')
+    items = page2('tag:downloads-manager', timeout=1).sr('#mainContainer')('#downloadsList').eles('tag:downloads-item')
+    for item in items:
+        loc = unquote(item.sr('tag:img').attr('src'))
+        file = re.search(r'path=(.+?)(&scale=\d+x)?$', loc).group(1)
+
+        files.append({
+            'file': file,
+            'url': unquote(item.sr('#url').attr('href'))
+        })
+
+    page2.close()
+
+    return files
+
+
+def search_download_file(file_name):
+    files = get_download_files()
+    for file in files:
+        if file_name in file['file']:
+            return file

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/ext/kq5034lib.py

@@ -45,26 +45,19 @@ class Xiaoetong:
     """
 
     def __init__(self):
-        self.app_id = ''
-        self.client_id = ''
-        self.secret_key = ''
         self.token = ''
 
     def login(self, app_id, client_id, secret_key):
         """ 登录，获取token
         """
-        self.app_id = app_id
-        self.client_id = client_id
-        self.secret_key = secret_key
-
         # 启用缓存
-        requests_cache.install_cache('access_token_cache', expire_after=600)  # 设置缓存过期时间xx（单位：秒）
+        # requests_cache.install_cache('access_token_cache', expire_after=None)  # 设置缓存过期时间xx（单位：秒）
         # 接口地址
         url = "https://api.xiaoe-tech.com/token"
         params = {
-            "app_id": self.app_id,
-            "client_id": self.client_id,
-            "secret_key": self.secret_key,
+            "app_id": app_id,
+            "client_id": client_id,
+            "secret_key": secret_key,
             "grant_type": "client_credential"
         }
         response = requests.get(url, params=params)
@@ -77,7 +70,8 @@ class Xiaoetong:
                 raise Exception("Error getting access token: {}".format(result['msg']))
         else:
             raise Exception("HTTP request failed with status code {}".format(response.status_code))
-
+        # return result['data']['access_token']
+    
     def get_alive_user_list(self, resource_id, page_size=100):
         """ 获取直播间用户
         """
@@ -966,7 +960,8 @@ class 网课考勤2(网课考勤):
         self.xe.login(self.app_id,
                       self.client_id,
                       self.secret_key)  # 获取了token
-
+        
+        return self.xe.token
     # 依据课程链接，获取资源id（与课次）
     def 获取课次与资源id(self):
         课程链接 = self.课程链接[1:]
@@ -1074,7 +1069,7 @@ class 网课考勤2(网课考勤):
             self.获取打卡参与用户(activity_id, self.root / "数据表" / path)
 
 
-class KqDb(Connection):
+class OldKqDb(Connection):
     """ 五一身心行修考勤工具 """
 
     def __init__(self, dbfile='5034.db', wbpath='考勤.xlsx', *args, **kwargs):

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/ext/robustprocfile.py

@@ -107,7 +107,7 @@ def process_dir_base(file_func, dir_path, correct_tag, error_tag,
 
     files = list(dir_path.rglob(pattern))
     for idx, file in enumerate(files, start=1):
-        tprint(f'处理第{idx}/{len(files)}个文件: {file.name} ==> {correct_tag}')
+        tprint(f'处理完第{idx}/{len(files)}个文件: {file.name} ==> {correct_tag}')
         process_file_base(file_func, file, correct_tag, error_tag,
                           correct_dir=correct_dir, error_dir=error_dir, reset=reset,
                           **kwargs)
@@ -281,7 +281,14 @@ def process_single_file(root,
                         if_exists='skip',
                         group_func=None,
                         batch_size=2000,
-                        debug=False):
+                        debug=False,
+                        show_tqdm=False):
+    def finished():
+        if total:
+            tprint(f'处理第{cur_idx}/{total}个文件：{infile.name} ==> {tag}')
+        else:
+            tprint(f'处理文件：{infile.name} ==> {tag}')
+
     # 1 缓存路径
     srcdir = XlPath(root)
     infile = XlPath.init(infile, root=srcdir)
@@ -309,6 +316,7 @@ def process_single_file(root,
 
     if check_files:
         if if_exists == 'skip':
+            finished()
             return
         elif if_exists == 'overwrite':
             for f in check_files:
@@ -316,6 +324,7 @@ def process_single_file(root,
         elif if_exists == 'error':
             raise FileExistsError(f'目标文件已存在：{check_files}')
         else:
+            finished()
             return
 
     # 3 处理数据
@@ -326,7 +335,7 @@ def process_single_file(root,
     if debug:  # 如果开启调试模式，则单独分错误文件
         errcgf = dstcgf
 
-    for line in tqdm(infile.yield_line(), disable=total):
+    for line in tqdm(infile.yield_line(), desc=f'{infile.name} ==> {tag}', disable=not show_tqdm):
         # todo 出错的数据，应该添加错误信息，也存成一个新的jsonl格式
         row = row_func(line)
         if row and (not isinstance(row, dict) or row.get('status', 'ok') == 'ok'):
@@ -339,10 +348,7 @@ def process_single_file(root,
 
     dstcgf.save_all()
     errcgf.save_all()
-    if total:
-        tprint(f'处理第{cur_idx}/{total}个文件：{infile.name} ==> {tag}')
-    else:
-        tprint(f'处理文件：{infile.name} ==> {tag}')
+    finished()
 
 
 class StructureAnalyzer:
@@ -418,6 +424,30 @@ class StructureAnalyzer:
 
         return res
 
+    @classmethod
+    def compare_keys(cls, items):
+        """ 对从 get_items_structures 获得的数据，对比structure的覆盖情况 """
+        import pandas as pd
+        from pyxllib.algo.stat import custom_fillna
+
+        ls = []
+        for x in items:
+            row = {}
+            row['__depth'] = x['depth']
+            row['__count'] = x['count']
+            if isinstance(x['structure'], dict):
+                row.update(x['structure'])
+            elif isinstance(x['structure'], list):
+                row.update({f'[{k}]': v for k, v in enumerate(x['structure'])})
+            ls.append(row)
+
+        df = pd.DataFrame.from_dict(ls)
+        # 筛选df，depth中没有'-'
+        df = df[df['__depth'].apply(lambda x: '-' not in x)]
+        df = custom_fillna(df).T
+
+        return df
+
 
 def process_batch_files(srcdir,
                         dsttag,
@@ -460,7 +490,8 @@ def process_batch_files(srcdir,
                                             if_exists=if_exists,
                                             group_func=group_func,
                                             batch_size=batch_size,
-                                            debug=debug)
+                                            debug=debug,
+                                            show_tqdm=processes_num == 1)
         tasks.append(task)
 
     Parallel(n_jobs=processes_num, backend=backend)(tasks)

{pyxllib-0.3.102 → pyxllib-0.3.104}/pyxllib/file/specialist/filelib.py

@@ -1140,12 +1140,12 @@ class XlPath(type(pathlib.Path())):
         f = open(self, 'r', encoding=encoding)
         return chunked(f, batch_size)
 
-    def write_text(self, data, encoding='utf8', errors=None, newline=None):
-        with open(self, 'w', encoding=encoding, errors=errors, newline=newline) as f:
+    def write_text(self, data, encoding='utf8', mode='w', errors=None, newline=None):
+        with open(self, mode, encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 
-    def write_text_unix(self, data, encoding='utf8', errors=None, newline='\n'):
-        with open(self, 'w', encoding=encoding, errors=errors, newline=newline) as f:
+    def write_text_unix(self, data, encoding='utf8', mode='w', errors=None, newline='\n'):
+        with open(self, mode, encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 
     def read_pkl(self):
@@ -1214,10 +1214,15 @@ class XlPath(type(pathlib.Path())):
         else:
             return data
 
-    def write_jsonl(self, list_data, ensure_ascii=False, default=None):
+    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w'):
         """ 由于这种格式主要是跟商汤这边对接，就尽量跟它们的格式进行兼容 """
         content = '\n'.join([json.dumps(x, ensure_ascii=ensure_ascii, default=default) for x in list_data])
-        self.write_text_unix(content + '\n')
+        self.write_text_unix(content + '\n', mode=mode)
+
+    def add_json_line(self, data, ensure_ascii=False, default=None, mode='a'):
+        """ 在文件末尾添加一行JSON数据 """
+        content = json.dumps(data, ensure_ascii=ensure_ascii, default=default)
+        self.write_text_unix(content + '\n', mode=mode)
 
     def read_csv(self, encoding='utf8', *, errors='strict', return_mode: bool = False,
                  delimiter=',', quotechar='"', **kwargs):
@@ -1437,10 +1442,6 @@ class XlPath(type(pathlib.Path())):
                 shutil.copytree(self, dst)
 
     def move(self, dst, if_exists=None):
-        return self.rename2(dst, if_exists)
-
-    def rename2(self, dst, if_exists=None):
-        """ 相比原版的rename，搞了更多骚操作，但性能也会略微下降，所以重写一个功能名 """
         if not self.exists():
             return self
 
@@ -1456,6 +1457,34 @@ class XlPath(type(pathlib.Path())):
             self.rename(dst)
         return dst
 
+    def rename2(self, new_name, if_exists=None):
+        """ 相比原版的rename，搞了更多骚操作，但性能也会略微下降，所以重写一个功能名
+
+        240416周二12:49，这个接口将真的只做重命名，不做移动！所以将会不再支持dst中出现"/"路径配置
+        """
+        if not self.exists():
+            return self
+
+        if '/' in new_name:
+            raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"/"')
+        elif '\\' in new_name:
+            raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"\\"')
+
+        if self.name == new_name:  # 没有修改名称，跟原来相同
+            return self
+
+        dst = self.parent / new_name
+        if self == dst:
+            # 同一个文件，可能是调整了大小写名称
+            if self.as_posix() != dst.as_posix():
+                tmp = self.tempfile(dir=self.parent)  # self不一定是file，也可能是dir，但这个名称通用
+                self.rename(tmp)
+                self.delete()
+                tmp.rename(dst)
+        elif dst.exist_preprcs(if_exists):
+            self.rename(dst)
+        return dst
+
     def delete(self):
         if self.is_file():
             os.remove(self)
@@ -1621,6 +1650,10 @@ class XlPath(type(pathlib.Path())):
 
             ext = '.' + t.extension
             ext0 = file_path.suffix
+            if ext == ext0:
+                continue
+            elif ext == '.xls' and ext0 == '.et':
+                continue
 
             if ext0 in ('.docx', '.xlsx', '.pptx', '.xlsm'):
                 ext0 = '.zip'