pyxllib 0.3.103__tar.gz → 0.3.105__tar.gz

{pyxllib-0.3.103/pyxllib.egg-info → pyxllib-0.3.105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.103
+Version: 0.3.105
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/data/echarts.py

@@ -58,7 +58,8 @@ class XlChart(Chart):
                 kwargs['label']['formatter'] = fmt
 
         self._append_color(color)
-        self._append_legend(name, is_selected=True)
+        # self._append_legend(name, is_selected=True)
+        self._append_legend(name)
 
         self.options.get('series').append(
             {
@@ -192,3 +193,4 @@ def draw_pareto_chart(data, accuracy=0.1, *, title='帕累托累积权重', valu
 if __name__ == '__main__':
     with TicToc(__name__):
         pass
+

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/data/pglib.py

@@ -34,6 +34,7 @@ import json
 import json
 import textwrap
 import datetime
+import re
 
 from tqdm import tqdm
 
@@ -49,6 +50,10 @@ from pyxllib.data.sqlite import SqlBase
 
 class Connection(psycopg.Connection, SqlBase):
 
+    def __init__(self, *args, **kwargs):
+        psycopg.Connection.__init__(self, *args, **kwargs)
+        SqlBase.__init__(self, *args, **kwargs)
+
     def __1_库(self):
         pass
 
@@ -95,6 +100,37 @@ class Connection(psycopg.Connection, SqlBase):
         self.execute(f"COMMENT ON COLUMN {table_name}.{col_name} IS $zyzf${comment}$zyzf$")
         self.commit()
 
+    def reset_table_item_id(self, table_name, item_id_name=None, counter_name=None):
+        """ 重置表格的数据的id值，也重置计数器
+
+        :param item_id_name: 表格的自增id字段名，有一套我自己风格的自动推算算法
+        :param counter_name: 计数器字段名，如果有的话，也会重置
+        """
+        # 1 重置数据的编号
+        if item_id_name is None:
+            m = re.match(r'(.+?)(_table)?$', table_name)
+            item_id_name = m.group(1) + '_id'
+
+        sql = f"""WITH cte AS (
+    SELECT {item_id_name}, ROW_NUMBER() OVER (ORDER BY {item_id_name}) AS new_{item_id_name}
+    FROM {table_name}
+)
+UPDATE {table_name}
+SET {item_id_name} = cte.new_{item_id_name}
+FROM cte
+WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
+        self.execute(sql)  # todo 这种sql写法好像偶尔有bug会出问题
+        self.commit()
+
+        # 2 重置计数器
+        if counter_name is None:
+            counter_name = f'{table_name}_{item_id_name}_seq'
+
+        # 找到目前最大的id值
+        max_id = self.exec2one(f'SELECT MAX({item_id_name}) FROM {table_name}')
+        self.execute(f'ALTER SEQUENCE {counter_name} RESTART WITH {max_id + 1}')
+        self.commit()
+
     def __3_execute(self):
         pass
 
@@ -140,7 +176,7 @@ class Connection(psycopg.Connection, SqlBase):
     def __5_增删改查(self):
         pass
 
-    def insert_row(self, table_name, cols, *, on_conflict='DO NOTHING', commit=True):
+    def insert_row(self, table_name, cols, *, on_conflict='DO NOTHING', commit=False):
         """ 【增】插入新数据
 
         :param dict cols: 用字典表示的要插入的值
@@ -168,11 +204,9 @@ class Connection(psycopg.Connection, SqlBase):
             on_conflict = f'ON CONFLICT {on_conflict}'
         query += f' {on_conflict}'
 
-        self.execute(query, params)
-        if commit:
-            self.commit()
+        self.commit_base(commit, query, params)
 
-    def insert_rows(self, table_name, keys, ls, *, on_conflict='DO NOTHING', commit=True):
+    def insert_rows(self, table_name, keys, ls, *, on_conflict='DO NOTHING', commit=False):
         """ 【增】插入新数据
 
         :param str keys: 要插入的字段名，一个字符串，逗号,隔开属性值
@@ -194,9 +228,7 @@ class Connection(psycopg.Connection, SqlBase):
             on_conflict = f'ON CONFLICT {on_conflict}'
         query += f' {on_conflict}'
 
-        self.execute(query, params)
-        if commit:
-            self.commit()
+        self.commit_base(commit, query, params)
 
 
 """
@@ -392,7 +424,7 @@ class XlprDb(Connection):
         self.insert_row('xlapi', {'input': input, 'output': output,
                                   'elapse_ms': elapse_ms, 'update_time': utc_timestamp(8)},
                         on_conflict=on_conflict)
-
+        self.commit()
         return self.execute('SELECT id FROM xlapi WHERE input=%s', (input,)).fetchone()[0]
 
     def insert_row2xlserver(self, request, xlapi_id=0, **kwargs):
@@ -402,7 +434,7 @@ class XlprDb(Connection):
               'xlapi_id': xlapi_id}
         kw.update(kwargs)
         print(kw)  # 监控谁在用api
-        self.insert_row('xlserver', kw)
+        self.insert_row('xlserver', kw, commit=True)
 
     def __3_host_trace相关可视化(self):
         """ TODO dbview 改名 host_trace """
@@ -442,7 +474,8 @@ class XlprDb(Connection):
 
             if status:
                 self.insert_row('host_trace',
-                                {'host_name': host_name, 'status': status, 'update_time': utc_timestamp(8)})
+                                {'host_name': host_name, 'status': status, 'update_time': utc_timestamp(8)},
+                                commit=True)
             print()
 
     def _get_host_trace_total(self, mode, title, yaxis_name, date_trunc, recent, host_attr):

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/data/sqlite.py

@@ -8,15 +8,22 @@ import json
 import re
 import sqlite3
 import warnings
+from collections import defaultdict
 
 import pandas as pd
 
+# 旧版的pandas警告
 warnings.filterwarnings('ignore', message="pandas only support SQLAlchemy connectable")
+# 新版的pandas警告多了个's‘
+warnings.filterwarnings('ignore', message="pandas only supports SQLAlchemy connectable")
 
 
 class SqlBase:
     """ Sql语法通用的功能 """
 
+    def __init__(self, *args, **kwargs):
+        self._commit_cache = defaultdict(list)
+
     def __1_库(self):
         pass
 
@@ -84,6 +91,12 @@ class SqlBase:
             cols = ','.join(map(str, cols))
         self.execute(f'CREATE INDEX {index_name} ON {table_name}({cols})')
 
+    def create_index2(self, table_name, cols):
+        """ 创建一个简单的索引，索引名字自动生成 """
+        if not isinstance(cols, str):
+            cols = ','.join(map(str, cols))
+        self.execute(f'CREATE INDEX idx_{table_name}_{cols.replace(",", "_")} ON {table_name}({cols})')
+
     def keep_top_n_rows(self, table_name, num, col_name='id'):
         """ 只保留一小部分数据，常用来做lite、demo数据示例文件
 
@@ -155,6 +168,34 @@ class SqlBase:
     def __5_增删改查(self):
         pass
 
+    def commit_base(self, commit_type, query, params=None):
+        """
+        :param commit_type:
+            -1，先真正缓存在本地
+            False，传统的事务机制，虽然不会更新数据，但每一条依然会连接数据库，其实速度回挺慢的
+            True，传统的事务机制，但每条都作为独立事务，直接更新了
+        """
+        if commit_type == -1:
+            self._commit_cache[query].append(params)
+        elif commit_type is False:
+            self.execute(query, params)
+        elif commit_type is True:
+            self.execute(query, params)
+            self.commit()
+
+    def commit_all(self):
+        if not self._commit_cache:
+            self.commit()
+            return
+
+        for query, params in self._commit_cache.items():
+            cur = self.cursor()
+            cur.executemany(query, params)
+            cur.close()
+            self.commit()
+
+        self._commit_cache = defaultdict(list)
+
     def update_row(self, table_name, cols, where, *, commit=False):
         """ 【改】更新数据
 
@@ -170,9 +211,9 @@ class SqlBase:
         kvs = ','.join([f'{k}=%s' for k in cols.keys()])
         ops = ' AND '.join([f'{k}=%s' for k in where.keys()])
         vals = list(cols.values()) + list(where.values())
-        self.execute(f'UPDATE {table_name} SET {kvs} WHERE {ops}', self.cvt_types(vals))
-        if commit:
-            self.commit()
+        self.commit_base(commit,
+                         f'UPDATE {table_name} SET {kvs} WHERE {ops}',
+                         self.cvt_types(vals))
 
     def select_col(self, table_name, col):
         """ 获得一列数据，常使用的功能，所以做了一个封装

pyxllib-0.3.105/pyxllib/ext/drissionlib.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Author : 陈坤泽
+# @Email  : 877362867@qq.com
+# @Date   : 2024/04/12
+
+from pyxllib.prog.pupil import check_install_package
+
+check_install_package('DrissionPage')
+
+import re
+from urllib.parse import unquote
+
+import DrissionPage
+from DrissionPage import ChromiumPage
+from DrissionPage._base.base import BasePage, BaseElement
+
+from pyxllib.prog.pupil import inject_members
+
+
+def get_latest_not_dev_tab(page=None):
+    """ 开发工具本身也会算一个tab，这个函数返回最新的一个不是开发工具的tab """
+    if page is None:
+        page = ChromiumPage()
+    tabs = page.get_tabs()
+    for tab in tabs:
+        if tab.url.startswith('devtools://'):
+            continue
+        return tab
+
+
+def set_input_text(input_ele, text):
+    """ 因为input输入框可能原本就自带了内容，为了不重复输入，先清空再输入 """
+    input_ele.clear()
+    input_ele.input(text)
+
+
+def search_download_file(file_name):
+    file_name = file_name.replace(':', '_')
+    files = ChromiumPage().get_download_files()
+    for file in files:
+        if file_name in file['file']:  # 正常情况下的匹配
+            return file
+        if file_name in file['file'].replace('+', ' '):  # 但有时候'+'好像有点特别
+            return file
+
+
+class XlBasePage(BasePage):
+    def get2(self, url, show_errmsg=False, retry=None, interval=None):
+        """
+        240418周四21:57，DrissionPage-4.0.4.21 官方自带page.get，有时候会有bug，不会实际刷新url，这里加个代码进行fix
+        """
+        old_url = self.url
+        if old_url == url:
+            # page.refresh()
+            return
+
+        ele = self.active_ele
+        self.get(url, show_errmsg=show_errmsg, retry=retry, interval=interval)
+        try:  # 如果新页面获取成功，理论上旧的ele会失效
+            ele
+            self.refresh()  # 如果不报错，这里网站要强制更新
+            return self
+        except DrissionPage.errors.ElementLostError:
+            return self
+
+    def get_download_files(self):
+        """ 获取下载列表
+
+        :param search_name: 搜索文件名，输入该参数时，只会从上往下找到第一个匹配的文件
+            否则返回一个list结构，存储下载清单里的文件
+        :return:
+
+        todo 默认应该显示是不全的，有时间可以考虑往下滑动继续检索的功能
+        """
+        files = []
+        page2 = self.new_tab('chrome://downloads/')
+        items = page2('tag:downloads-manager', timeout=1).sr('#mainContainer')('#downloadsList').eles(
+            'tag:downloads-item')
+        for item in items:
+            loc = unquote(item.sr('tag:img').attr('src').replace('+', ' '))
+            file = re.search(r'path=(.+?)(&scale=\d+x)?$', loc).group(1)
+
+            files.append({
+                'file': file,
+                'url': unquote(item.sr('#url').attr('href'))
+            })
+
+        page2.close()
+
+        return files
+
+
+inject_members(XlBasePage, BasePage)
+
+
+class XlBaseElement(BaseElement):
+    def input2(self, vals, clear=False, by_js=False):
+        self.clear()
+        self.input(vals)
+
+
+inject_members(XlBaseElement, BaseElement)

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/ext/kq5034lib.py

@@ -45,26 +45,19 @@ class Xiaoetong:
     """
 
     def __init__(self):
-        self.app_id = ''
-        self.client_id = ''
-        self.secret_key = ''
         self.token = ''
 
     def login(self, app_id, client_id, secret_key):
         """ 登录，获取token
         """
-        self.app_id = app_id
-        self.client_id = client_id
-        self.secret_key = secret_key
-
         # 启用缓存
-        requests_cache.install_cache('access_token_cache', expire_after=600)  # 设置缓存过期时间xx（单位：秒）
+        # requests_cache.install_cache('access_token_cache', expire_after=None)  # 设置缓存过期时间xx（单位：秒）
         # 接口地址
         url = "https://api.xiaoe-tech.com/token"
         params = {
-            "app_id": self.app_id,
-            "client_id": self.client_id,
-            "secret_key": self.secret_key,
+            "app_id": app_id,
+            "client_id": client_id,
+            "secret_key": secret_key,
             "grant_type": "client_credential"
         }
         response = requests.get(url, params=params)
@@ -77,7 +70,8 @@ class Xiaoetong:
                 raise Exception("Error getting access token: {}".format(result['msg']))
         else:
             raise Exception("HTTP request failed with status code {}".format(response.status_code))
-
+        # return result['data']['access_token']
+    
     def get_alive_user_list(self, resource_id, page_size=100):
         """ 获取直播间用户
         """
@@ -966,7 +960,8 @@ class 网课考勤2(网课考勤):
         self.xe.login(self.app_id,
                       self.client_id,
                       self.secret_key)  # 获取了token
-
+        
+        return self.xe.token
     # 依据课程链接，获取资源id（与课次）
     def 获取课次与资源id(self):
         课程链接 = self.课程链接[1:]
@@ -1074,7 +1069,7 @@ class 网课考勤2(网课考勤):
             self.获取打卡参与用户(activity_id, self.root / "数据表" / path)
 
 
-class KqDb(Connection):
+class OldKqDb(Connection):
     """ 五一身心行修考勤工具 """
 
     def __init__(self, dbfile='5034.db', wbpath='考勤.xlsx', *args, **kwargs):

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/ext/robustprocfile.py

@@ -107,7 +107,7 @@ def process_dir_base(file_func, dir_path, correct_tag, error_tag,
 
     files = list(dir_path.rglob(pattern))
     for idx, file in enumerate(files, start=1):
-        tprint(f'处理第{idx}/{len(files)}个文件: {file.name} ==> {correct_tag}')
+        tprint(f'处理完第{idx}/{len(files)}个文件: {file.name} ==> {correct_tag}')
         process_file_base(file_func, file, correct_tag, error_tag,
                           correct_dir=correct_dir, error_dir=error_dir, reset=reset,
                           **kwargs)

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/ext/utools.py

@@ -130,8 +130,8 @@ class UtoolsBase:
         except OSError:  # 忽略错误的扩展名
             f1 = XlPath.init('left', XlPath.tempdir())
             f2 = XlPath.init('right', XlPath.tempdir())
-        f1.write_text(suffix)
-        f2.write_text(self.cmds['ClipText'])
+        f1.write_text(self.cmds['ClipText'])
+        f2.write_text(self.cmds['ClipText'] + '\n')
         bcompare(f1, f2, wait=False)
 
 

{pyxllib-0.3.103 → pyxllib-0.3.105}/pyxllib/file/specialist/filelib.py

@@ -1140,12 +1140,12 @@ class XlPath(type(pathlib.Path())):
         f = open(self, 'r', encoding=encoding)
         return chunked(f, batch_size)
 
-    def write_text(self, data, encoding='utf8', errors=None, newline=None):
-        with open(self, 'w', encoding=encoding, errors=errors, newline=newline) as f:
+    def write_text(self, data, encoding='utf8', mode='w', errors=None, newline=None):
+        with open(self, mode, encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 
-    def write_text_unix(self, data, encoding='utf8', errors=None, newline='\n'):
-        with open(self, 'w', encoding=encoding, errors=errors, newline=newline) as f:
+    def write_text_unix(self, data, encoding='utf8', mode='w', errors=None, newline='\n'):
+        with open(self, mode, encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 
     def read_pkl(self):
@@ -1214,10 +1214,15 @@ class XlPath(type(pathlib.Path())):
         else:
             return data
 
-    def write_jsonl(self, list_data, ensure_ascii=False, default=None):
+    def write_jsonl(self, list_data, ensure_ascii=False, default=None, mode='w'):
         """ 由于这种格式主要是跟商汤这边对接，就尽量跟它们的格式进行兼容 """
         content = '\n'.join([json.dumps(x, ensure_ascii=ensure_ascii, default=default) for x in list_data])
-        self.write_text_unix(content + '\n')
+        self.write_text_unix(content + '\n', mode=mode)
+
+    def add_json_line(self, data, ensure_ascii=False, default=None, mode='a'):
+        """ 在文件末尾添加一行JSON数据 """
+        content = json.dumps(data, ensure_ascii=ensure_ascii, default=default)
+        self.write_text_unix(content + '\n', mode=mode)
 
     def read_csv(self, encoding='utf8', *, errors='strict', return_mode: bool = False,
                  delimiter=',', quotechar='"', **kwargs):
@@ -1437,10 +1442,6 @@ class XlPath(type(pathlib.Path())):
                 shutil.copytree(self, dst)
 
     def move(self, dst, if_exists=None):
-        return self.rename2(dst, if_exists)
-
-    def rename2(self, dst, if_exists=None):
-        """ 相比原版的rename，搞了更多骚操作，但性能也会略微下降，所以重写一个功能名 """
         if not self.exists():
             return self
 
@@ -1456,6 +1457,34 @@ class XlPath(type(pathlib.Path())):
             self.rename(dst)
         return dst
 
+    def rename2(self, new_name, if_exists=None):
+        """ 相比原版的rename，搞了更多骚操作，但性能也会略微下降，所以重写一个功能名
+
+        240416周二12:49，这个接口将真的只做重命名，不做移动！所以将会不再支持dst中出现"/"路径配置
+        """
+        if not self.exists():
+            return self
+
+        if '/' in new_name:
+            raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"/"')
+        elif '\\' in new_name:
+            raise ValueError(f'rename2只能做重命名操作，目标路径中不能包含"\\"')
+
+        if self.name == new_name:  # 没有修改名称，跟原来相同
+            return self
+
+        dst = self.parent / new_name
+        if self == dst:
+            # 同一个文件，可能是调整了大小写名称
+            if self.as_posix() != dst.as_posix():
+                tmp = self.tempfile(dir=self.parent)  # self不一定是file，也可能是dir，但这个名称通用
+                self.rename(tmp)
+                self.delete()
+                tmp.rename(dst)
+        elif dst.exist_preprcs(if_exists):
+            self.rename(dst)
+        return dst
+
     def delete(self):
         if self.is_file():
             os.remove(self)
@@ -1621,6 +1650,10 @@ class XlPath(type(pathlib.Path())):
 
             ext = '.' + t.extension
             ext0 = file_path.suffix
+            if ext == ext0:
+                continue
+            elif ext == '.xls' and ext0 == '.et':
+                continue
 
             if ext0 in ('.docx', '.xlsx', '.pptx', '.xlsm'):
                 ext0 = '.zip'