pyxllib 0.3.117__tar.gz → 0.3.118__tar.gz

{pyxllib-0.3.117/pyxllib.egg-info → pyxllib-0.3.118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.117
+Version: 0.3.118
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/algo/pupil.py

@@ -157,6 +157,9 @@ class ValuesStat:
 
 class ValuesStat2:
     """ 240509周四17:33，第2代统计器
+
+    240628周五14:05 todo 关于各种特殊格式数据，怎么计算是个问题
+        这问题可能有些复杂，近期估计没空折腾，留以后有空折腾的一个大坑了
     """
 
     def __init__(self, values=None, raw_values=None, data_type=None):
@@ -307,6 +310,33 @@ class ValuesStat2:
 
         return ratio_dict
 
+    def group_count(self, max_entries=None, min_count=None):
+        """ 统计每种取值出现的次数，并根据条件过滤结果
+
+        :param max_entries: 最多显示的条目数
+        :param min_count: 显示的条目至少出现的次数
+        """
+        from collections import Counter
+
+        # 使用Counter来计数每个值出现的次数
+        counts = Counter(self.values or self.raw_values)
+
+        # 根据min_count过滤计数结果
+        if min_count is not None:
+            counts = {k: v for k, v in counts.items() if v >= min_count}
+
+        # 根据max_entries限制结果数量
+        if max_entries is not None:
+            # 按出现次数降序排列，然后选取前max_entries项
+            most_common = counts.most_common(max_entries)
+            # 转换回字典形式
+            counts = dict(most_common)
+        else:
+            # 如果没有指定max_entries，则保持所有满足min_count的结果
+            counts = dict(sorted(counts.items(), key=lambda item: item[1], reverse=True))
+
+        return counts
+
 
 class Groups:
     def __init__(self, data):

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/data/pglib.py

@@ -42,7 +42,7 @@ import psycopg
 import psycopg.rows
 
 from pyxllib.prog.newbie import round_int, human_readable_number
-from pyxllib.prog.pupil import utc_now, utc_timestamp
+from pyxllib.prog.pupil import utc_now, utc_timestamp, is_valid_identifier
 from pyxllib.prog.specialist import XlOsEnv
 from pyxllib.algo.pupil import ValuesStat2
 from pyxllib.file.specialist import get_etag, StreamJsonlWriter
@@ -271,12 +271,14 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
     def __6_高级统计(self):
         pass
 
-    def get_column_valuesstat(self, table_name, column, percentile_count=5,
-                              filter_condition=None, by_data=False):
+    def get_column_valuesstat(self, table_name, column, filter_condition=None,
+                              percentile_count=5,
+                              by_data=False, data_type=None):
         """ 获得指定表格的某个字段的统计特征ValuesStat2对象
 
         :param table_name: 表名
         :param column: 用于计算统计数据的字段名
+            不一定是标准的字段名
         :param percentile_count: 分位数的数量，例如 3 表示只计算中位数
         :param by_data: 是否获得原始数据
             默认只获得统计特征，不获得原始数据
@@ -289,7 +291,15 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
             values = self.exec2col(sql.build_select(column))
             if data_type == 'numeric':
                 values = [x and float(x) for x in values]
-            return ValuesStat2(raw_values=values, data_type=data_type)
+            vs = ValuesStat2(raw_values=values, data_type=data_type)
+
+            if data_type == 'text' and is_valid_identifier(column):
+                vs0 = self.get_column_valuesstat(table_name, column, filter_condition=filter_condition,
+                                                 percentile_count=percentile_count, by_data=False)
+                vs.n = vs0.n
+                vs.dist = vs0.dist
+
+            return vs
 
         def init_from_db():
             # 1 构建基础的 SQL 查询
@@ -341,7 +351,12 @@ WHERE {table_name}.{item_id_name} = cte.{item_id_name}"""
 
             return x
 
-        data_type = self.get_column_data_type(table_name, column)
+        data_type = data_type or self.get_column_data_type(table_name, column)
+
+        # 如果不是标准的列名，强制获取数据
+        if not is_valid_identifier(column):
+            by_data = True
+
         if by_data:
             vs = init_from_db_data()
         else:

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/data/sqlite.py

@@ -89,6 +89,8 @@ class SqlBuilder:
         """ 使用一种特殊的格式化标记来设置规则
 
         :param desc: 例如 'id%2=1'
+
+        todo 我好像傻了，画蛇添足，这个功能好像直接用 .where 就能完成~
         """
         if not desc:
             return
@@ -148,6 +150,7 @@ class SqlBuilder:
             sql.append(f"WHERE {' AND '.join(self._where)}")
         return '\n'.join(sql)
 
+
     def __3_build_中级命令(self):
         pass
 
@@ -168,7 +171,6 @@ class SqlBuilder:
         sql._where = self._where.copy()
         return sql.build_select()
 
-
 class SqlBase:
     """ Sql语法通用的功能 """
 

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/ext/drissionlib.py

@@ -18,7 +18,6 @@ from DrissionPage._base.base import BasePage, BaseElement
 from pyxllib.prog.pupil import inject_members
 from pyxllib.text.pupil import strfind
 
-
 def get_latest_not_dev_tab(page=None):
     """ 开发工具本身也会算一个tab，这个函数返回最新的一个不是开发工具的tab """
     if page is None:
@@ -80,7 +79,7 @@ class XlBasePage(BasePage):
             'tag:downloads-item')
         for item in items:
             loc = unquote(item.sr('tag:img').attr('src').replace('+', ' '))
-            file = re.search(r'path=(.+?)(&scale=\d+x)?$', loc).group(1)
+            file = re.search(r'path=(.+?)(&scale=(\d+(\.\d+)?)x)?$', loc).group(1)
 
             files.append({
                 'file': file,

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/file/specialist/filelib.py

@@ -1124,6 +1124,22 @@ class XlPath(type(pathlib.Path())):
         else:
             return s
 
+    def read_text2(self):
+        """ 智能识别编码的文本读取，这里收集了我见过的一些常见类型 """
+        for encoding in ['utf8',
+                         'gbk',
+                         'gb18030',
+                         'utf_16',
+                         'cp932',  # 日文，Shift-JIS
+                         'Big5',  # 繁体字，Big5
+                         'big5hkscs',  # 繁体字
+                         ]:
+            try:
+                content = self.read_text(encoding=encoding)
+                return content, encoding
+            except (UnicodeDecodeError, UnicodeError):
+                continue
+
     def readlines_batch(self, batch_size, *, encoding='utf8'):
         """ 将文本行打包，每次返回一个批次多行数据
 

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/file/xlsxlib.py

@@ -7,6 +7,8 @@
 """
 扩展了些自己的openpyxl工具
 """
+import copy
+
 import time
 
 from pyxllib.prog.pupil import check_install_package, run_once
@@ -2463,7 +2465,7 @@ def extract_workbook_summary2(file_path, *,
     """
     :param keep_links: 是否保留外部表格链接数据。如果保留，打开好像会有点问题。
     :param mode:
-        0，最原始的summary3摘要
+        0，最原始的summary2摘要
         1，添加当前工作表、单元格位置的信息
     :param kwargs: 捕捉其他参数，主要是向下兼容，其实现在并没有用
 
@@ -3015,7 +3017,10 @@ def extract_workbook_summary3b(file_path,
 
     :param summary_limit_len: 摘要长度限制
     :param timeout_seconds: 超时限制
-    :param return_mode: 返回模式，0表示只返回摘要，1表示返回摘要和耗时
+    :param return_mode: 返回模式
+        0，表示只返回摘要
+        1，表示返回摘要和耗时
+        2, 再增加返回summary2
     :param len_mode:
         0, 使用len作为token长度评估
         1, 使用模型评估实际token长度
@@ -3024,6 +3029,7 @@ def extract_workbook_summary3b(file_path,
     res = {}
     res['fileName'] = Path(file_path).name
     load_time = summary2_time = summary3_time = -1
+    summary2_res = {}
 
     def reduce_summary(summary):
         """ 如果转json后的summary超过4K，去掉可能的sheets字段 """
@@ -3037,6 +3043,7 @@ def extract_workbook_summary3b(file_path,
             start_time = time.time()
             res, load_time = extract_workbook_summary2(file_path, mode=1, return_mode=1, **kwargs)
             # res = convert_to_json_compatible(res)
+            summary2_res = copy.deepcopy(res)
             summary2_time = time.time() - start_time - load_time
             start_time = time.time()
             if len_mode == 1:
@@ -3055,8 +3062,13 @@ def extract_workbook_summary3b(file_path,
         res['error'] = f'提取摘要时发生错误：{format_exception(e, 2)}'
         reduce_summary(res)
 
-    if return_mode == 1:
-        return res, {'load_time': human_readable_number(load_time),
+    time_dict = {'load_time': human_readable_number(load_time),
                      'summary2_time': human_readable_number(summary2_time),
                      'summary3_time': human_readable_number(summary3_time)}
+
+    if return_mode == 1:
+        return res, time_dict
+    elif return_mode == 2:
+        return res, time_dict, summary2_res
+
     return res

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/prog/pupil.py

@@ -34,6 +34,7 @@ import threading
 import time
 import traceback
 from urllib.parse import urlparse
+import re
 
 from pyxllib.prog.newbie import classproperty, typename
 
@@ -1133,10 +1134,14 @@ def support_multi_processes(default_processes=1):
                 if process_id is None:
                     return func(*args, **kwargs)
                 else:
-                    return func(*args, **kwargs, process_count=process_count, process_id=process_id)
+                    return func(*args, **kwargs, process_count=process_count, process_id=int(process_id))
             else:
                 mpl = MultiProcessLauncher()
                 for i in range(int(process_count)):
+                    if isinstance(process_id, int) and i != process_id:
+                        continue
+
+                    # todo 这样使用有个坑，process_count、process_id都是以str类型传入的，开发者下游使用容易出问题
                     cmds = [func.__name__,
                             '--process_count', str(process_count),
                             '--process_id', str(i)
@@ -1303,3 +1308,8 @@ def check_counter(data, top_n=10):
 def tprint(*args, **kwargs):
     """ 带时间戳的print """
     print(utc_now2(), *args, **kwargs)
+
+
+def is_valid_identifier(name):
+    """ 判断是否是合法的标识符 """
+    return re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name)

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/prog/specialist/browser.py

@@ -230,6 +230,9 @@ class Browser(Explorer):
             file = file.rename(get_etag(str(file)) + file.suffix, if_exists='replace')
         self.__call__(arg, file, **kwargs)
 
+    def url(self, *args, wait=True, **kwargs):
+        super().__call__(*args, wait=wait, **kwargs)
+
     def __call__(self, arg, file=None, *, wait=True, clsmsg=True, to_html_args=None,
                  **kwargs):  # NOQA Browser的操作跟标准接口略有差异
         """ 该版本会把arg转存文件重设为文件名

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/prog/specialist/tictoc.py

@@ -12,6 +12,7 @@ from humanfriendly import format_timespan
 
 from pyxllib.algo.pupil import natural_sort, ValuesStat
 from pyxllib.text.pupil import shorten, listalign
+from pyxllib.prog.newbie import human_readable_number
 
 __tictoc = """
 基于 pytictoc 代码，做了些自定义扩展
@@ -40,13 +41,15 @@ class TicToc:
     TicToc.elapsed   #t.end - t.start; i.e., time elapsed from t.start when t.toc() or t.tocvalue() was last called
     """
 
-    def __init__(self, title='', *, disable=False):
+    def __init__(self, title='', *, disable=False, min_display_seconds=None):
         """Create instance of TicToc class."""
         self.start = timeit.default_timer()
         self.end = float('nan')
         self.elapsed = float('nan')
         self.title = title
         self.disable = disable
+        # 只有达到这个时间，才会显示耗时
+        self.min_display_seconds = min_display_seconds
 
     def tic(self):
         """Start the timer."""
@@ -64,7 +67,7 @@ class TicToc:
         self.elapsed = self.end - self.start
         if not self.disable:
             # print(f'{self.title} {msg} {self.elapsed:.3f} 秒.')
-            print(f'{self.title} {msg} elapsed {format_timespan(self.elapsed)}.')
+            print(f'{self.title} {msg} elapsed {human_readable_number(self.elapsed, "秒")}.')
         if restart:
             self.start = timeit.default_timer()
 
@@ -84,14 +87,14 @@ class TicToc:
     @staticmethod
     def process_time(msg='time.process_time():'):
         """计算从python程序启动到目前为止总用时"""
-        print(f'{msg} {format_timespan(time.process_time())}.')
+        print(f'{msg} {human_readable_number(time.process_time(), "秒")}.')
 
     def __enter__(self):
         """Start the timer when using TicToc in a context manager."""
         from pyxllib.prog.specialist import get_xllog
 
         if self.title == '__main__' and not self.disable:
-            get_xllog().info(f'time.process_time(): {format_timespan(time.process_time())}.')
+            get_xllog().info(f'time.process_time(): {human_readable_number(time.process_time(), "秒")}.')
         self.start = timeit.default_timer()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -102,10 +105,10 @@ class TicToc:
         xllog = get_xllog()
 
         if exc_tb is None:
-            if not self.disable:
-                xllog.info(f'{self.title} finished in {format_timespan(elapsed)}.')
+            if not self.disable and (self.min_display_seconds is None or elapsed >= self.min_display_seconds):
+                xllog.info(f'{self.title} finished in {human_readable_number(elapsed, "秒")}.')
         else:
-            xllog.info(f'{self.title} interrupt in {format_timespan(elapsed)},')
+            xllog.info(f'{self.title} interrupt in {human_readable_number(elapsed, "秒")},')
 
 
 __timer = """

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib/text/charclasslib.py

@@ -69,7 +69,10 @@ def get_charclass_num(content):
 
 
 def get_charclass_rate(content, round_digits=4):
-    """ 检查字符类型分布数量 """
+    """ 检查字符类型分布数量
+
+    :param str content: 文本内容
+    """
     ct = get_charclass_num(content)
 
     # 这一类权重要翻倍
@@ -93,6 +96,8 @@ def check_language_from_counter(ct):
         这个比例最早是给表格设计的，表格很可能表头是中文，中间都是英文、数字数据，此时仍然归为中文
         但如果是一般性的文档，可能要提高中文的比例，才能视为中文文档
     """
+    ct['常用汉字'] = ct.get('常用汉字', 0)
+    ct['偏僻汉字'] = ct.get('偏僻汉字', 0)
 
     if '繁体汉字' in ct:
         if safe_div(ct['繁体汉字'], ct['常用汉字']) > 0.1:

{pyxllib-0.3.117 → pyxllib-0.3.118/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.117
+Version: 0.3.118
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.117 → pyxllib-0.3.118}/pyxllib.egg-info/requires.txt

@@ -21,29 +21,28 @@ Jinja2
 pandas>=0.23.4
 
 [xlai]
-pynvml
-opsdroid-get-image-size
 opencv-python
-pillow
-xlcocotools
+opsdroid-get-image-size
+pynvml
 visdom
+pillow
 
 [xlall]
-flask-cors
-html2text
+flask-jwt-extended
 ujson
 flask-wtf
-sentencepiece
-flask-jwt-extended
-flask
-opsdroid-get-image-size
 opencv-python
-flask-restful
-pillow
+opsdroid-get-image-size
 zhconv
+html2text
 xlrd
+flask
+sentencepiece
+flask-cors
+flask-restful
+pillow
 
 [xlcv]
-pillow
-opsdroid-get-image-size
 opencv-python
+opsdroid-get-image-size
+pillow

{pyxllib-0.3.117 → pyxllib-0.3.118}/setup.py

@@ -22,8 +22,8 @@ opencv-python
 xlai = """
 pynvml
 visdom
-xlcocotools
 """
+# xlcocotools
 # fvcore
 
 # 全量的依赖，自用
@@ -39,12 +39,13 @@ ujson
 html2text
 flask-jwt-extended
 """
+# pywin32
 
 _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.117',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.118',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',