pyxllib 0.3.55__tar.gz → 0.3.57__tar.gz

{pyxllib-0.3.55/pyxllib.egg-info → pyxllib-0.3.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.55
+Version: 0.3.57
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxllib/ext/JLineViewer.py

@@ -75,7 +75,7 @@ class TextEditDelegate(QItemDelegate):
 
     def sizeHint(self, option, index):
         size = super().sizeHint(option, index)
-        size.setHeight(20)  # 限制最大高度为20像素
+        size.setHeight(20)  # 限制最大高度为20像素，限定每个条目只展示一行
         return size
 
     def setModelData(self, editor, model, index):
@@ -173,13 +173,14 @@ class JLineViewer(QMainWindow):
         self.setWindowTitle('JLineEditor')
         self.treeView.setAlternatingRowColors(True)
         self.treeView.setIndentation(20)
-        self.treeView.setSortingEnabled(True)
-        self.treeView.setStyleSheet(LargeStrings.treeViewStyles)
+        # self.treeView.setSortingEnabled(True)
+        # self.treeView.setStyleSheet(LargeStrings.treeViewStyles)
         self.plainTextEdit.setWordWrapMode(QTextOption.WordWrap)
         self.plainTextEdit.setReadOnly(True)
         self.showMaximized()
 
         self.treeView.setSortingEnabled(False)  # 禁止排序
+        self.treeView.setAnimated(False)
         self.plainTextEdit.textChanged.connect(self.updateJson)  # 连接 textChanged 信号到新的槽函数
 
     def addPane(self, widget, title):

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxllib/ext/kq5034lib.py

@@ -1334,10 +1334,10 @@ class 课次数据:
         else:
             return '未开始学习'
 
-    def 小鹅通考勤结果2(self, user_id, 返款梯度, 需求分钟=30):
+    def 小鹅通考勤结果2(self, user_id, 返款梯度, 要求在线分钟=30):
         """ 输入视频返款的梯度，这个函数相比`小鹅通考勤结果`，还会返回单元格颜色格式，对应的返款额 """
         # 1 判断课程回放是不是结束了
-        text = self.小鹅通考勤结果(user_id, 需求分钟)
+        text = self.小鹅通考勤结果(user_id, 要求在线分钟)
         if text == '未开始学习' or '不足' in text:  # 判断该课次是否已经结束了
             delta_day = datetime.timedelta(days=len(返款梯度))
             if (delta_day + self.start_day) <= datetime.date.today():

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxllib/file/specialist/__init__.py

@@ -38,6 +38,8 @@ class JsonlDataFile:
                 if filepaths:
                     filepath = filepaths[0]  # 找到第1个匹配的文件
                     self.infile = XlPath(filepath)
+            else:
+                self.infile = filepath
 
         if self.infile is not None:
             if num_records is None:

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxllib/file/specialist/filelib.py

@@ -2211,6 +2211,7 @@ class TwinDirs:
 
         :param XlPath src_file: 原文件位置（其实输入目录类型也是可以的，内部实现逻辑一致的）
         """
+        src_file = XlPath(src_file)
         _src_file = src_file.relpath(self.src_dir)
         dst_file = self.dst_dir / _src_file
         dst_dir = dst_file.parent

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxllib/prog/pupil.py

@@ -84,14 +84,6 @@ def is_file(arg, exists=True):
     return os.path.isfile(arg)
 
 
-def get_hostname():
-    return socket.getfqdn()
-
-
-def get_username():
-    return os.path.split(os.path.expanduser('~'))[-1]
-
-
 def len_in_dim2_min(arr):
     """ 计算类List结构在第2维上的最小长度
 
@@ -981,3 +973,13 @@ def xlmd5(content):
         return content
     else:
         return hashlib.md5(content).hexdigest()
+
+
+@run_once()
+def get_hostname():
+    return socket.getfqdn()
+
+
+@run_once()
+def get_username():
+    return os.path.split(os.path.expanduser('~'))[-1]

{pyxllib-0.3.55 → pyxllib-0.3.57/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.55
+Version: 0.3.57
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.55 → pyxllib-0.3.57}/pyxlpr/data/gptlib.py

@@ -3,6 +3,7 @@
 # @Author : 陈坤泽
 # @Email  : 877362867@qq.com
 # @Date   : 2023/07/13 14:26
+import copy
 
 from pyxllib.prog.pupil import check_install_package
 
@@ -20,7 +21,7 @@ from transformers import GPT2TokenizerFast
 
 from pyxllib.prog.specialist import browser
 from pyxllib.algo.pupil import ValuesStat
-from pyxllib.file.specialist import XlPath, JsonlDataFile
+from pyxllib.file.specialist import XlPath, JsonlDataFile, TwinDirs
 
 
 def __1_生成提问数据():
@@ -105,18 +106,16 @@ def print_statistics(data, indent_level=1, price_base=0.0015):
 class GptChatJsonl(JsonlDataFile):
     """ GPT问答批量执行脚本的jsonl生成、读取器 """
 
-    def __init__(self, file=None, *, start_id=None):
+    def __init__(self, file=None, num_records=None, *, start_id=None):
         from datetime import date
+
+        super().__init__(file, num_records)
         if start_id is None:
             today = date.today().strftime("%Y%m%d")
             self.start_id = int(today + "00000000")
         else:
             self.start_id = start_id
 
-        super().__init__()
-        if file is not None:
-            self.read_jsonl(file)
-
     def read_jsonl(self, file):
         """ 从一个文件加载数据
         """
@@ -238,7 +237,7 @@ class GptChatJsonl(JsonlDataFile):
 
     def find_indices_by_qlength(self):
         """ 返回提问(q,question)内容从短到长的数据下标 """
-        lens = [(i, len(''.join(x['text']))) for i, x in enumerate(self.records)]
+        lens = [(i, len(''.join([t['content'] for t in x['text']]))) for i, x in enumerate(self.records)]
         # 根据长度进行排序，得到的元组的第一个元素为原列表的下标，第二个元素为对应的长度
         sorted_lens = sorted(lens, key=lambda x: x[1])
         # 取出排序后的下标
@@ -297,7 +296,7 @@ class GptChatJsonl(JsonlDataFile):
                 ls.append(t)
             else:
                 if "file_path" in t:
-                    ls.append(("filep_path=" + t["file_path"] + "\n\n") + t["content"])
+                    ls.append(("filep_path=" + str(t["file_path"]) + "\n\n") + t["content"])
                 else:
                     ls.append(t["content"])
         return ls
@@ -338,27 +337,124 @@ class GptChatJsonl(JsonlDataFile):
         # 输出过滤后的sessions数量
         print(f"过滤后的sessions数量：{len(self.records)}")
 
+    @classmethod
+    def _parse_single_record_answer_contents(cls, record):
+        """ 注意本函数不做record备份 """
+        for answer in record.get('all_answers', []):
+            if isinstance(answer, dict) and 'contents' in answer:
+                n = len(answer['contents'])
+                for i in range(n - 1, -1, -1):
+                    message = answer['contents'][i]['message']
+                    if message and 'content' in message and 'error' not in message:
+                        break
+                else:
+                    answer['contents'] = ''
+                    continue
+
+                content = message['content']
+                if 'parts' in content:
+                    content = '\n'.join(content['parts'])
+                else:
+                    content = content['text']
+                answer['contents'] = content
+
+    @classmethod
+    def _parse_single_record_answer_downloads(cls, record):
+        for answer in record.get('all_answers', []):
+            if 'downloads' in answer:
+                for i, link in enumerate(answer['downloads']):
+                    m = re.search(r'filename%3D(.+?)&sig=', link)
+                    if m:
+                        answer['downloads'][i] = unquote(unquote(m.group(1)))
+
+    @classmethod
+    def parse_single_record_answer(cls, record):
+        cls._parse_single_record_answer_contents(record)
+        cls._parse_single_record_answer_downloads(record)
+
     def parse_answer_contents(self):
         """ 简化解释器返回结果中，contents的结构信息 """
         for record in self.records:
-            for answer in record['all_answers']:
-                if isinstance(answer, dict) and 'contents' in answer:
-                    content = answer['contents'][-1]['message']['content']
-                    if 'parts' in content:
-                        content = '\n'.join(content['parts'])
-                    else:
-                        content = content['text']
-                    answer['contents'] = content
+            self._parse_single_record_answer_contents(record)
 
     def parse_answer_downloads(self):
         """ 解析，简化下载链接的表达形式 """
         for record in self.records:
-            for answer in record['all_answers']:
-                if 'downloads' in answer:
-                    for i, link in enumerate(answer['downloads']):
-                        m = re.search(r'filename%3D(.+?)&sig=', link)
-                        if m:
-                            answer['downloads'][i] = unquote(m.group(1))
+            self._parse_single_record_answer_downloads(record)
+
+        # 目录里的文件名也同理做精简
+        for f in self.infile.parent.glob_files():
+            if f.name.startswith('OpenAI-download-'):
+                f.rename2(f.parent / re.sub(r'OpenAI-download-\d+-', '', f.name),
+                          if_exists='replace')
+
+    def filter_to_rechat(self, check_func, rechat_path=None):
+        """ 筛选失败的数据到一个新的目录，常用于对chatted数据筛选出未成功的样例，上池子重跑
+        这个不是简单的找出得不到all_answers的，而是可以很精细，包含复杂post、verify的情况
+
+        :param check_func: 一个函数，接收一个record，返回True或False
+            True，表示这个record是对的
+            False，表示这个record是错的，要挑选出来
+        :param rechat_path: 把挑选出来的数据放到新路径
+        """
+        if rechat_path is None:
+            rechat_path = XlPath(self.infile.parent.as_posix() + '_rechat/in.jsonl')
+
+        rechat_path = XlPath(rechat_path)
+        td = TwinDirs(self.infile.parent, rechat_path.parent)
+
+        gcj = type(self)()
+        for record in self.records:
+            if not check_func(record):
+                record2 = {}
+                for k in ['id', 'text', 'first_text_length', 'extra']:
+                    record2[k] = record[k]
+                gcj.records.append(record2)
+                for x in record['text']:
+                    if 'file_path' in x:
+                        td.copy_file(td.src_dir / x['file_path'])
+
+        gcj.save(rechat_path)
+        return gcj
+
+    def update_from_rechat(self, check_func, rechat_path=None):
+        """ 从另一份rechat的数据，更新回主master数据
+
+        :param check_func: 原chatted没过，但是rechatted通过的，需要把数据更新过来
+        :param rechat_path: 注意只能传路径，因为可能涉及到文件操作，需要知道目录所在
+            依据这个文件里的record记录更新回self
+        """
+        if rechat_path is None:
+            rechat_path = XlPath(self.infile.parent.as_posix() + '_rechat') / 'out.jsonl'
+
+        rechat_path = XlPath(rechat_path)
+        td = TwinDirs(rechat_path.parent, self.infile.parent)
+
+        id2index = {x['id']: i for i, x in enumerate(self.records)}
+
+        gcj = type(self)(rechat_path)
+        gcj.parse_answer_contents()
+        gcj.parse_answer_downloads()
+
+        # 需要处理下下载链接名称
+        self.parse_answer_downloads()
+        gcj.parse_answer_downloads()
+
+        for y in gcj.records:
+            index = id2index[y['id']]
+            x = self.records[index]
+            if not check_func(x) and check_func(y):
+                # 先把x相关的数据删掉
+                if 'all_answers' in x:
+                    for answer in x['all_answers']:
+                        for fname in answer.get('downloads', []):
+                            (XlPath(self.infile.parent) / fname).delete()
+                # 再把y拷贝过来
+                for answer in y['all_answers']:
+                    for fname in answer.get('downloads', []):
+                        td.copy_file(td.src_dir / fname)
+                self.records[index] = y
+        return gcj
 
 
 GptQuestionJsonl = GptChatJsonl  # 名称向下兼容
@@ -477,3 +573,101 @@ class GptTrainJsonl(JsonlDataFile):
 
         # 或者返回HTML字符串
         return html_content
+
+
+def __4_综合集成类():
+    pass
+
+
+class GptChatDir:
+    """ 一个目录，包含了一个任务的所有数据，包括in、out、post等文件 """
+
+    def __init__(self, root):
+        self.root = root = XlPath(root)
+
+        self.chat_file = root / 'in.jsonl'
+        self.chatted_file = root / 'out.jsonl'
+        self.post_file = root / 'post.jsonl'
+
+        self.upload_files_dir = root / 'upload_files'
+        self.download_files_dir = root / 'download_files'
+
+        for dir_path in [self.root, self.upload_files_dir, self.download_files_dir]:
+            if not dir_path.is_dir():
+                dir_path.mkdir(parents=True, exist_ok=True)
+
+    def init(self):
+        """ 一些基础处理 """
+        # 1 chat信息
+        if self.chatted_file.is_file():
+            gcj1 = GptChatJsonl(self.chatted_file)
+        elif self.chat_file.is_file():
+            gcj1 = GptChatJsonl(self.chat_file)
+        else:
+            print('请确认是否有生成初始的chat数据')
+            return
+
+        print(f'【{self.root.name}】')
+        n = len(gcj1.records)
+        m = sum([len(x['text']) for x in gcj1.records])
+        print(f'1、chat：{n}条会话*{m / n:.2g}条消息')
+        gcj1.check_records()
+        print()
+
+        # 2 chatted信息
+        filter_records = [x for x in gcj1.records if 'all_answers' in x]
+        if filter_records:
+            print(f'2、chatted：已获得{len(filter_records)}条会话')
+        else:
+            print('2、chatted：暂未获得生成数据')
+
+        # 3 post信息
+        if filter_records and not self.post_file.is_file():  # 待生成后处理文件
+            gcj2 = GptChatJsonl()
+            gcj2.infile = self.post_file
+            gcj2.records = filter_records
+            gcj2.parse_answer_contents()
+            gcj2.parse_answer_downloads()
+            gcj2.save()
+        elif self.post_file.is_file():  # 已经有后处理文件
+            gcj2 = GptChatJsonl(self.post_file)
+        else:
+            return
+
+        print(f'3、post：{len(gcj2.records)}条会话')
+
+    def merge_rechatted(self, check_func, rechat_dir):
+        """ 从另一份重跑的数据，更新数据回来
+
+        :param check_func: 原chatted没过，但是rechatted通过的，需要把数据更新过来
+        :param rechat_dir: 注意是传另一份数据所在的目录
+
+        TODO 这个函数还没写完
+        """
+        rechat_dir = XlPath(rechat_dir)
+        td = TwinDirs(rechat_dir, self.root)
+
+        gcj1 = GptChatJsonl(self.chatted_file)
+        gcj2 = type(self)(rechat_dir / 'out.jsonl')
+
+        id2index = {x['id']: i for i, x in enumerate(gcj1.records)}
+
+        # 需要处理下下载链接名称
+        gcj2.parse_answer_downloads()
+
+        for y in gcj2.records:
+            index = id2index[y['id']]
+            x = gcj1.records[index]
+            x2 = copy.deepcopy(x)
+            gcj1.parse_single_record_answer(x2)
+            if not check_func(x2) and check_func(y):
+                # 先把x相关的数据删掉
+                if 'all_answers' in x2:
+                    for answer in x2['all_answers']:
+                        for fname in answer.get('downloads', []):
+                            (self.root / (str(x['id']) + '-' + fname)).delete()
+                # 再把y拷贝过来
+                for answer in y['all_answers']:
+                    for fname in answer.get('downloads', []):
+                        td.copy_file(td.src_dir / fname)
+                gcj1.records[index] = y

{pyxllib-0.3.55 → pyxllib-0.3.57}/setup.py

@@ -30,7 +30,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.55',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.57',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',