maque 0.2.1__py3-none-any.whl

maque/utils/helper_metrics.py

@@ -0,0 +1,121 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+模型评估指标工具
+"""
+
+from maque.ai_platform.metrics import MetricsCalculator, save_pred_metrics
+from maque.utils.helper_parser import parse_generic_tags
+from pathlib import Path
+import pandas as pd
+
+
+def calc_binary_metrics(
+    df: pd.DataFrame,
+    response_col="response",
+    label_col="labels",
+    parse_response_to_pred=False,
+    pred_parsed_tag="answer",
+    record_root_dir="record",
+):
+    """
+    计算交叉二分类指标
+
+    Args:
+    df (pd.DataFrame): 包含预测结果和标签的数据框。
+    response_col (str, optional): 预测结果所在的列名，默认为 'response'。
+    label_col (str, optional): 标签所在的列名，默认为 'labels'。
+    parse_response_to_pred (bool, optional): 是否将预测结果解析为特定的格式，默认为 False。
+    record_root_dir (str, optional): 记录根目录，默认为 'record'。 会将预测结果保存到record_root_dir文件夹下。
+
+    Returns:
+    pd.DataFrame: 包含每个标签下的二分类指标的数据框。
+
+    """
+
+    # 如果需要将预测结果解析为特定的格式
+    if parse_response_to_pred:
+        parsed_results = [parse_generic_tags(response) for response in df[response_col]]
+        parsed_dict = {}
+        all_keys = set()
+        for d in parsed_results:
+            all_keys |= set(d.keys())
+        for d in parsed_results:
+            for key in all_keys:
+                parsed_dict.setdefault(key, []).append(d.get(key, "不规范"))
+        df["parsed_results"] = parsed_results
+        for param, values in parsed_dict.items():
+            df[f"parsed_{param}"] = values
+        df["preds"] = df[f"parsed_{pred_parsed_tag}"]
+    else:
+        # 如果不需要解析预测结果，则直接处理
+        df["preds"] = df[response_col].apply(
+            lambda x: x if not pd.isna(x) else "不规范"
+        )
+
+    df["binary_pred"] = df["preds"].apply(
+        lambda x: "白" if x in ["正常", "不违规", "[不违规]"] else "黑"
+    )
+
+    if label_col is None:
+        timestamp = pd.Timestamp.now().strftime("%Y%m%d_%H%M%S")
+        p = Path(f"{record_root_dir}/binary_pred-{timestamp}.xlsx")
+        p.parent.mkdir(parents=True, exist_ok=True)
+        df.to_excel(p, index=False, engine="openpyxl")
+    else:
+        # 保存预测指标
+        # save_pred_metrics(
+        #     df=df,
+        #     pred_col="preds",
+        #     label_col=label_col,
+        #     record_folder=f"{record_root_dir}",
+        # )
+
+        df["binary_label"] = df[label_col].apply(
+            lambda x: "白" if x in ["正常", "不违规", "[不违规]"] else "黑"
+        )
+
+        # 替换 target_label 列的 无害 为 不违规
+        save_pred_metrics(
+            df,
+            pred_col="binary_pred",
+            label_col="binary_label",
+            record_folder=f"{record_root_dir}/binary",
+        )
+        # 计算labels 列下的每个值groupby的指标
+        metrics_dict = {}
+        for name, _df in df.groupby(label_col):
+            metrics_calculator = MetricsCalculator(
+                _df, pred_col="binary_pred", label_col="binary_label"
+            )
+            metrics = metrics_calculator.get_metrics()
+            classification_report = metrics["classification_report"]
+            # 判断分类报告中是否存在'黑'类别
+            if "黑" in classification_report:
+                binary_recall, support = (
+                    classification_report["黑"]["recall"],
+                    classification_report["黑"]["support"],
+                )
+                metrics_dict[name] = {
+                    "binary_recall": binary_recall,
+                    "binary_true": len(_df[_df["binary_pred"] == "黑"]),
+                    "support": support,
+                }
+            else:
+                # 如果不存在'黑'类别，则使用'白'类别
+                binary_recall, support = (
+                    classification_report["白"]["recall"],
+                    classification_report["白"]["support"],
+                )
+                metrics_dict[name] = {
+                    "binary_recall": binary_recall,
+                    "binary_true": len(_df[_df["binary_pred"] == "白"]),
+                    "support": support,
+                }
+        binary_metrics_df = pd.DataFrame(metrics_dict).T
+        markdown_str = f"{binary_metrics_df.to_markdown()}"
+
+        with open(f"{record_root_dir}/binary_metrics.md", "w") as f:
+            f.write(markdown_str)
+        print(markdown_str)
+        return binary_metrics_df

maque/utils/helper_parser.py

@@ -0,0 +1,168 @@
+# !/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""解析器"""
+
+import re
+from typing import List
+import pandas as pd
+
+
+def parse_generic_tags(text: str, strict: bool = False) -> dict:
+    """
+    通用的标签解析函数，可以解析字符串中混合存在的 <label>content</label> 和 <label>content 格式。
+
+    Args:
+        text: 待解析的字符串。
+        strict: 布尔值，用于控制是否启用严格匹配模式。
+                - False (默认): 解析所有闭合标签和开放式标签，并优先处理闭合标签。
+                - True: 仅解析格式完好的闭合标签 (<label>content</label>)，忽略所有开放式标签。
+
+    Returns:
+        dict: 解析结果，格式为 {label: content, ...}
+    """
+    if not text:
+        return {}
+
+    result = {}
+
+    if strict:
+        # --- 严格模式 ---
+        # 只匹配拥有正确闭合标签的 <label>content</label> 格式
+        pattern_with_closing = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        matches = re.findall(pattern_with_closing, text, re.DOTALL)
+        for label, content in matches:
+            result[label.strip()] = content.strip()
+
+    else:
+        # --- 兼容模式 (非严格) ---
+        remaining_text = str(text)
+
+        # 1. 优先处理闭合标签，并从文本中“移除”它们
+        def process_closed_tag(match_obj):
+            label = match_obj.group(1).strip()
+            content = match_obj.group(2).strip()
+            result[label] = content
+            return ""
+
+        pattern_with_closing = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        remaining_text = re.sub(
+            pattern_with_closing, process_closed_tag, remaining_text, flags=re.DOTALL
+        )
+
+        # 2. 在剩余文本中处理开放式标签
+        pattern_open = r"<([^>]+)>\s*(.*?)(?=<[^>]+>|$)"
+        matches_open = re.findall(pattern_open, remaining_text, re.DOTALL)
+        for label, content in matches_open:
+            label_stripped = label.strip()
+            if label_stripped not in result:
+                result[label_stripped] = content.strip()
+
+    return result
+
+
+def split_urls(text: str) -> List[str]:
+    """从文本中提取所有URL
+
+    Args:
+        text: 包含URL的字符串
+
+    Returns:
+        List[str]: 提取到的URL列表
+    """
+    if not text:
+        return []
+
+    # 使用正则表达式匹配所有URL
+    url_pattern = r"https?://[^\s]+"
+    urls = re.findall(url_pattern, text)
+    urls = re.findall(r"https?://.*?(?=https?://|$)", text)
+    url_list = [url.strip(" ,|;；") for url in urls]
+    return url_list
+
+
+def split_image_paths(text: str, separators: List[str] = None) -> List[str]:
+    """从文本中提取所有图像路径（包括HTTP URL和本地路径）
+    
+    优化版本：能够正确处理URL中包含逗号等分隔符的情况，避免错误切割。
+
+    Args:
+        text: 包含图像路径的字符串
+        separators: 分隔符列表，如 [",", ";"] 或 ["\n"]，默认为 [",", ";", "\n", "\r"]
+
+    Returns:
+        List[str]: 提取到的图像路径列表
+    """
+    if not text or pd.isna(text):
+        return []
+
+    text = str(text).strip()
+    if not text:
+        return []
+
+    # 如果用户提供了自定义分隔符，优先使用用户的设置，不使用智能检测
+    if separators is not None:
+        # 用户明确指定了分隔符，跳过智能检测，直接使用用户的分隔符
+        pass
+    else:
+        # 只有在用户没有指定分隔符时，才使用智能提取完整的URL
+        # 使用正则表达式匹配完整的HTTP(S) URL
+        url_pattern = r'https?://[^\s<>"\';]*'
+        urls = re.findall(url_pattern, text)
+        
+        # 如果找到了URL，检查是否覆盖了整个文本（说明是单个URL）
+        if urls:
+            # 如果只有一个URL且几乎覆盖了整个文本，直接返回
+            if len(urls) == 1 and len(urls[0]) > len(text) * 0.8:
+                return [urls[0]]
+            
+            # 如果找到多个URL，返回所有URL
+            if len(urls) > 1:
+                return urls
+            
+            # 如果只找到一个URL但不覆盖全文，可能有其他路径，继续原逻辑
+    
+    # 如果没有找到HTTP URL，或需要处理混合内容，使用原来的分隔符逻辑
+    # 但要更小心地处理逗号，避免切断URL参数
+    
+    # 默认分隔符列表
+    if separators is None:
+        separators = [";", "\n", "\r"]  # 永远不使用逗号作为分隔符，因为URL参数中常包含逗号
+    
+    # 使用正则表达式进行分割，支持多个分隔符
+    if len(separators) == 1:
+        # 单个分隔符，直接分割
+        paths = text.split(separators[0])
+    else:
+        # 多个分隔符，构建正则表达式
+        escaped_separators = [re.escape(sep) for sep in separators]
+        pattern = "|".join(escaped_separators)
+        paths = re.split(pattern, text)
+
+    # 清理和过滤路径
+    cleaned_paths = []
+    for path in paths:
+        path = path.strip()
+        if path:
+            # 检查是否为HTTP(S) URL
+            if re.match(r"https?://", path):
+                cleaned_paths.append(path)
+            # 检查是否为有效的文件路径
+            elif (
+                path.startswith("./")
+                or path.startswith("../")
+                or path.startswith("/")
+                or path.startswith("\\")
+                or re.match(r"^[A-Za-z]:[/\\]", path)  # Windows绝对路径如 C:\ 或 D:/
+                or (
+                    ("/" in path or "\\" in path)
+                    and (
+                        "." in path
+                        or path.endswith(
+                            (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp")
+                        )
+                    )
+                )
+            ):
+                cleaned_paths.append(path)
+
+    return cleaned_paths

maque/utils/net.py

@@ -0,0 +1,64 @@
+import socket
+import re
+
+
+def get_inner_ip():
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+        s.connect(('8.8.8.8', 80))
+        return s.getsockname()[0]
+
+
+def get_outer_ip():
+    import requests
+    # headers = {
+    #     'User-Agent':
+    #     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36"
+    # }
+    # return requests.get('http://ip.42.pl/raw', headers=headers).text.strip()
+    return requests.get('http://ifconfig.me/ip', timeout=1).text.strip()
+
+
+def get_ip(env="inner"):
+    """Get in-net / out-ner ip address.
+    env: "inner" or "outer"
+    """
+    if env == "inner":
+        return get_inner_ip()
+    elif env == "outer":
+        return get_outer_ip()
+    else:
+        raise ValueError("`env` invalid!")
+
+
+def domain2ip(*domains):
+    domain_ip = [(domain, socket.gethostbyname(domain.strip())) for domain in domains]
+    return domain_ip
+
+
+def get_github_ip():
+    import requests
+    def get_ip(website):
+        request = requests.get('https://ipaddress.com/website/' + website)
+        domain_ip = None
+        if request.status_code == 200:
+            ips = re.findall(r"<strong>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}?)</strong>", request.text)
+            domain_ip = {ip for ip in ips}
+        return domain_ip
+
+    ip_list = ['github.com',
+               'github.global.ssl.fastly.net',
+               'assets-cdn.github.com',
+               # 'codeload.github.com',
+               # 'google.com'
+               ]
+    return [[i, get_ip(i)] for i in ip_list]
+
+
+# getip('assets-cdn.github.com')
+# getip('github.global.ssl.fastly.net')
+
+if __name__ == "__main__":
+    print(get_inner_ip())
+    print(get_outer_ip())
+    print(domain2ip("www.baidu.com", "www.google.com", "github.com"))
+    # print(get_github_ip())

maque/utils/nvidia_stat.py

@@ -0,0 +1,140 @@
+import subprocess
+import io
+import csv
+import collections
+
+
+def commandexists(shellcommand):
+    status, output = subprocess.getstatusoutput(shellcommand)
+    exists = status == 0
+    if not exists:
+        print("Could not execute: {0}".format(shellcommand))
+    return exists
+
+
+def command(args):
+    return subprocess.check_output(args).decode()
+
+
+def csvtodictdict(csvdata, colnames, keycols, fmtcols={}):
+    """
+    Returns a dict of dicts from csv file with specified column names and primary key column
+    accepts and optional element formatting per column as a dictionary of format functions
+    """
+    fmtcols = collections.defaultdict(lambda: lambda x: x, **fmtcols)
+    d = {}
+    rows = csv.reader(csvdata)
+    for row in rows:
+        drow = {colname: fmtcols[colname](val) for colname, val in zip(colnames, row)}
+        if isinstance(keycols, str):
+            key = drow.pop(keycols)
+        else:
+            key = tuple([drow.pop(keycol) for keycol in keycols])
+        d[key] = drow
+    return d
+
+
+def csvheaderargs(fmtcol, cols):
+    return ",".join([fmtcol.format(col) for col in cols])
+
+
+def commandtodictdict(baseargs, cols, keycols=None, queryargfmt="{0}", colargfmt="{0}", outputfmt={}, skipheader=False):
+    queryarg = queryargfmt.format(csvheaderargs(colargfmt, cols))
+    args = baseargs + [queryarg]
+    csvoutput = io.StringIO(command(args))
+    if skipheader:
+        csvoutput.readline()
+    if keycols is None:
+        keycols = cols[0]
+    return csvtodictdict(csvoutput, cols, keycols, fmtcols=outputfmt)
+
+
+def renamekeys(d, names):
+    '''
+    updates key names in d based on dict of old/new name pairs
+    returning resulting updated dict
+    '''
+    for oldname, newname in names.items():
+        d[newname] = d.pop(oldname)
+    return d
+
+
+def docker_gpu_stat():
+    # get results of all commands without container arguments
+    dockerps = commandtodictdict(['docker', 'ps', '--format'],
+                                 ['ID', 'Image', 'Ports'],
+                                 keycols='ID',
+                                 queryargfmt="'{0}'",
+                                 colargfmt="{{{{.{0}}}}}",
+                                 outputfmt={'ID': lambda s: s[1:]})
+    dockerstats = commandtodictdict(['docker', 'stats', '--no-stream', '--format'],
+                                    ['Container', 'MemUsage', 'CPUPerc'],
+                                    keycols='Container',
+                                    queryargfmt="'{0}'",
+                                    colargfmt="{{{{.{0}}}}}",
+                                    outputfmt={'Container': lambda s: s[1:]})
+    unitstats = commandtodictdict(['nvidia-smi', '--format=csv'],
+                                  ['gpu_uuid', 'utilization.gpu', 'utilization.memory'],
+                                  keycols='gpu_uuid',
+                                  queryargfmt="--query-gpu={0}",
+                                  outputfmt={'gpu_uuid': lambda s: s.lstrip()},
+                                  skipheader=True)
+    unitprocstats = commandtodictdict(['nvidia-smi', '--format=csv'],
+                                      ['pid', 'process_name', 'gpu_uuid', 'used_memory'],
+                                      keycols=['pid', 'gpu_uuid'],
+                                      queryargfmt="--query-compute-apps={0}",
+                                      outputfmt={'gpu_uuid': lambda s: s.lstrip()},
+                                      skipheader=True)
+
+    # map gpu_uuids to short ids in unit info rename columns
+    shortunitids = {gpu_uuid: "{0}".format(shortid) for gpu_uuid, shortid in
+                    zip(unitstats.keys(), range(len(unitstats)))}
+    colnames = {'utilization.gpu': 'used_gpu'}
+    unitstats = {shortunitids[gpu_uuid]: renamekeys(stats, colnames) for gpu_uuid, stats in unitstats.items()}
+    unitprocstats = {(pid, shortunitids[gpu_uuid]): stats for (pid, gpu_uuid), stats in unitprocstats.items()}
+
+    # display fmt data
+    basedisplaycols = collections.OrderedDict([('Container', 12),
+                                               ('Image', 18)])
+    optdisplaycols = collections.OrderedDict([('pid', 7),
+                                              ('gpu_uuid', 8),
+                                              ('used_memory', 12),
+                                              ('used_gpu', 9)])
+    displaycols = collections.OrderedDict(list(basedisplaycols.items()) +
+                                          list(optdisplaycols.items()))
+
+    # display fmt strings
+    basedisplayfmt = '\t'.join(['{{{0}:{1}.{1}}}'.format(col, width) for col, width in basedisplaycols.items()])
+    optdisplayfmt = '\t'.join(['{{{0}:{1}.{1}}}'.format(col, width) for col, width in optdisplaycols.items()])
+    displayfmt = '\t'.join([basedisplayfmt, optdisplayfmt])
+
+    # print rows of relevant container processes
+    # (everything below a bit janky in terms of argument expectations and generalization)
+    dockerall = {container: {**dockerps[container], **dockerstats[container]} for container in dockerstats.keys()}
+    someunitsactive = False
+    print(displayfmt.format(**{col: col for col in displaycols.keys()}))
+    for container, dockerinfo in dockerall.items():
+        # very particular incantation needed here for top options to function correctly:
+        # https://www.projectatomic.io/blog/2016/01/understanding-docker-top-and-ps/
+        pids = command(['docker', 'top', container, '-eo', 'pid']).split('\n')[1:-1]  # obviously could be a bit brittle
+
+        containerunitstatslist = [((proc, unit), stats) for (proc, unit), stats in sorted(unitprocstats.items()) if
+                                  proc in pids]
+        containerunitstats = collections.OrderedDict(containerunitstatslist)
+
+        if containerunitstats:
+            someunitsactive = True
+            basedisplaystr = basedisplayfmt.format(Container=container, **dockerinfo)
+            print(basedisplaystr)
+            for (pid, gpu_uuid), stats in containerunitstats.items():
+                print(optdisplayfmt.rjust(99).format(pid=pid, gpu_uuid=gpu_uuid, **stats, **unitstats[gpu_uuid]))
+    if not someunitsactive:
+        print("\n\t\t no gpu units being used by docker containers ")
+
+
+if __name__ == '__main__':
+    # check for existence of docker and nvidia-smi commands
+    if commandexists('docker') and commandexists('nvidia-smi'):
+        docker_gpu_stat()
+    else:
+        print('Command(s) not found')

maque/utils/ops.py

@@ -0,0 +1,53 @@
+import re
+
+
+def find_all_index(pattern, string, flags=0):
+    """find all matched index of string"""
+    return [i.span() for i in re.finditer(pattern, string, flags=flags)]
+
+
+def string_add(string: str, dx=1):
+    # count_points = string.count('.')
+    items = find_all_index(r"\.", string)
+    number_list = [i for i in string.split(".")]
+    number_str = "".join(number_list)
+    number_len = len(number_str)
+    number = int(number_str)
+    number += dx
+    new_number_str = f"{number:0>{number_len}d}"
+    new_number_list = list(new_number_str)
+    [new_number_list.insert(idx[0], ".") for idx in items]
+    return "".join(new_number_list)
+
+
+def index_char(size=1000):
+    """
+    Get the index of all characters.
+    use chr()
+    """
+    index_token = {i: chr(i) for i in range(size)}
+    token_index = {chr(i): i for i in range(size)}
+    return index_token, token_index
+
+
+def find_match(start, end, S, flag=0):
+    """find the string between `start` and `end` of `S`
+    flag=0 defaults, means no special specification
+    flag options:
+        re.I    IGNORECASE， 忽略大小写的匹配模式
+        re.M    MULTILINE，多行模式, 改变 ^ 和 $ 的行为
+        re.S  　DOTALL，此模式下 '.' 的匹配不受限制，可匹配任何字符，包括换行符，也就是默认是不能匹配换行符
+        re.X    VERBOSE，冗余模式， 此模式忽略正则表达式中的空白和#号的注释
+    """
+    try:
+        START = re.search(start, S, flags=flag).span()[1]
+        END = re.search(end, S, flags=flag).span()[0]
+        return S[START:END]
+    except:
+        print("Do not match anything.")
+        return None
+
+
+def find_match2(pattern, S, flag=0):
+    res = re.search(pattern, S, flags=flag)
+    return res.group()

maque/utils/packages.py

@@ -0,0 +1,31 @@
+import importlib.metadata
+import importlib.util
+
+
+def _is_package_available(name: str) -> bool:
+    return importlib.util.find_spec(name) is not None
+
+
+def _get_package_version(name: str) -> str:
+    try:
+        return importlib.metadata.version(name)
+    except Exception:
+        return "0.0.0"
+
+
+def is_flash_attn2_available():
+    return _is_package_available("flash_attn") and _get_package_version("flash_attn").startswith("2")
+
+
+def is_jieba_available():
+    return _is_package_available("jieba")
+
+def is_levenshtein_available():
+    return _is_package_available("Levenshtein")
+
+def is_nltk_available():
+    return _is_package_available("nltk")
+
+
+def is_rouge_available():
+    return _is_package_available("rouge_chinese")

maque/utils/path.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+from typing import Union, List
+import inspect
+from glob import glob as _glob
+import os
+import sys
+
+
+def rel_to_abs(rel_path: Union[str, Path], parents=0, return_str=True, strict=False):
+    """Return absolute path relative to the called file
+    args:
+        parent: <int> The number of times `f_back` will be calledd.
+    """
+    currentframe = inspect.currentframe()
+    f = currentframe.f_back
+    for _ in range(parents):
+        f = f.f_back
+    current_path = Path(f.f_code.co_filename).parent
+    pathlib_path = current_path / rel_path
+    pathlib_path = pathlib_path.resolve(strict=strict)
+    if return_str:
+        return str(pathlib_path)
+    else:
+        return pathlib_path
+
+
+def rel_path_join(*paths: Union[str, Path], return_str=True):
+    return rel_to_abs(os.path.join(*paths), parents=1, return_str=return_str)
+
+
+def ls(_dir, *patterns, relp=True, concat='extend', recursive=False) -> List[str]:
+    """
+    Example:
+    --------
+        >>> ls("./data/", "*.jpg", "*.png")
+    """
+    if relp:
+        _dir = rel_to_abs(_dir, parents=1, return_str=True, strict=False)
+    path_list = []
+    for pattern in patterns:
+        if concat == 'extend':
+            path_list.extend(_glob(os.path.join(_dir, pattern), recursive=recursive))
+        else:
+            path_list.append(_glob(os.path.join(_dir, pattern), recursive=recursive))
+    return path_list
+
+
+def add_env_path(*rel_paths: str):
+    """
+    Example:
+    --------
+        >>> add_env_path('..')
+        >>> add_env_path('..', '../..')
+    """
+
+    for i in rel_paths:
+        sys.path.append(rel_to_abs(i, parents=1))