smartpush 1.1.1__py3-none-any.whl → 1.1.4__py3-none-any.whl

smartpush/export/basic/ExcelExportChecker.py

@@ -1,10 +1,8 @@
-import os
-import re
-import warnings
-from io import BytesIO
+import copy
+import json
 from urllib.parse import unquote
-import pandas as pd
-from requests import request
+
+from smartpush.export.basic.ReadExcel import *
 
 """
 用于excel校验
@@ -12,139 +10,6 @@ from requests import request
 warnings.simplefilter("ignore")
 
 
-def read_excel_from_oss(url="", method="get"):
-    """读取oss的excel内容并写入到本地csv"""
-    try:
-        result = request(method=method, url=url)
-        excel_data = BytesIO(result.content)
-        print(f"成功读取oss文件内容: {url}")
-        return excel_data
-    except Exception as e:
-        print(f"读取oss报错 {url} 时出错：{e}")
-
-
-def read_excel_data(excel_data, **kwargs):
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", category=UserWarning, module=re.escape('openpyxl.styles.stylesheet'))
-        dfs = pd.read_excel(excel_data, sheet_name=None, engine="openpyxl", na_filter=False, **kwargs)
-        return dfs
-
-
-def read_excel_and_write_to_dict(excel_data=None, file_name=None, **kwargs):
-    """excel内容并写入到内存dict中
-    :param excel_data：excel的io对象, 参数和file_name互斥
-    :file_name: excel文件名称，目前读取check_file目录下文件，参数和excel_data互斥
-    """
-    try:
-        if excel_data is not None and file_name is not None:
-            pass
-        elif file_name is not None:
-            excel_data = os.path.join(os.path.dirname(os.getcwd()) + "/check_file/" + file_name)
-        dfs = read_excel_data(excel_data)
-        # 将DataFrame转换为字典，以行为单位存储数据
-        row_dict = {}  # 创建一个空字典来存储按行转换的数据
-        for index, row in dfs.iterrows():  # 遍历DataFrame中的每一行
-            row_dict[index] = row.to_dict()  # 将每一行转换为字典并存储在row_dict中
-        return row_dict
-    except Exception as e:
-        print(f"excel写入dict时出错：{e}")
-
-
-def read_excel_and_write_to_list(excel_data=None, sheet_name=None, file_name=None, **kwargs):
-    """excel内容并写入到内存list中
-    :param excel_data：excel的io对象, 参数和file_name互斥
-    :file_name: excel文件名称，目前读取check_file目录下文件，参数和excel_data互斥
-
-    io：可以是文件路径、文件对象或 ExcelFile 对象，代表要读取的 Excel 文件。
-    sheet_name：指定要读取的工作表，默认为第一个工作表（索引为 0）。
-    header：指定哪一行作为列名，默认为第一行（索引为 0）。
-    names：可以为列提供自定义名称，如果设置了这个，会覆盖文件中的列名。
-    index_col：可以指定某一列或多列作为索引。
-    usecols：可以指定要读取的列，可以是列的索引、列名或一个筛选函数。
-    dtype：可以指定数据类型，控制数据的类型转换。
-    engine：指定使用的 Excel 引擎，比如 xlrd、openpyxl 等。
-    converters：可以为不同的列指定自定义的转换函数，以字典形式存储。
-    true_values 和 false_values：定义哪些值会被视为 True 或 False。
-    skiprows：可以指定要跳过的行，可以是一个整数序列、一个整数或一个函数。
-    nrows：可以指定要读取的行数。
-    na_values：可以指定哪些值会被视为 NaN。
-    keep_default_na：决定是否使用默认的 NaN 值。
-    na_filter：决定是否过滤 NaN 值。
-    verbose：决定是否输出详细信息。
-    parse_dates：决定是否解析日期，可以是一个列表、字典或布尔值。
-    date_parser：自定义的日期解析函数。
-    date_format：日期的格式设置。
-    thousands：千位分隔符。
-    decimal：小数点分隔符。
-    comment：注释字符，以该字符开头的行将被跳过。
-    skipfooter：指定要跳过的文件末尾的行数。
-    storage_options：存储选项。
-    dtype_backend：数据类型后端。
-    engine_kwargs：传递给引擎的额外参数。
-    @param sheet_name:
-
-
-    """
-    try:
-        if excel_data is not None and file_name is not None:
-            pass
-        elif file_name is not None:
-            excel_data = os.path.join(os.path.dirname(os.getcwd()) + "/check_file/" + file_name)
-
-        dfs = read_excel_data(excel_data)
-        rows_list = []
-        # 多sheet处理
-        for name, df in dfs.items():
-            rows_list.append(df.values.tolist())
-        if len(dfs) <= 1:
-            rows_list = rows_list[0]
-        return rows_list
-    except Exception as e:
-        print(f"excel写入list时出错：{e}")
-
-
-def read_excel_and_write_to_csv(excel_data, file_name, **kwargs):
-    """excel内容并写入到csv中"""
-    try:
-        df = pd.read_excel(excel_data, engine="openpyxl")
-        local_csv_path = os.path.join(os.path.dirname(os.getcwd()) + "/temp_file/" + file_name)
-        df.to_csv(local_csv_path, index=False, **kwargs)
-        return local_csv_path
-    except Exception as e:
-        print(f"excel写入csv时出错：{e}")
-
-
-def read_excel_data_for_oss_write_to_dict(oss, **kwargs) -> dict:
-    """
-    1、根据oss link 直接读出 dict-list
-    2、支持多sheet，默认sheet_name =None查全部
-    3、返回dict结构 {'sheet_name':[rows_list]}
-    """
-    try:
-        dfs = read_excel_data(read_excel_from_oss(oss))
-        result = {}
-        for sheet_name, df in dfs.items():
-            rows_list = df.values.tolist()
-            result[sheet_name] = rows_list
-        return result
-    except Exception as e:
-        print(f"excel生成dict出错：{e}")
-
-
-def read_excel_header_for_oss_write_to_list(oss) -> list:
-    """
-    1、根据oss link 直接读出excel的头列 list
-    """
-    try:
-        dfs = read_excel_data(read_excel_from_oss(oss))
-        result = []
-        for sheet_name, df in dfs.items():
-            result.append(df.keys().values.tolist())
-        return result
-    except Exception as e:
-        print(f"excel生成header-dict出错：{e}")
-
-
 def check_excel(check_type="content", **kwargs):
     """对比excel
     :param: type: 需要对比类型，
@@ -160,18 +25,22 @@ def check_excel(check_type="content", **kwargs):
                 return check_excel_content(actual=kwargs["actual"], expected=kwargs["expected"])
             else:
                 return check_excel_content(
-                    actual=read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["actual_oss"])),
-                    expected=read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["expected_oss"]))
+                    actual=read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["actual_oss"]),
+                                                        **kwargs),
+                    expected=read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["expected_oss"]),
+                                                          **kwargs)
                 )
         elif check_type == "excelName":
             return check_excel_name(actual_oss=kwargs["actual_oss"], expected_oss=kwargs["expected_oss"])
         elif check_type == "all":
-            actual_content = read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["actual_oss"]))
-            expected_content = read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["expected_oss"]))
+            actual_content = read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["actual_oss"]),
+                                                          **kwargs)
+            expected_content = read_excel_and_write_to_list(excel_data=read_excel_from_oss(url=kwargs["expected_oss"]),
+                                                            **kwargs)
             flag1, content_result = check_excel_content(actual=actual_content, expected=expected_content)
             flag2, name_result = check_excel_name(actual_oss=kwargs["actual_oss"], expected_oss=kwargs["expected_oss"])
-            flag3 = check_excel_header(actual_oss=kwargs["actual_oss"], expected_oss=kwargs["expected_oss"])
-            return all([flag1, flag2, flag3]), {"文件名称": name_result, "导出内容": content_result}
+            flag3, header_result = check_excel_header(actual=actual_content, expected=expected_content, **kwargs)
+            return all([flag1, flag2, flag3]), {"文件名称": name_result, "导出内容": content_result, "表头比较:": header_result}
         else:
             return False, f"不支持此类型: {check_type}"
     except Exception as e:
@@ -179,51 +48,58 @@ def check_excel(check_type="content", **kwargs):
         return False, [e]
 
 
-# 定义比较类型和对应处理函数的映射
-comparison_functions = {
-    # 内容
-    "content": lambda kwargs: check_excel_content(kwargs["actual"], kwargs[
-        "expected"])
-    if "actual" in kwargs and "expected" in kwargs
-    else check_excel_content(kwargs["actual_oss"], kwargs["expected_oss"]),
-    # excelName
-    "excelName": lambda kwargs: check_excel_name(kwargs["actual_oss"], kwargs["expected_oss"]),
-    'header': lambda kwargs: check_excel_header(kwargs["actual_oss"], kwargs["expected_oss"]),
-    # 全部
-    "all": lambda kwargs: check_excel_all(kwargs["actual_oss"], kwargs["expected_oss"])
-}
-
-
-def check_excel_for_lu(check_type="content", **kwargs):
-    """对比excel
-    :param: type: 需要对比类型，
-            枚举：
-            content：对比两表格内容
-                方式1：传参actual_oss和expected_oss，参数类型str,url
-                放松1：传参actual和expected，参数类型list or dict
-            excelName: 对比两表格文件名称，传oss链接
-            all: 对比所有内容，传oss链接
+# # 定义比较类型和对应处理函数的映射
+# comparison_functions = {
+#     # 内容
+#     "content": lambda kwargs: check_excel_content(kwargs["actual"], kwargs["expected"]),
+#     # excelName
+#     "excelName": lambda kwargs: check_excel_name(kwargs["actual_oss"], kwargs["expected_oss"]),
+#     'header': lambda kwargs: check_excel_header(kwargs["actual"], kwargs["expected"]),
+#     # 全部
+#     "all": lambda kwargs: check_excel_all(kwargs["actual_oss"], kwargs["expected_oss"])
+# }
+#
+#
+# def check_excel_for_lu(check_type="content", **kwargs):
+#     """对比excel
+#     :param: type: 需要对比类型，
+#             枚举：
+#             content：对比两表格内容
+#                 方式1：传参actual_oss和expected_oss，参数类型str,url
+#                 放松1：传参actual和expected，参数类型list or dict
+#             excelName: 对比两表格文件名称，传oss链接
+#             all: 对比所有内容，传oss链接
+#     """
+#     try:
+#         # 根据 check_type 获取对应的处理函数
+#         compare_func = comparison_functions.get(check_type)
+#         if compare_func:
+#             return compare_func(kwargs)
+#         else:
+#             return False, f"不支持此类型: {check_type}"
+#     except KeyError as ke:
+#         # raise ke
+#         print(f"类型对应参数缺失异常：{ke}")
+#         return False, [str(ke)]
+#     except Exception as e:
+#         print(f"对比 Excel 异常：{e}")
+#         return False, [str(e)]
+
+
+def check_excel_content_form_dict(actual, expected, **kwargs):
     """
-    try:
-        # 根据 check_type 获取对应的处理函数
-        compare_func = comparison_functions.get(check_type)
-        if compare_func:
-            return compare_func(kwargs)
-        else:
-            return False, f"不支持此类型: {check_type}"
-    except KeyError as ke:
-        # raise ke
-        print(f"类型对应参数缺失异常：{ke}")
-        return False, [str(ke)]
-    except Exception as e:
-        print(f"对比 Excel 异常：{e}")
-        return False, [str(e)]
+    通过 OSS URL 比较 Excel 内容,支持多sheet并且包含表头
+    """
+    actual, expected = read_excel_and_write_to_dict(actual, **kwargs), read_excel_and_write_to_dict(
+        expected, **kwargs)
+    return check_excel_content(actual, expected)
 
 
-def check_excel_content_form_oss(actual_oss, expected_oss):
-    """通过 OSS URL 比较 Excel 内容"""
-    expected, actual = read_excel_and_write_to_list(read_excel_from_oss(expected_oss)), read_excel_and_write_to_list(
-        read_excel_from_oss(actual_oss))
+def check_excel_content_form_list(actual, expected):
+    """
+    通过 内容 比较 Excel 内容,不包含表头
+    """
+    expected, actual = read_excel_and_write_to_list(actual), read_excel_and_write_to_list(expected)
     return check_excel_content(actual=actual, expected=expected)
 
 
@@ -231,11 +107,31 @@ def check_excel_all(actual_oss, expected_oss):
     """
     校验所有内容
     """
-    flag1, content_result = check_excel_content_form_oss(actual_oss, expected_oss)
+    file_type = check_and_get_file_suffix_name(expected_oss, actual_oss)
+    actual, expected = read_excel_from_oss(actual_oss), read_excel_from_oss(expected_oss)
+    actual_data_copy = copy.deepcopy(actual)
+    expected_data_copy = copy.deepcopy(expected)
+    flag1, content_result = check_excel_content_form_dict(actual, expected, type=file_type)
     flag2, name_result = check_excel_name(actual_oss, expected_oss)
-    flag3 = check_excel_header(actual_oss, expected_oss)
-    return all([flag1, flag2, flag3]), {"文件名称": name_result, "导出内容": content_result,
-                                        "校验结果": [flag1, flag2, flag3]}
+    flag3, header_result = check_excel_header(actual_data_copy,expected_data_copy, type=file_type)
+    return all([flag1, flag2]), json.dumps({f"文件名称-{flag1}": name_result, f"导出内容-{flag2}": content_result,f"表头校验-{flag3}": header_result},
+                                           ensure_ascii=False)
+
+
+def check_and_get_file_suffix_name(actual_oss, expected_oss) -> str:
+    """
+    校验并获取oss的后缀类型
+    @param actual_oss:
+    @param expected_oss:
+    @return:
+    """
+    actual_file_suffix_name = get_file_suffix_name(actual_oss)
+    expected_file_suffix_name = get_file_suffix_name(expected_oss)
+    try:
+        assert actual_file_suffix_name == expected_file_suffix_name
+        return actual_file_suffix_name
+    except Exception:
+        raise Exception("oss文件类型不一致,请检查oss链接是否为相同类型")
 
 
 def check_excel_name(actual_oss, expected_oss):
@@ -260,7 +156,6 @@ def check_excel_content(actual, expected):
        :param expected:预期内容：list或dict类型
      """
     try:
-        # TODO 嵌套list -dict 比较失败
         if actual == expected:
             return True, ["excel内容-完全匹配"]
         else:
@@ -281,28 +176,44 @@ def check_excel_content(actual, expected):
                 num = len(expected)
             else:
                 num = len(actual)
-            for i in range(num):
-                if actual[i] == expected[i]:
-                    continue
-                else:
-                    errors.append(
-                        "excel内容-第" + str(i + 1) + "行不匹配，预期为：" + str(expected[i]) + ", 实际为: " + str(actual[i]))
-            return False, errors
+            if isinstance(actual, list) and isinstance(expected, list):
+                for i in range(num):
+                    if actual[i] == expected[i]:
+                        continue
+                    else:
+                        errors.append(
+                            "excel内容-第" + str(i + 1) + "行不匹配，预期为：" + str(expected[i]) + ", 实际为: " + str(
+                                actual[i]))
+                return False, errors
+            else:
+                return False, compare_dicts(actual, expected)
     except Exception as e:
         print(f"：excel内容-服务异常{e}")
         return False, [e]
 
 
-def check_excel_header(actual_oss, expected_oss):
+def check_excel_header(actual, expected, **kwargs):
     """
     比较两个文档第一列的header是否一致
-    @param actual_oss:
-    @param expected_oss:
+    @param actual:
+    @param expected:
+    @return:
     @return:
     """
-    actual, expected = read_excel_data_for_oss_write_to_dict(actual_oss), read_excel_data_for_oss_write_to_dict(
-        expected_oss)
-    return actual == expected
+    try:
+        if all([isinstance(actual, str), isinstance(expected, str)]):
+            actual1, expected1 = read_excel_header(read_excel_from_oss(actual), **kwargs), read_excel_header(
+                read_excel_from_oss(
+                    expected), **kwargs)
+        else:
+            actual1, expected1 = read_excel_header(actual, **kwargs), read_excel_header(expected, **kwargs)
+        try:
+            assert actual1 == expected1
+            return True, "表头校验值与顺序一致"
+        except Exception as e:
+            return False, f"表头校验值与顺序失败 {e}"
+    except Exception as e:
+        return False, f"表头校验异常 {e}"
 
 
 def del_temp_file(file_name=""):
@@ -315,3 +226,34 @@ def del_temp_file(file_name=""):
         print(f"文件 {file_path} 不存在。")
     except Exception as e:
         print(f"删除文件 {file_path} 时出错：{e}")
+
+
+def compare_dicts(dict1, dict2):
+    diff = {}
+    # 找出只在 dict1 中存在的键
+    only_in_dict1 = set(dict1.keys()) - set(dict2.keys())
+    if only_in_dict1:
+        diff['only_in_dict1'] = {key: dict1[key] for key in only_in_dict1}
+    # 找出只在 dict2 中存在的键
+    only_in_dict2 = set(dict2.keys()) - set(dict1.keys())
+    if only_in_dict2:
+        diff['only_in_dict2'] = {key: dict2[key] for key in only_in_dict2}
+    # 处理两个字典都有的键
+    common_keys = set(dict1.keys()) & set(dict2.keys())
+    for key in common_keys:
+        value1 = dict1[key]
+        value2 = dict2[key]
+        if isinstance(value1, dict) and isinstance(value2, dict):
+            # 如果值是字典，递归比较
+            sub_diff = compare_dicts(value1, value2)
+            if sub_diff:
+                diff[f'different_sub_dicts_at_{key}'] = sub_diff
+        elif isinstance(value1, list) and isinstance(value2, list):
+            # 如果值是列表，比较列表元素
+            if value1 != value2:
+                diff[f'different_lists_at_{key}'] = (value1, value2)
+        else:
+            # 其他情况，直接比较值
+            if value1 != value2:
+                diff[f'different_values_at_{key}'] = (value1, value2)
+    return diff

smartpush/export/basic/GetOssUrl.py

@@ -1,9 +1,22 @@
-import requests
-from retry import retry
+# from retry import retry
 import json
+
+import requests
+import tenacity
+from tenacity import retry, stop_after_attempt, wait_fixed
+
 from smartpush.utils.StringUtils import StringUtils
 
 
+# 用于技术第几次重试，无需修改
+def log_attempt(retry_state):
+    """
+    回调函数，在每次重试时记录并打印重试次数
+    """
+    attempt_number = retry_state.attempt_number
+    print(f"当前重试次数: {attempt_number}")
+
+
 def get_oss_address_with_retry(target_id, url, requestHeader, requestParam, **kwargs) -> str:
     """
     创建带有动态重试配置的获取 OSS 地址
@@ -14,19 +27,26 @@ def get_oss_address_with_retry(target_id, url, requestHeader, requestParam, **kw
     :param requestHeader:
     :return: 带有重试配置的获取 OSS 地址的
     """
+    tries = kwargs.get('tries', 10)  # 重试次数
+    delay = kwargs.get('delay', 2)
+
 
-    @retry(tries=10, delay=2, backoff=1)
+    @retry(stop=stop_after_attempt(tries), wait=wait_fixed(delay), after=log_attempt)
     def get_oss_address():
+        _url = url + '/bulkOps/query'
+        result = None
         if StringUtils.is_empty(target_id):
             print(f"缺少参数:target_id")
             return
         try:
-            response = requests.request(url=url, headers=requestHeader, data=json.dumps(requestParam), method="post")
+            response = requests.request(url=_url, headers=requestHeader, data=json.dumps(requestParam),
+                                        method="post")
             response.raise_for_status()
             result = response.json()
             id_url_dict = {item["id"]: item["url"] for item in result["resultData"]["datas"]}
             if target_id in id_url_dict:
                 if len(id_url_dict[target_id]) == 1:
+                    print(f"{target_id} oss链接为：{id_url_dict[target_id][0]}")
                     return id_url_dict[target_id][0]
                 else:
                     raise ValueError(f"存在多条 id 为 {target_id} 的记录，记录为：{id_url_dict[target_id]}")
@@ -34,4 +54,45 @@ def get_oss_address_with_retry(target_id, url, requestHeader, requestParam, **kw
                 raise ValueError(f"未找到 id 为 {target_id} 的记录，未包含有效的 OSS 地址,")
         except (KeyError, json.JSONDecodeError) as e:
             raise ValueError(f"响应数据格式错误,响应结果: {result},异常: {e}")
-    return get_oss_address()
+        except requests.RequestException as e:
+            print(f"请求发生异常: {e}，正在重试...")
+            raise
+
+    def cancel_export_file(_target_id):
+        """
+        用于失败后取消导出
+        :param _target_id:
+        :return:
+        """
+        cancel_url = url + '/bulkOps/cancel'
+        response = requests.request(url=cancel_url, headers=requestHeader, params={'id': _target_id}, method="get")
+        response.raise_for_status()
+        result = response.json()
+        print(f"获取Oss Url失败，取消 {_target_id} 的导出记录，响应：{result}")
+        return result
+
+    try:
+        return get_oss_address()
+    except Exception as e:
+        # print(f"最终失败，错误信息: {e}")
+        if isinstance(e, tenacity.RetryError):
+            cancel_export_file(target_id)
+        return None
+
+
+if __name__ == '__main__':
+    url = "https://test.smartpushedm.com/api-em-ec2"
+    requestHeader = {
+        "cookie": "osudb_appid=SMARTPUSH;osudb_oar=#01#SID0000122BBLon+0gwvStide+qtdJAK57ZSK1ty+iW8b7tv/Uwl6Zo4gDfUg6B83n+jgqTVjoZ5qRGyRsuLaXc9woDN2WRh3mu1yn7anglBmaFoemhCy/ttS8nqv/y0kj8khbu6mtBmQrseNfnO/Mir8PQP+S;osudb_subappid=1;osudb_uid=4213785247;ecom_http_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NDI1Mjk2NTQsImp0aSI6ImM2MTA4MGJkLTU4MGUtNDJiNi05NzU5LTU0ZTNmZDExZDA4OSIsInVzZXJJbmZvIjp7ImlkIjowLCJ1c2VySWQiOiI0MjEzNzg1MjQ3IiwidXNlcm5hbWUiOiIiLCJlbWFpbCI6ImZlbGl4LnNoYW9Ac2hvcGxpbmVhcHAuY29tIiwidXNlclJvbGUiOiJvd25lciIsInBsYXRmb3JtVHlwZSI6Nywic3ViUGxhdGZvcm0iOjEsInBob25lIjoiIiwibGFuZ3VhZ2UiOiJ6aC1oYW5zLWNuIiwiYXV0aFR5cGUiOiIiLCJhdHRyaWJ1dGVzIjp7ImNvdW50cnlDb2RlIjoiQ04iLCJjdXJyZW5jeSI6IkpQWSIsImN1cnJlbmN5U3ltYm9sIjoiSlDCpSIsImRvbWFpbiI6InNtYXJ0cHVzaDQubXlzaG9wbGluZXN0Zy5jb20iLCJsYW5ndWFnZSI6ImVuIiwibWVyY2hhbnRFbWFpbCI6ImZlbGl4LnNoYW9Ac2hvcGxpbmUuY29tIiwibWVyY2hhbnROYW1lIjoiU21hcnRQdXNoNF9lYzJf6Ieq5Yqo5YyW5bqX6ZO6IiwicGhvbmUiOiIiLCJzY29wZUNoYW5nZWQiOmZhbHNlLCJzdGFmZkxhbmd1YWdlIjoiemgtaGFucy1jbiIsInN0YXR1cyI6MCwidGltZXpvbmUiOiJBc2lhL01hY2FvIn0sInN0b3JlSWQiOiIxNjQ0Mzk1OTIwNDQ0IiwiaGFuZGxlIjoic21hcnRwdXNoNCIsImVudiI6IkNOIiwic3RlIjoiIiwidmVyaWZ5IjoiIn0sImxvZ2luVGltZSI6MTczOTkzNzY1NDc2Mywic2NvcGUiOlsiZW1haWwtbWFya2V0IiwiY29va2llIiwic2wtZWNvbS1lbWFpbC1tYXJrZXQtbmV3LXRlc3QiLCJlbWFpbC1tYXJrZXQtbmV3LWRldi1mcyIsImFwaS11Yy1lYzIiLCJhcGktc3UtZWMyIiwiYXBpLWVtLWVjMiIsImZsb3ctcGx1Z2luIl0sImNsaWVudF9pZCI6ImVtYWlsLW1hcmtldCJ9.X2Birt-jiWILAvEjjwknUchil2ys8Y11omeRYgZ3K0I;",
+        "Content-Type": "application/json"
+    }
+    requestParam = {
+        "page": 1,
+        "pageSize": 20,
+        "type": "EXPORT",
+        "status": None,
+        "startTime": 1740033265288,
+        "endTime": 1740044065288
+    }
+    id = "2334659"
+    get_oss_address_with_retry(2334659, url, requestHeader, requestParam)

smartpush/export/basic/ReadExcel.py

@@ -0,0 +1,139 @@
+import io
+import os
+import re
+import warnings
+from io import BytesIO
+import pandas as pd
+from requests import request
+
+warnings.simplefilter("ignore")
+excel_extensions = ['.xlsb', '.xlsx', '.xlsm', '.xls', '.xltx', '.xltm', '.xlam', None]
+csv_extensions = ['.csv']
+
+
+def read_excel_from_oss(url="", method="get"):
+    """读取oss的excel内容转为io流数据"""
+    try:
+        result = request(method=method, url=url)
+        excel_data = BytesIO(result.content)
+        print(f"成功读取oss文件内容: {url}")
+        return excel_data
+    except Exception as e:
+        print(f"读取oss报错 {url} 时出错：{e}")
+
+
+def read_excel_header(excel_data, **kwargs) -> list:
+    """
+    1、读出excel的头列 list
+    """
+    try:
+        dfs = read_excel_csv_data(excel_data,**kwargs)
+        result = []
+        if kwargs['type'] in excel_extensions:
+            for sheet_name, df in dfs.items():
+                result.append(df.keys().values.tolist())
+        else:
+            result = dfs.keys().values.tolist()
+        return result
+    except Exception as e:
+        print(f"excel生成header-list出错：{e}")
+        raise
+
+
+def read_excel_csv_data(excel_data, **kwargs):
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning, module=re.escape('openpyxl.styles.stylesheet'))
+        if kwargs['type'] in excel_extensions:
+            dfs = pd.read_excel(excel_data, sheet_name=None, na_filter=False) if isinstance(excel_data,
+                                                                                                      io.BytesIO) \
+                else excel_data
+        else:
+            dfs = pd.read_csv(excel_data, na_filter=False)
+        return dfs
+
+
+def read_excel_and_write_to_dict(excel_data=None, file_name=None, **kwargs):
+    """excel内容并写入到内存dict中
+    :param excel_data：excel的io对象, 参数和file_name互斥
+    :file_name: excel文件名称，目前读取check_file目录下文件，参数和excel_data互斥
+    """
+    try:
+        if excel_data is not None and file_name is not None:
+            pass
+        elif file_name is not None:
+            excel_data = os.path.join(os.path.dirname(os.getcwd()) + "/check_file/" + file_name)
+        dfs = read_excel_csv_data(excel_data, **kwargs)
+        if kwargs['type'] in excel_extensions:
+            # 将DataFrame转换为字典，以行为单位存储数据
+            row_dict = {}  # 创建一个空字典来存储按行转换的数据
+            for sheet_name, row in dfs.items():
+                row_dict[sheet_name] = row.to_dict(orient='records')
+        else:
+            row_dict = dfs.to_dict()
+        return row_dict
+    except Exception as e:
+        print(f"excel写入dict时出错：{e}")
+
+
+def read_excel_and_write_to_list(excel_data=None, sheet_name=None, file_name=None, **kwargs):
+    """excel内容并写入到内存list中
+    :param excel_data：excel的io对象, 参数和file_name互斥
+    :file_name: excel文件名称，目前读取check_file目录下文件，参数和excel_data互斥
+    """
+    try:
+        if excel_data is not None and file_name is not None:
+            pass
+        elif file_name is not None:
+            excel_data = os.path.join(os.path.dirname(os.getcwd()) + "/check_file/" + file_name)
+        dfs = read_excel_csv_data(excel_data, **kwargs)
+        rows_list = []
+        # 多sheet处理
+        for name, df in dfs.items():
+            rows_list.append(df.values.tolist())
+        if len(dfs) <= 1:
+            rows_list = rows_list[0]
+        return rows_list
+    except Exception as e:
+        print(f"excel写入list时出错：{e}")
+
+
+def read_excel_and_write_to_csv(excel_data, file_name, **kwargs):
+    """excel内容并写入到csv中"""
+    try:
+        df = pd.read_excel(excel_data, engine="openpyxl")
+        local_csv_path = os.path.join(os.path.dirname(os.getcwd()) + "/temp_file/" + file_name)
+        df.to_csv(local_csv_path, index=False, **kwargs)
+        return local_csv_path
+    except Exception as e:
+        print(f"excel写入csv时出错：{e}")
+
+
+def read_excel_data_for_oss_write_to_dict(oss, **kwargs) -> dict:
+    """
+    1、根据oss link 直接读出 dict-list
+    2、支持多sheet，默认sheet_name =None查全部
+    3、返回dict结构 {'sheet_name':[rows_list]}
+    """
+    try:
+        dfs = read_excel_csv_data(read_excel_from_oss(oss), **kwargs)
+        result = {}
+        for sheet_name, df in dfs.items():
+            rows_list = df.values.tolist()
+            result[sheet_name] = rows_list
+        return result
+    except Exception as e:
+        print(f"excel生成dict出错：{e}")
+
+
+# 从 URL 中提取文件名
+def get_file_suffix_name(url):
+    last_filename = ""
+    filename = url.split("/")[-1]
+    # 从文件名中提取文件后缀
+    if '.' in filename:
+        last_filename = '.' + filename.split('.')[-1].lower()
+    support_types = excel_extensions + csv_extensions
+    if last_filename in support_types:
+        return last_filename
+    else:
+        raise Exception(f"[{last_filename}] 该类型暂不支持！目前只支持 {support_types}")

smartpush/test.py

@@ -1,18 +1,19 @@
-from smartpush.export.basic.ExcelExportChecker import *
+# -*- codeing = utf-8 -*-
+# @Time :2025/2/20 00:27
+# @Author :luzebin
+from smartpush.export.basic.ExcelExportChecker import check_excel_all
+from smartpush.export.basic.ReadExcel import read_excel_from_oss
+from smartpush.export.basic.ReadExcel import read_excel_and_write_to_dict
 from smartpush.export.basic.GetOssUrl import get_oss_address_with_retry
 
-import urllib3
-
 if __name__ == '__main__':
-    # print(check_excel_for_lu("content",actual_oss="https://sl-smartfile.oss-ap-southeast-1.aliyuncs.com/material_ec2_prod/2025-01-20/fcb98e2965314ef2862db65760dcce1f/ab%E5%BC%B9%E7%AA%97%E6%B4%BB%E5%8A%A8-%E8%BD%AC%E5%8C%96%E7%8E%87%E8%8E%B7%E8%83%9C%E9%94%80%E5%94%AE%E9%A2%9D%E6%98%8E%E7%BB%86%E6%95%B0%E6%8D%AE.xlsx",expected_oss="https://sl-smartfile.oss-ap-southeast-1.aliyuncs.com/material_ec2_prod/2025-01-20/fcb98e2965314ef2862db65760dcce1f/ab%E5%BC%B9%E7%AA%97%E6%B4%BB%E5%8A%A8-%E8%BD%AC%E5%8C%96%E7%8E%87%E8%8E%B7%E8%83%9C%E9%94%80%E5%94%AE%E9%A2%9D%E6%98%8E%E7%BB%86%E6%95%B0%E6%8D%AE.xlsx"))
-    _id = 10901
-    url = "https://test.smartpushedm.com/api-em-ec2/bulkOps/query"
-    requestHeaders = {
-            'cookie': 'osudb_appid=SMARTPUSH;osudb_oar=#01#SID0000121BJe/0W0PdWQj0Wo/Cr4G9H5S58u/YpvUYbOxsyvHQXmU5iToD8h3GX0+/3Af1efOroDv2jIwJIPVx2F1/XCP08l/NOaWMIZ/xm1/ugKB7eA1k1akIdCSTOHJcJ95Ahp7Yz0cBgOwtr8OgF77WNxX;osudb_subappid=1;osudb_uid=4213785247;ecom_http_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NDIwMTEyNDcsImp0aSI6IjY4M2E1NDg1LTEwYjAtNDRhZS04MGMxLWQ1MmFkN2YxNmViNCIsInVzZXJJbmZvIjp7ImlkIjowLCJ1c2VySWQiOiI0MjEzNzg1MjQ3IiwidXNlcm5hbWUiOiIiLCJlbWFpbCI6ImZlbGl4LnNoYW9Ac2hvcGxpbmVhcHAuY29tIiwidXNlclJvbGUiOiJvd25lciIsInBsYXRmb3JtVHlwZSI6Nywic3ViUGxhdGZvcm0iOjEsInBob25lIjoiIiwibGFuZ3VhZ2UiOiJ6aC1oYW5zLWNuIiwiYXV0aFR5cGUiOiIiLCJhdHRyaWJ1dGVzIjp7ImNvdW50cnlDb2RlIjoiQ04iLCJjdXJyZW5jeSI6IkpQWSIsImN1cnJlbmN5U3ltYm9sIjoiSlDCpSIsImRvbWFpbiI6InNtYXJ0cHVzaDQubXlzaG9wbGluZXN0Zy5jb20iLCJsYW5ndWFnZSI6ImVuIiwibWVyY2hhbnRFbWFpbCI6ImZlbGl4LnNoYW9Ac2hvcGxpbmUuY29tIiwibWVyY2hhbnROYW1lIjoiU21hcnRQdXNoNF9lYzJf6Ieq5Yqo5YyW5bqX6ZO6IiwicGhvbmUiOiIiLCJzY29wZUNoYW5nZWQiOmZhbHNlLCJzdGFmZkxhbmd1YWdlIjoiemgtaGFucy1jbiIsInN0YXR1cyI6MCwidGltZXpvbmUiOiJBc2lhL01hY2FvIn0sInN0b3JlSWQiOiIxNjQ0Mzk1OTIwNDQ0IiwiaGFuZGxlIjoic21hcnRwdXNoNCIsImVudiI6IkNOIiwic3RlIjoiIiwidmVyaWZ5IjoiIn0sImxvZ2luVGltZSI6MTczOTQxOTI0Nzg2OSwic2NvcGUiOlsiZW1haWwtbWFya2V0IiwiY29va2llIiwic2wtZWNvbS1lbWFpbC1tYXJrZXQtbmV3LXRlc3QiLCJlbWFpbC1tYXJrZXQtbmV3LWRldi1mcyIsImFwaS11Yy1lYzIiLCJhcGktc3UtZWMyIiwiYXBpLWVtLWVjMiIsImZsb3ctcGx1Z2luIl0sImNsaWVudF9pZCI6ImVtYWlsLW1hcmtldCJ9.PJGM1sSZyvxTriMK4e1g90krqBUq9OVNc5vEyKxsXyQ;',
-            'Content-Type': 'application/json'}
-    requestParams = {'page': 1, 'pageSize': 10, 'type': 'EXPORT', 'status': None, 'startTime': None, 'endTime': None}
-    oss=get_oss_address_with_retry(_id, url, requestHeaders, requestParams, tries=1, delay=1, backoff=1)
-    check_excel_for_lu("all",oss,oss)
-    # oss = "https://cdn.smartpushedm.com/material_sf/2025-02-19/c70bb6f3588b4c89bde6ed75813ea4f3/%E5%AF%BC%E5%87%BA%E5%85%A8%E9%83%A8%E5%AE%A2%E6%88%B7.csv"
-    # eexcelxcel =read_excel_and_write_to_list(read_excel_from_oss(oss))
-    print(eexcelxcel)
+    oss1 = "https://cdn.smartpushedm.com/material_ec2/2025-02-19/4d98418295524ab1b52340c2ed2afa4a/AutoTest-%E5%9B%BA%E5%AE%9AB-2025-02-14%20%E5%88%9B%E5%BB%BA%E7%9A%84Email33%E6%95%B0%E6%8D%AE%E6%A6%82%E8%A7%88.xlsx"
+    # oss2 = "https://cdn.smartpushedm.com/material_ec2/2025-02-19/ddbe9965d83840199e678a66dc414518/%E8%90%A5%E9%94%80%E4%BB%BB%E5%8A%A1%E6%95%B0%E6%8D%AE%E6%A6%82%E8%A7%88.xlsx"
+    print(check_excel_all(oss1, oss1))
+
+    # expected_oss ="https://cdn.smartpushedm.com/material_ec2_prod/2025-02-20/dae941ec20964ca5b106407858676f89/%E7%BE%A4%E7%BB%84%E6%95%B0%E6%8D%AE%E6%A6%82%E8%A7%88.xlsx"
+    # actual_oss = "https://cdn.smartpushedm.com/material_ec2_prod/2025-02-20/dae941ec20964ca5b106407858676f89/%E7%BE%A4%E7%BB%84%E6%95%B0%E6%8D%AE%E6%A6%82%E8%A7%88.xlsx"
+    # #actual_oss= get_oss_address_with_retry("23161","https://cdn.smartpushedm.com/material_ec2_prod/2025-02-20/dae941ec20964ca5b106407858676f89/%E7%BE%A4%E7%BB%84%E6%95%B0%E6%8D%AE%E6%A6%82%E8%A7%88.xlsx","",'{"page":1,"pageSize":10,"type":null,"status":null,"startTime":null,"endTime":null}')
+    # res=read_excel_and_write_to_dict(read_excel_from_oss(actual_oss))
+    # print(res)
+    # print(check_excel_all(actual_oss,expected_oss))

{smartpush-1.1.1.dist-info → smartpush-1.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: smartpush
-Version: 1.1.1
+Version: 1.1.4
 Summary: 用于smartpush自动化测试工具包
 Author: 卢泽彬、邵宇飞、周彦龙
 

smartpush-1.1.4.dist-info/RECORD

@@ -0,0 +1,14 @@
+smartpush/__init__.py,sha256=XJrl1vhGATHSeSVqKmPXxYqxyseriUpvY5tLIXir3EE,24
+smartpush/get_jira_info.py,sha256=dmCwkKa94xwyE2hegE1KBI3cV_LbrJ67P9osORUGPt4,2633
+smartpush/test.py,sha256=m_Y67qrUeFphwM5jqBOTqheAufmnnjnBVF71nXRXUxg,1598
+smartpush/export/__init__.py,sha256=D9GbWcmwnetEndFDty5XbVienFK1WjqV2yYcQp3CM84,99
+smartpush/export/basic/ExcelExportChecker.py,sha256=anvovO657rmo2PF8Mbn_BLFlhSOdyCut4yYoLFUK85c,10807
+smartpush/export/basic/GetOssUrl.py,sha256=K64PNHNpshYp0ag37IS-wjViCSKS8G4v8AuafHgM31Y,4946
+smartpush/export/basic/ReadExcel.py,sha256=rGPDq36ujLKA1dlz_33iQQU6I8dhlc5l2ij_JAQX2a0,5145
+smartpush/export/basic/__init__.py,sha256=6tcrS-2NSlsJo-UwEsnGUmwCf7jgOsh_UEbM0FD-gYE,70
+smartpush/utils/StringUtils.py,sha256=NXomJ4qmyBRAFnGj5hrFRWwQnRQMTcPzy20fk1dunSw,3980
+smartpush/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+smartpush-1.1.4.dist-info/METADATA,sha256=pg3q4VEGLs7d_rDvijLlWHcmOFD4WNF-el-qQzknCvE,145
+smartpush-1.1.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+smartpush-1.1.4.dist-info/top_level.txt,sha256=5_CXqu08EfbPaKLjuSAOAqCmGU6shiatwDU_ViBGCmg,10
+smartpush-1.1.4.dist-info/RECORD,,

smartpush-1.1.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-smartpush/__init__.py,sha256=XJrl1vhGATHSeSVqKmPXxYqxyseriUpvY5tLIXir3EE,24
-smartpush/get_jira_info.py,sha256=dmCwkKa94xwyE2hegE1KBI3cV_LbrJ67P9osORUGPt4,2633
-smartpush/test.py,sha256=FTfTwcJEZFPz_Rq8pAooOW_qBN1sbVQbm2htF8mgqDo,2939
-smartpush/export/__init__.py,sha256=D9GbWcmwnetEndFDty5XbVienFK1WjqV2yYcQp3CM84,99
-smartpush/export/basic/ExcelExportChecker.py,sha256=Y4agi5F8j1sIgM10Z3yKKs6y7rg4p6-6GSyKMpEITzs,13173
-smartpush/export/basic/GetOssUrl.py,sha256=O2-HtcYzbabck9dKgLu8ga21_AiyDIzgdfoDgvqBY3c,1541
-smartpush/export/basic/__init__.py,sha256=6tcrS-2NSlsJo-UwEsnGUmwCf7jgOsh_UEbM0FD-gYE,70
-smartpush/utils/StringUtils.py,sha256=NXomJ4qmyBRAFnGj5hrFRWwQnRQMTcPzy20fk1dunSw,3980
-smartpush/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-smartpush-1.1.1.dist-info/METADATA,sha256=5tCwRmx5MPLlVdUC695sEQEUDgcVr5TIZJeSVakIJr4,145
-smartpush-1.1.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-smartpush-1.1.1.dist-info/top_level.txt,sha256=5_CXqu08EfbPaKLjuSAOAqCmGU6shiatwDU_ViBGCmg,10
-smartpush-1.1.1.dist-info/RECORD,,