tfduck-bsd 0.18.2__tar.gz → 0.18.3__tar.gz

{tfduck-bsd-0.18.2/tfduck_bsd.egg-info → tfduck-bsd-0.18.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.2
+Version: 0.18.3
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.18.2 → tfduck-bsd-0.18.3}/setup.py

@@ -8,7 +8,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="tfduck-bsd",
-    version="0.18.2",
+    version="0.18.3",
     author="yuanxiao",
     author_email="yuan6785@163.com",
     description="A small example package",

tfduck-bsd-0.18.3/tfduck/__init__.py

@@ -0,0 +1 @@
+__version__="0.18.3"

{tfduck-bsd-0.18.2 → tfduck-bsd-0.18.3}/tfduck/tga/tga.py

@@ -11,9 +11,10 @@ finally:
     BMOBJ.remove_file(local_file)
 
 版本记录:
-pyhive=0.6.2 
+pyhive=0.6.2
 requests=2.23.0 2.27.1
 """
+
 import requests
 import pandas
 import json
@@ -22,16 +23,18 @@ import os
 import uuid
 import urllib3
 from tfduck.common.defines import BMOBJ, Et
+
 # from django.conf import settings
 from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, FIRST_COMPLETED
 from tfduck.tga.base_tga import BaseTga
 
+
 class ThinkDataQuery(BaseTga):
     """
     @des: thinkdata openapi查询基础类----这个只能再thinkdata内网执行
     """
 
-    def __init__(self, query_uri, token, hive_conn_info=['host', 0]):
+    def __init__(self, query_uri, token, hive_conn_info=["host", 0]):
         """
         @des:初始化类
         """
@@ -52,53 +55,81 @@ class ThinkDataQuery(BaseTga):
         file_path = os.path.join(base_dir, real_name)
         return file_path
 
-    def g_to_csv_notmp(self, filepath, df, index=True, compression=None, mode='w', header=True):
+    def g_to_csv_notmp(
+        self, filepath, df, index=True, compression=None, mode="w", header=True
+    ):
         """
         @des: pandas生成csv文件---用于追加文件，不能用临时文件
-        compression: 压缩格式 ‘gzip’, ‘bz2’, ‘zip’, ‘xz’. 
+        compression: 压缩格式 ‘gzip’, ‘bz2’, ‘zip’, ‘xz’.
         """
         tmp_filepath = filepath
         if index is None:  # 不保存行索引
             if compression is None:  # 不压缩
                 df.to_csv(tmp_filepath, index=None, mode=mode, header=header)
             else:
-                df.to_csv(tmp_filepath, index=None,
-                          compression=compression, mode=mode, header=header)
+                df.to_csv(
+                    tmp_filepath,
+                    index=None,
+                    compression=compression,
+                    mode=mode,
+                    header=header,
+                )
         else:
             if compression is None:  # 不压缩
                 df.to_csv(tmp_filepath, mode=mode, header=header)
             else:
-                df.to_csv(tmp_filepath, compression=compression,
-                          mode=mode, header=header)
+                df.to_csv(
+                    tmp_filepath, compression=compression, mode=mode, header=header
+                )
         return True
 
-    def get_data_csv_i(self, ctx, unique_path, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, conn_timeout=30, tga_data_timeout=600):
+    def get_data_csv_i(
+        self,
+        ctx,
+        unique_path,
+        sql,
+        block_size=100000,
+        print_size=100000,
+        read_timeout=600,
+        upcount=None,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
         session = requests.session()
-        post_data = {'token': self.token, 'sql': sql, 'timeoutSeconds': tga_data_timeout}
+        post_data = {
+            "token": self.token,
+            "sql": sql,
+            "timeoutSeconds": tga_data_timeout,
+        }
         #
         unique_path = self.gen_local_unique_file()
         #
         BMOBJ.log_error("in query")
         #
-        r = session.post(self.query_uri, data=post_data, stream=True,
-                         verify=False, timeout=(conn_timeout, read_timeout))
+        r = session.post(
+            self.query_uri,
+            data=post_data,
+            stream=True,
+            verify=False,
+            timeout=(conn_timeout, read_timeout),
+        )
         datas = []
-        i = 0    # 循环引用计数
+        i = 0  # 循环引用计数
         icount = 0  # 数据的数量
         cols = []  # 表头
         try:
-            row = ''
+            row = ""
             # iter_lines iter_content, chunk_size字节, 下面取100M
-            for row in r.iter_lines(chunk_size=1024*1024*100):
+            for row in r.iter_lines(chunk_size=1024 * 1024 * 100):
                 if not row:
                     continue
                 data = None
                 if i == 0:  # 处理header
                     data = json.loads(row)
-                    if(data["return_code"] == 0):
+                    if data["return_code"] == 0:
                         cols = data["data"]["headers"]
                         df = pandas.DataFrame(data=[], columns=cols)  # 保存表头
                         self.g_to_csv_notmp(unique_path, df, index=None)
@@ -121,8 +152,9 @@ class ThinkDataQuery(BaseTga):
                 i += 1
                 if len(datas) == block_size:  # 1000000条保存一次
                     df = pandas.DataFrame(data=datas, columns=cols)  # 保存表头
-                    self.g_to_csv_notmp(unique_path, df, index=None,
-                                        mode='a', header=False)  # 追加保存
+                    self.g_to_csv_notmp(
+                        unique_path, df, index=None, mode="a", header=False
+                    )  # 追加保存
                     icount += block_size
                     datas = []
                 if i % print_size == 0:
@@ -130,8 +162,9 @@ class ThinkDataQuery(BaseTga):
             BMOBJ.clog(ctx, f"total: {i}")
             if len(datas) > 0:  # 保存最后收尾的
                 df = pandas.DataFrame(data=datas, columns=cols)  # 保存表头
-                self.g_to_csv_notmp(unique_path, df, index=None,
-                                    mode='a', header=False)  # 追加保存
+                self.g_to_csv_notmp(
+                    unique_path, df, index=None, mode="a", header=False
+                )  # 追加保存
                 icount += len(datas)
                 datas = []
         except Exception as e:
@@ -152,7 +185,18 @@ class ThinkDataQuery(BaseTga):
                 pass
         return unique_path
 
-    def get_data_csv(self, ctx, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, retry_count=2, conn_timeout=30, tga_data_timeout=600):
+    def get_data_csv(
+        self,
+        ctx,
+        sql,
+        block_size=100000,
+        print_size=100000,
+        read_timeout=600,
+        upcount=None,
+        retry_count=2,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
@@ -162,7 +206,16 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_csv_i(
-                    ctx, unique_path, sql, block_size, print_size, read_timeout, upcount, conn_timeout, tga_data_timeout)
+                    ctx,
+                    unique_path,
+                    sql,
+                    block_size,
+                    print_size,
+                    read_timeout,
+                    upcount,
+                    conn_timeout,
+                    tga_data_timeout,
+                )
                 return result
             except Exception as e:
                 gol_e = e
@@ -178,20 +231,218 @@ class ThinkDataQuery(BaseTga):
                     continue
         if gol_e is not None:
             raise gol_e
-    
-    def get_data_raw_pyhive(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
+
+    def get_data_csv_by_str_i(
+        self,
+        ctx,
+        unique_path,
+        sql,
+        block_size=100000,
+        print_size=100000,
+        read_timeout=600,
+        upcount=None,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
+        """
+        @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
+        """
+        session = requests.session()
+        post_data = {
+            "token": self.token,
+            "sql": sql,
+            "timeoutSeconds": tga_data_timeout,
+        }
+        #
+        unique_path = self.gen_local_unique_file()
+        #
+        BMOBJ.log_error("in query")
+        #
+        r = session.post(
+            self.query_uri,
+            data=post_data,
+            stream=True,
+            verify=False,
+            timeout=(conn_timeout, read_timeout),
+        )
+        datas = []
+        i = 0  # 循环引用计数
+        icount = 0  # 数据的数量
+        cols = []  # 表头
+        try:
+            row = ""
+            # iter_lines iter_content, chunk_size字节, 下面取100M
+            for row in r.iter_lines(chunk_size=1024 * 1024 * 100):
+                if not row:
+                    continue
+                data = None
+                if i == 0:  # 处理header
+                    data = json.loads(row)
+                    if data["return_code"] == 0:
+                        cols = data["data"]["headers"]
+                        df = pandas.DataFrame(
+                            data=[], columns=cols, type=object
+                        )  # 保存表头
+                        # 解决科学计数法的问题
+                        df = df.astype(str)
+                        df = df.astype("string")
+                        #
+                        self.g_to_csv_notmp(unique_path, df, index=None)
+                        data = None
+                    else:
+                        BMOBJ.log_error("sql error:", data)
+                        # BMOBJ.log_error(sql)
+                        try:
+                            BMOBJ.clog(ctx, "sql error:", data)
+                        except:
+                            pass
+                        datas = []
+                        raise Exception("sql error")
+                        break  # 表示查询出错，没有消息
+                else:
+                    if row.strip() not in [b"", ""]:
+                        data = json.loads(row)
+                if data is not None:
+                    datas.append(data)
+                i += 1
+                if len(datas) == block_size:  # 1000000条保存一次
+                    df = pandas.DataFrame(
+                        data=datas, columns=cols, type=object
+                    )  # 保存表头
+                    # 解决科学计数法的问题
+                    df = df.astype(str)
+                    df = df.astype("string")
+                    #
+                    self.g_to_csv_notmp(
+                        unique_path, df, index=None, mode="a", header=False
+                    )  # 追加保存
+                    icount += block_size
+                    datas = []
+                if i % print_size == 0:
+                    BMOBJ.clog(ctx, i)
+            BMOBJ.clog(ctx, f"total: {i}")
+            if len(datas) > 0:  # 保存最后收尾的
+                df = pandas.DataFrame(data=datas, columns=cols, type=object)  # 保存表头
+                # 解决科学计数法的问题
+                df = df.astype(str)
+                df = df.astype("string")
+                #
+                self.g_to_csv_notmp(
+                    unique_path, df, index=None, mode="a", header=False
+                )  # 追加保存
+                icount += len(datas)
+                datas = []
+        except Exception as e:
+            BMOBJ.clog(ctx, "get data error", str(e), row)
+            if upcount is not None:
+                if i < upcount:  # 看是否达到可以接受的数量，否则重新查询
+                    raise e
+            else:
+                raise e
+        finally:
+            try:
+                r.close()
+            except:
+                pass
+            try:
+                session.close()
+            except:
+                pass
+        return unique_path
+
+    def get_data_csv_by_str(
+        self,
+        ctx,
+        sql,
+        block_size=100000,
+        print_size=100000,
+        read_timeout=600,
+        upcount=None,
+        retry_count=2,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
+        """
+        @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
+        """
+        # unique_path = "./test.csv"
+        unique_path = self.gen_local_unique_file()
+        gol_e = None
+        for i in range(retry_count):
+            try:
+                result = self.get_data_csv_by_str_i(
+                    ctx,
+                    unique_path,
+                    sql,
+                    block_size,
+                    print_size,
+                    read_timeout,
+                    upcount,
+                    conn_timeout,
+                    tga_data_timeout,
+                )
+                return result
+            except Exception as e:
+                gol_e = e
+                BMOBJ.remove_file(unique_path)
+                BMOBJ.remove_folder(unique_path)
+                # modify by yx 2024-05-08---加强出错重试---
+                # if str(e).find("Read timed out") != -1 or str(e).find("Connection broken") != -1:
+                #     BMOBJ.clog(ctx, f'retry {i}')
+                #     continue
+                # else:
+                #     break
+                if 1:
+                    continue
+        if gol_e is not None:
+            raise gol_e
+
+    def get_data_raw_pyhive(
+        self,
+        ctx,
+        sql,
+        block_size=100000,
+        fetch_size=10000,
+        retry_count=2,
+        read_timeout=300,
+        upcount=None,
+        print_size=100000,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
         """
         @des: 接口装饰器--修改为get_data_csv，防止全面修改代码
         """
-        result = self.get_data_csv(ctx, sql, block_size, print_size, read_timeout, upcount, retry_count, conn_timeout, tga_data_timeout)
+        result = self.get_data_csv(
+            ctx,
+            sql,
+            block_size,
+            print_size,
+            read_timeout,
+            upcount,
+            retry_count,
+            conn_timeout,
+            tga_data_timeout,
+        )
         return result
-        
 
-    def get_data_raw_pyhive_bck(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
+    def get_data_raw_pyhive_bck(
+        self,
+        ctx,
+        sql,
+        block_size=100000,
+        fetch_size=10000,
+        retry_count=2,
+        read_timeout=300,
+        upcount=None,
+        print_size=100000,
+        conn_timeout=30,
+        tga_data_timeout=600,
+    ):
         '''
         @des:presto直连方式读取-----重试的方式----当get_data_csv接口出问题，则启用这个接口
         tobj = ThinkDataQuery("http://queryhost:port/querySql", "查询token",
-                          ["presto直连的host", 直连的port])  
+                          ["presto直连的host", 直连的port])
         sql = """select * from v_event_7 where "$part_date"='2022-02-24' limit 100 """
         unique_path = tobj.get_data_raw_pyhive({}, sql)
         '''
@@ -201,25 +452,46 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_raw_pyhive_i(
-                    ctx, unique_path, sql, block_size, fetch_size, read_timeout, upcount, print_size, conn_timeout)
+                    ctx,
+                    unique_path,
+                    sql,
+                    block_size,
+                    fetch_size,
+                    read_timeout,
+                    upcount,
+                    print_size,
+                    conn_timeout,
+                )
                 return result
             except Exception as e:
                 gol_e = e
                 BMOBJ.remove_file(unique_path)
                 BMOBJ.remove_folder(unique_path)
                 if str(e).find("Read timed out") != -1:
-                    BMOBJ.clog(ctx, f'retry {i}')
+                    BMOBJ.clog(ctx, f"retry {i}")
                     continue
                 else:
                     break
         if gol_e is not None:
             raise gol_e
 
-    def get_data_raw_pyhive_i(self, ctx, unique_path, sql, block_size=100000, fetch_size=10000, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30):
-        '''
+    def get_data_raw_pyhive_i(
+        self,
+        ctx,
+        unique_path,
+        sql,
+        block_size=100000,
+        fetch_size=10000,
+        read_timeout=300,
+        upcount=None,
+        print_size=100000,
+        conn_timeout=30,
+    ):
+        """
         @des: 内部调用
-        '''
+        """
         from pyhive import presto
+
         #
         # unique_path = self.gen_local_unique_file()
         # unique_path = "./test.csv"
@@ -228,20 +500,25 @@ class ThinkDataQuery(BaseTga):
         session = requests.session()
         #
         datas = []
-        i = 0    # 循环引用计数
+        i = 0  # 循环引用计数
         icount = 0  # 数据的数量
         cols = []  # 表头
         try:
-            conn = presto.connect(host=self.hive_conn_info[0],
-                                  port=int(self.hive_conn_info[1]),
-                                  username='ta', catalog='hive',
-                                  schema='ta',
-                                  requests_session = session,
-                                  # 这里stream为true和false没有关系，fetchmany每次都会通过request_session传nexturl重新get获取数据
-                                  # 参考pyhive/presto.py的_fetch_more，每次fetchmany其实是多次fetchone
-                                  requests_kwargs={"timeout": (
-                                      conn_timeout, read_timeout), "stream": True, "verify": False}
-                                  )
+            conn = presto.connect(
+                host=self.hive_conn_info[0],
+                port=int(self.hive_conn_info[1]),
+                username="ta",
+                catalog="hive",
+                schema="ta",
+                requests_session=session,
+                # 这里stream为true和false没有关系，fetchmany每次都会通过request_session传nexturl重新get获取数据
+                # 参考pyhive/presto.py的_fetch_more，每次fetchmany其实是多次fetchone
+                requests_kwargs={
+                    "timeout": (conn_timeout, read_timeout),
+                    "stream": True,
+                    "verify": False,
+                },
+            )
             cursor = conn.cursor()
             cursor.execute(sql)
             BMOBJ.clog(ctx, "文件大小")
@@ -266,6 +543,7 @@ class ThinkDataQuery(BaseTga):
             def yx_fetch_many():
                 myres = cursor.fetchmany(fetch_size)
                 return myres
+
             rows = yx_fetch_many()
             while rows:
                 for row in rows:
@@ -275,8 +553,9 @@ class ThinkDataQuery(BaseTga):
                     i += 1
                     if len(datas) == block_size:  # 1000000条保存一次
                         df = pandas.DataFrame(data=datas, columns=cols)  # 保存表头
-                        self.g_to_csv_notmp(unique_path, df, index=None,
-                                            mode='a', header=False)  # 追加保存
+                        self.g_to_csv_notmp(
+                            unique_path, df, index=None, mode="a", header=False
+                        )  # 追加保存
                         icount += block_size
                         datas = []
                     if i % print_size == 0:
@@ -286,8 +565,9 @@ class ThinkDataQuery(BaseTga):
             BMOBJ.clog(ctx, f"total: {i}")
             if len(datas) > 0:  # 保存最后收尾的
                 df = pandas.DataFrame(data=datas, columns=cols)  # 保存表头
-                self.g_to_csv_notmp(unique_path, df, index=None,
-                                    mode='a', header=False)  # 追加保存
+                self.g_to_csv_notmp(
+                    unique_path, df, index=None, mode="a", header=False
+                )  # 追加保存
                 icount += len(datas)
                 datas = []
         except Exception as e:
@@ -302,13 +582,13 @@ class ThinkDataQuery(BaseTga):
             try:
                 conn.close()
             except:
-                pass  
+                pass
             try:
                 session.close()
             except:
-                pass 
+                pass
         return unique_path
-    
+
     """
     数据打入接口--start--
     """
@@ -316,7 +596,7 @@ class ThinkDataQuery(BaseTga):
     def set_tga_user_data(self, tga_app_no, sec_token, url, data, is_set_once=False):
         """
         @des: 用户数据打入tga
-        @params: 
+        @params:
         tga_app_no: tga项目的id，注意不是app_id
         sec_token:  安全的二次部署服务器的token
         url: 二次部署的服务器打入的url
@@ -328,7 +608,7 @@ class ThinkDataQuery(BaseTga):
     def set_tga_event_data_trac(self, tga_app_no, sec_token, url, data):
         """
         @des: 普通事件数据打入tga
-        @params: 
+        @params:
         tga_app_no: tga项目的id，注意不是app_id
         sec_token:  安全的二次部署服务器的token
         url: 二次部署的服务器打入的url
@@ -340,7 +620,7 @@ class ThinkDataQuery(BaseTga):
     def set_tga_event_data_trac_update(self, tga_app_no, sec_token, url, data):
         """
         @des: 可更新事件数据打入tga, 重写部分数据
-        @params: 
+        @params:
         tga_app_no: tga项目的id，注意不是app_id
         sec_token:  安全的二次部署服务器的token
         url: 二次部署的服务器打入的url
@@ -349,10 +629,10 @@ class ThinkDataQuery(BaseTga):
         """
         pass
 
-    def set_tga_event_data_trac_overwrite(self, tga_app_no,  sec_token, url, data):
+    def set_tga_event_data_trac_overwrite(self, tga_app_no, sec_token, url, data):
         """
         @des: 可更新事件数据打入tga, 重写全部数据
-        @params: 
+        @params:
         tga_app_no: tga项目的id，注意不是app_id
         sec_token:  安全的二次部署服务器的token
         url: 二次部署的服务器打入的url
@@ -363,4 +643,4 @@ class ThinkDataQuery(BaseTga):
 
     """
     数据打入接口----end-----
-    """
+    """

{tfduck-bsd-0.18.2 → tfduck-bsd-0.18.3/tfduck_bsd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.2
+Version: 0.18.3
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

tfduck-bsd-0.18.2/tfduck/__init__.py

@@ -1 +0,0 @@
-__version__="0.18.2"