tfduck-bsd 0.17.9__tar.gz → 0.18.1__tar.gz

{tfduck-bsd-0.17.9/tfduck_bsd.egg-info → tfduck-bsd-0.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.17.9
+Version: 0.18.1
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.17.9 → tfduck-bsd-0.18.1}/setup.py

@@ -8,7 +8,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="tfduck-bsd",
-    version="0.17.9",
+    version="0.18.1",
     author="yuanxiao",
     author_email="yuan6785@163.com",
     description="A small example package",

tfduck-bsd-0.18.1/tfduck/__init__.py

@@ -0,0 +1 @@
+__version__="0.18.1"

{tfduck-bsd-0.17.9 → tfduck-bsd-0.18.1}/tfduck/tga/tga.py

@@ -72,24 +72,25 @@ class ThinkDataQuery(BaseTga):
                           mode=mode, header=header)
         return True
 
-    def get_data_csv_i(self, ctx, unique_path, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None):
+    def get_data_csv_i(self, ctx, unique_path, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, conn_timeout=30, tga_data_timeout=600):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
         session = requests.session()
-        post_data = {'token': self.token, 'sql': sql}
+        post_data = {'token': self.token, 'sql': sql, 'timeoutSeconds': tga_data_timeout}
         #
         unique_path = self.gen_local_unique_file()
         #
         BMOBJ.log_error("in query")
         #
         r = session.post(self.query_uri, data=post_data, stream=True,
-                         verify=False, timeout=(30, read_timeout))
+                         verify=False, timeout=(conn_timeout, read_timeout))
         datas = []
         i = 0    # 循环引用计数
         icount = 0  # 数据的数量
         cols = []  # 表头
         try:
+            row = ''
             # iter_lines iter_content, chunk_size字节, 下面取100M
             for row in r.iter_lines(chunk_size=1024*1024*100):
                 if not row:
@@ -133,7 +134,7 @@ class ThinkDataQuery(BaseTga):
                 icount += len(datas)
                 datas = []
         except Exception as e:
-            BMOBJ.clog(ctx, "get data error", str(e))
+            BMOBJ.clog(ctx, "get data error", str(e), row)
             if upcount is not None:
                 if i < upcount:  # 看是否达到可以接受的数量，否则重新查询
                     raise e
@@ -150,7 +151,7 @@ class ThinkDataQuery(BaseTga):
                 pass
         return unique_path
 
-    def get_data_csv(self, ctx, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, retry_count=2):
+    def get_data_csv(self, ctx, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, retry_count=2, conn_timeout=30, tga_data_timeout=600):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
@@ -160,7 +161,7 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_csv_i(
-                    ctx, unique_path, sql, block_size, print_size, read_timeout, upcount)
+                    ctx, unique_path, sql, block_size, print_size, read_timeout, upcount, conn_timeout, tga_data_timeout)
                 return result
             except Exception as e:
                 gol_e = e
@@ -174,15 +175,15 @@ class ThinkDataQuery(BaseTga):
         if gol_e is not None:
             raise gol_e
     
-    def get_data_raw_pyhive(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
         """
         @des: 接口装饰器--修改为get_data_csv，防止全面修改代码
         """
-        result = self.get_data_csv(ctx, sql, block_size, print_size, read_timeout, upcount, retry_count)
+        result = self.get_data_csv(ctx, sql, block_size, print_size, read_timeout, upcount, retry_count, conn_timeout, tga_data_timeout)
         return result
         
 
-    def get_data_raw_pyhive_bck(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive_bck(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
         '''
         @des:presto直连方式读取-----重试的方式----当get_data_csv接口出问题，则启用这个接口
         tobj = ThinkDataQuery("http://queryhost:port/querySql", "查询token",
@@ -196,7 +197,7 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_raw_pyhive_i(
-                    ctx, unique_path, sql, block_size, fetch_size, read_timeout, upcount, print_size)
+                    ctx, unique_path, sql, block_size, fetch_size, read_timeout, upcount, print_size, conn_timeout)
                 return result
             except Exception as e:
                 gol_e = e
@@ -210,7 +211,7 @@ class ThinkDataQuery(BaseTga):
         if gol_e is not None:
             raise gol_e
 
-    def get_data_raw_pyhive_i(self, ctx, unique_path, sql, block_size=100000, fetch_size=10000, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive_i(self, ctx, unique_path, sql, block_size=100000, fetch_size=10000, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30):
         '''
         @des: 内部调用
         '''
@@ -235,7 +236,7 @@ class ThinkDataQuery(BaseTga):
                                   # 这里stream为true和false没有关系，fetchmany每次都会通过request_session传nexturl重新get获取数据
                                   # 参考pyhive/presto.py的_fetch_more，每次fetchmany其实是多次fetchone
                                   requests_kwargs={"timeout": (
-                                      30, read_timeout), "stream": True, "verify": False}
+                                      conn_timeout, read_timeout), "stream": True, "verify": False}
                                   )
             cursor = conn.cursor()
             cursor.execute(sql)
@@ -303,3 +304,59 @@ class ThinkDataQuery(BaseTga):
             except:
                 pass 
         return unique_path
+    
+    """
+    数据打入接口--start--
+    """
+
+    def set_tga_user_data(self, tga_app_no, sec_token, url, data, is_set_once=False):
+        """
+        @des: 用户数据打入tga
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None', '打入的用户属性为一个dict' ], ...,]
+              例如: [['eqw31231231', 'fads21321312312',   {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac(self, tga_app_no, sec_token, url, data):
+        """
+        @des: 普通事件数据打入tga
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None', '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac_update(self, tga_app_no, sec_token, url, data):
+        """
+        @des: 可更新事件数据打入tga, 重写部分数据
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', '事件唯一id', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None',   , '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session', 'event_id_123',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac_overwrite(self, tga_app_no,  sec_token, url, data):
+        """
+        @des: 可更新事件数据打入tga, 重写全部数据
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', '事件唯一id', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None',   , '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session', 'event_id_123',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    """
+    数据打入接口----end-----
+    """

{tfduck-bsd-0.17.9 → tfduck-bsd-0.18.1}/tfduck/tga/train_sql_ltv.py

@@ -335,7 +335,7 @@ class TrainFeatureSql(BaseTga):
             -- 直接用user_user_id作为排序值，排序的必须是唯一的，否则下面会对不上，会产生很多null的数据
             -- user_user_id as tt_stable_rand  -- 这种方式最保险，但是不能乱序，这样采样的数据就不是随机分布在每天的 
             -- bitwise_xor(user_user_id, 906867964886667264) as tt_stable_rand  -- 这种方式可能会产生left null的情况，但是是少数，过滤掉就行，不影响结果，但支持乱序采样
-            a3.user_distinct_id as tt_stable_rand  -- 这种方式最保险，即是乱序也是唯一
+            a3.user_distinct_id as tt_stable_rand  -- 这种方式最保险，即是乱序也是唯一, 就算limit分几部分取特征值，这几部分也是可以对应上的
 
         FROM
         (

{tfduck-bsd-0.17.9 → tfduck-bsd-0.18.1}/tfduck/tga/train_sql_retain.py

@@ -107,7 +107,7 @@ class TrainFeatureSql(BaseTga):
                  user_cols1='1=1',
                  user_cols2='1=1',
                  event_cols1='1=1',
-                 addon_attrs=[], 
+                 addon_attrs=[],
                  **kwargs):
         """
         @des: 参数说明看上面的文档说明

{tfduck-bsd-0.17.9 → tfduck-bsd-0.18.1/tfduck_bsd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.17.9
+Version: 0.18.1
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

tfduck-bsd-0.17.9/tfduck/__init__.py

@@ -1 +0,0 @@
-__version__="0.17.9"