tfduck-bsd 0.17.8__py3-none-any.whl → 0.18.0__py3-none-any.whl

tfduck/__init__.py

@@ -1 +1 @@
-__version__="0.17.8"
+__version__="0.18.0"

tfduck/common/defines_clean.py

@@ -156,45 +156,42 @@ class BaseMethod(object):
 
     def get_record_now(self, ctx={}, tz="UTC"):
         """
-        @des: 获取执行任务记录的创建时间，方便dag里面取根据现在时间去执行part_date
+        @des: 获取执行任务记录的创建时间，方便dag里面取根据现在时间去执行part_date-----其实这个方法用不到，因为都是在dag里面用,dag用的是defines.BMOBJ，因为这个dag是必须带默认环境的
         start = arrow.now(tz="Asia/Shanghai")
         @ return: datetime 带时区
         """
-        if self.get_current_env() == "local":
+        # 如果是查看任务，而不是任务记录，是不会有这两个参数的
+        task_type = ctx.get('task_type', None)
+        trid = ctx.get('trid', None)
+        if trid is None:
             root_create_time = arrow.now(tz=tz)
         else:
-            # 如果是查看任务，而不是任务记录，是不会有这两个参数的
-            task_type = ctx.get('task_type', None)
-            trid = ctx.get('trid', None)
-            if trid is None:
-                root_create_time = arrow.now(tz=tz)
-            else:
-                Record = None
-                if task_type == "dtask":
-                    # Record = apps.get_model("dtask", 'RunRecord')
-                    app_name = "dtask"
-                    model_name = "RunRecord"
-                elif task_type == "retask":
-                    # Record = apps.get_model("retask", 'RETaskRecord')
-                    app_name = "retask"
-                    model_name = "RETaskRecord"
-                elif task_type == "sptask":
-                    # Record = apps.get_model("sptask", 'DPTaskRecord')
-                    app_name = "sptask"
-                    model_name = "DPTaskRecord"
-                if Record is not None:
-                    # obj = Record.objects.get(id=trid)
-                    objs = self.http_api.get_model_data(app_name, model_name, {"id": trid}, ["extra", "create_time"])
-                    #
-                    obj = objs[0]
-                    obj['extras'] = json.loads(obj['extra'])
-                    #
-                    root_create_time = obj['extras'].get("root_create_time", None)
-                    if root_create_time is None:  # 如果拿不到，那么obj肯定是根节点
-                        root_create_time = arrow.get(obj['create_time']).to(tz)
-                    else:  # 读取extra的信息，即根节点
-                        # root_create_time带有时区信息，可以直接get
-                        root_create_time = arrow.get(root_create_time).to(tz)
+            # Record = None
+            if task_type == "dtask":
+                # Record = apps.get_model("dtask", 'RunRecord')
+                app_name = "dtask"
+                model_name = "RunRecord"
+            elif task_type == "retask":
+                # Record = apps.get_model("retask", 'RETaskRecord')
+                app_name = "retask"
+                model_name = "RETaskRecord"
+            elif task_type == "sptask":
+                # Record = apps.get_model("sptask", 'DPTaskRecord')
+                app_name = "sptask"
+                model_name = "DPTaskRecord"
+            if app_name is not None:
+                # obj = Record.objects.get(id=trid)
+                objs = self.http_api.get_model_data(app_name, model_name, {"id": trid}, ["extra", "create_time"])
+                #
+                obj = objs[0]
+                obj['extras'] = json.loads(obj['extra'])
+                #
+                root_create_time = obj['extras'].get("root_create_time", None)
+                if root_create_time is None:  # 如果拿不到，那么obj肯定是根节点
+                    root_create_time = arrow.get(obj['create_time']).to(tz)
+                else:  # 读取extra的信息，即根节点
+                    # root_create_time带有时区信息，可以直接get
+                    root_create_time = arrow.get(root_create_time).to(tz)
         # 返回
         return root_create_time.datetime
 
@@ -247,6 +244,27 @@ class BaseMethod(object):
             result.append(str(e2))
         return "\n".join(result)
 
+    def logerr(self, e):
+        errorMeg = ''
+        try:
+            for file, lineno, function, text in traceback.extract_tb(sys.exc_info()[2]):
+                errorMeg += '%s\n%s, in %s\n%s:                %s!' % (
+                    str(e), file, function, lineno, text)
+            self.log_error("error"+"*"*50)
+            for error in errorMeg.split("\n"):
+                self.log_error(error, "error")
+            try:
+                error_self = "".join(traceback.format_exception_only(type(e), e))  # 捕获错误本身,这样才能捕获到compile里面的错误
+                self.log_error(error_self, "error")
+            except:
+                pass
+            try:
+                self.log_error(getattr(e, '_msg', 'exception'))
+            except Exception as e1:
+                self.log_error(getattr(e, 'msg', 'exception'))
+        except Exception as e2:
+            self.log_error(e2)
+
     def get_log_str(self, *msgs):
         try:
             msg_str = " ".join([str(msg) for msg in msgs])
@@ -283,6 +301,8 @@ class Dj44HttpApi(object):
         self.user_token = ''
     
     def get_user_token(self):
+        if self.user_token == '':
+            raise Et(2, "请先设置user_token")
         return self.user_token
     
     def set_user_token(self, user_token):
@@ -291,7 +311,12 @@ class Dj44HttpApi(object):
 
     def get_host_name(self):
         # 后面可以改成从配置中心获取
-        return {'s': 1, 'v': "http://tfduck.163py.com"}  
+        PROJECT_ENV = os.environ.get("PROJECT_ENV", "dev")
+        if PROJECT_ENV == "dev": # 本地测试
+            endpoint_url = "http://localhost:8000"
+        else:
+            endpoint_url = "http://tfduck.163py.com"   # 后面可以改成从配置中心获取
+        return {'s': 1, 'v': endpoint_url}  
     
     # 旧的接口也从tfduck的外部接口调用
 

tfduck/tga/tga.py

@@ -72,19 +72,19 @@ class ThinkDataQuery(BaseTga):
                           mode=mode, header=header)
         return True
 
-    def get_data_csv_i(self, ctx, unique_path, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None):
+    def get_data_csv_i(self, ctx, unique_path, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, conn_timeout=30, tga_data_timeout=600):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
         session = requests.session()
-        post_data = {'token': self.token, 'sql': sql}
+        post_data = {'token': self.token, 'sql': sql, 'timeoutSeconds': tga_data_timeout}
         #
         unique_path = self.gen_local_unique_file()
         #
         BMOBJ.log_error("in query")
         #
         r = session.post(self.query_uri, data=post_data, stream=True,
-                         verify=False, timeout=(30, read_timeout))
+                         verify=False, timeout=(conn_timeout, read_timeout))
         datas = []
         i = 0    # 循环引用计数
         icount = 0  # 数据的数量
@@ -150,7 +150,7 @@ class ThinkDataQuery(BaseTga):
                 pass
         return unique_path
 
-    def get_data_csv(self, ctx, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, retry_count=2):
+    def get_data_csv(self, ctx, sql, block_size=100000, print_size=100000, read_timeout=600, upcount=None, retry_count=2, conn_timeout=30, tga_data_timeout=600):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
         """
@@ -160,7 +160,7 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_csv_i(
-                    ctx, unique_path, sql, block_size, print_size, read_timeout, upcount)
+                    ctx, unique_path, sql, block_size, print_size, read_timeout, upcount, conn_timeout, tga_data_timeout)
                 return result
             except Exception as e:
                 gol_e = e
@@ -174,15 +174,15 @@ class ThinkDataQuery(BaseTga):
         if gol_e is not None:
             raise gol_e
     
-    def get_data_raw_pyhive(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
         """
         @des: 接口装饰器--修改为get_data_csv，防止全面修改代码
         """
-        result = self.get_data_csv(ctx, sql, block_size, print_size, read_timeout, upcount, retry_count)
+        result = self.get_data_csv(ctx, sql, block_size, print_size, read_timeout, upcount, retry_count, conn_timeout, tga_data_timeout)
         return result
         
 
-    def get_data_raw_pyhive_bck(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive_bck(self, ctx, sql, block_size=100000, fetch_size=10000, retry_count=2, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30, tga_data_timeout=600):
         '''
         @des:presto直连方式读取-----重试的方式----当get_data_csv接口出问题，则启用这个接口
         tobj = ThinkDataQuery("http://queryhost:port/querySql", "查询token",
@@ -196,7 +196,7 @@ class ThinkDataQuery(BaseTga):
         for i in range(retry_count):
             try:
                 result = self.get_data_raw_pyhive_i(
-                    ctx, unique_path, sql, block_size, fetch_size, read_timeout, upcount, print_size)
+                    ctx, unique_path, sql, block_size, fetch_size, read_timeout, upcount, print_size, conn_timeout)
                 return result
             except Exception as e:
                 gol_e = e
@@ -210,7 +210,7 @@ class ThinkDataQuery(BaseTga):
         if gol_e is not None:
             raise gol_e
 
-    def get_data_raw_pyhive_i(self, ctx, unique_path, sql, block_size=100000, fetch_size=10000, read_timeout=300, upcount=None, print_size=100000):
+    def get_data_raw_pyhive_i(self, ctx, unique_path, sql, block_size=100000, fetch_size=10000, read_timeout=300, upcount=None, print_size=100000, conn_timeout=30):
         '''
         @des: 内部调用
         '''
@@ -235,7 +235,7 @@ class ThinkDataQuery(BaseTga):
                                   # 这里stream为true和false没有关系，fetchmany每次都会通过request_session传nexturl重新get获取数据
                                   # 参考pyhive/presto.py的_fetch_more，每次fetchmany其实是多次fetchone
                                   requests_kwargs={"timeout": (
-                                      30, read_timeout), "stream": True, "verify": False}
+                                      conn_timeout, read_timeout), "stream": True, "verify": False}
                                   )
             cursor = conn.cursor()
             cursor.execute(sql)
@@ -303,3 +303,59 @@ class ThinkDataQuery(BaseTga):
             except:
                 pass 
         return unique_path
+    
+    """
+    数据打入接口--start--
+    """
+
+    def set_tga_user_data(self, tga_app_no, sec_token, url, data, is_set_once=False):
+        """
+        @des: 用户数据打入tga
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None', '打入的用户属性为一个dict' ], ...,]
+              例如: [['eqw31231231', 'fads21321312312',   {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac(self, tga_app_no, sec_token, url, data):
+        """
+        @des: 普通事件数据打入tga
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None', '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac_update(self, tga_app_no, sec_token, url, data):
+        """
+        @des: 可更新事件数据打入tga, 重写部分数据
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', '事件唯一id', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None',   , '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session', 'event_id_123',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    def set_tga_event_data_trac_overwrite(self, tga_app_no,  sec_token, url, data):
+        """
+        @des: 可更新事件数据打入tga, 重写全部数据
+        @params: 
+        tga_app_no: tga项目的id，注意不是app_id
+        sec_token:  安全的二次部署服务器的token
+        url: 二次部署的服务器打入的url
+        data: 数据  [['事件名称', '事件唯一id', 'tga用户的distinct_id,为空就传None', 'tga用户的account_id为空就传None',   , '打入的用户属性为一个dict' ], ...,]
+              例如: [['new_session', 'event_id_123',  'eqw31231231', 'fads21321312312',  {'a':1, 'b':2}], ...,]
+        """
+        pass
+
+    """
+    数据打入接口----end-----
+    """

tfduck/tga/train_sql_ltv.py

@@ -335,7 +335,7 @@ class TrainFeatureSql(BaseTga):
             -- 直接用user_user_id作为排序值，排序的必须是唯一的，否则下面会对不上，会产生很多null的数据
             -- user_user_id as tt_stable_rand  -- 这种方式最保险，但是不能乱序，这样采样的数据就不是随机分布在每天的 
             -- bitwise_xor(user_user_id, 906867964886667264) as tt_stable_rand  -- 这种方式可能会产生left null的情况，但是是少数，过滤掉就行，不影响结果，但支持乱序采样
-            a3.user_distinct_id as tt_stable_rand  -- 这种方式最保险，即是乱序也是唯一
+            a3.user_distinct_id as tt_stable_rand  -- 这种方式最保险，即是乱序也是唯一, 就算limit分几部分取特征值，这几部分也是可以对应上的
 
         FROM
         (

tfduck/tga/train_sql_retain.py

@@ -107,7 +107,7 @@ class TrainFeatureSql(BaseTga):
                  user_cols1='1=1',
                  user_cols2='1=1',
                  event_cols1='1=1',
-                 addon_attrs=[], 
+                 addon_attrs=[],
                  **kwargs):
         """
         @des: 参数说明看上面的文档说明

{tfduck_bsd-0.17.8.dist-info → tfduck_bsd-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.17.8
+Version: 0.18.0
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao
@@ -13,6 +13,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.5
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: arrow (>=0.15.5)
 Requires-Dist: requests (>=2.20.0)
 Requires-Dist: django (==2.2.12)
 Requires-Dist: oss2 (==2.15.0)

{tfduck_bsd-0.17.8.dist-info → tfduck_bsd-0.18.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-tfduck/__init__.py,sha256=JJRjv3zZ3ki2vwIQ0GH-QI69Reem5LIvEvqJU3y0eCM,20
+tfduck/__init__.py,sha256=xg0NwcLTzmqN-Xa9iGt2_NsDBIOcmZkITwbvTeLIC6U,20
 tfduck/main.py,sha256=zNTC16wkwGJ0QX1-i8vzlGophOxmFuO4SLsF1tkjsbE,14670
 tfduck/bdp_sdk_py/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tfduck/bdp_sdk_py/example.py,sha256=Xq1_gcSyu0zho_iMmMYfYgMblcgF8a-GwRBWPTw0FuU,2879
@@ -10,7 +10,7 @@ tfduck/bdp_sdk_py/opends/opends.py,sha256=ny2bcB7gCv6aYXrdL2fbGe2kJDQ-hamxunerYq
 tfduck/bdp_sdk_py/opends/sdk.py,sha256=zagIhg0gQYnGmEnFgfxBwo4TW17GjeVrYaS7Qt-Vtow,19136
 tfduck/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tfduck/common/defines.py,sha256=pHhQ35TdB83GWs75hEN-b5qGqLO6Ipf8EfQsHolGZkE,17629
-tfduck/common/defines_clean.py,sha256=D5W1t7RCKJoWIp31RYDrhB6E0qENadH5PeSzWdwuThU,19483
+tfduck/common/defines_clean.py,sha256=1U6jNw_9YTTO_EznH6vBavSGsYI6G9i2TnSbVxXlMwk,20642
 tfduck/common/extendEncoder.py,sha256=k2s_FEJdBDVCx5CU3ADwJGlr7NUkPP9HKu_epZcpiTo,1525
 tfduck/oss/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tfduck/oss/oss.py,sha256=bQuEjUKzPdV2MbGd33pzm5z9l50KHXwVW23wOuRJ9NE,19871
@@ -26,16 +26,16 @@ tfduck/tga/base_tga.py,sha256=rg1BHaIKGSwUVbOlWSA3Y1arB8J_caemjTjH9jh8GYM,2184
 tfduck/tga/predict_sql_ltv.py,sha256=25rpOZdHyMcEU3O8u67oUpLsTiZburEPsvXR38hTUJ0,3589
 tfduck/tga/predict_sql_retain.py,sha256=Nsl0lSZ_CC8j_GB4jLrJgkoqqDRAAPYwzo7vAOYZ764,19772
 tfduck/tga/predict_sql_yh.py,sha256=uYeuCZX2btxO-pvjrhlmuG35PquJbLvVtQEuEZHu8Cs,3588
-tfduck/tga/tga.py,sha256=_0kDEDLI7TMGkCUqmBO-yMwuEy3bRTXUSIJAHcQgyEc,12654
+tfduck/tga/tga.py,sha256=p9AcaiPiGrHaOvfJGnpzNaFK8OeDL5NbCU770OaOTWc,15605
 tfduck/tga/tga_test.py,sha256=A3n2LdvgQWlkX6E54K6cnsIUeYrgHlzG3QYbA7ZKgHk,2750
-tfduck/tga/train_sql_ltv.py,sha256=hBksB-fukA4Q6LuBo7B93XyFc0VNWyVj8pVNaV8xYYc,26592
-tfduck/tga/train_sql_retain.py,sha256=o64RBfnUTJtfnTKBVE4zy_kbEtB7g5YmshyWczbcSe0,24336
+tfduck/tga/train_sql_ltv.py,sha256=VZqGy0FbwLHkjQClV1IIMd6_E_H6BrIqL5p_WUfTUXc,26668
+tfduck/tga/train_sql_retain.py,sha256=AIOJKWC37j4UdM8JLFS6LdJFtABjMq9gOi3xvAs4fAE,24335
 tfduck/tga/train_sql_yh.py,sha256=nb5BO_vOv0eKY2kOVt5ZOfM1cvfI8j2no8cYpCL_rNE,24378
 tfduck/thinkdata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tfduck/thinkdata/query.py,sha256=DsfcxjZrc0ZFTwN2pI5fKdM1Bwr6ageoPcA2MP3r2bE,1314
-tfduck_bsd-0.17.8.data/scripts/tfduck,sha256=UsuoAs4peJW4I-e6Gn91gEToP_YyuUp-rUUg3ObKneY,192
-tfduck_bsd-0.17.8.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
-tfduck_bsd-0.17.8.dist-info/METADATA,sha256=hBLN9umQHLIC--gwZwfdBa_gA-4XVUXx5COIW-Cz3qo,971
-tfduck_bsd-0.17.8.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
-tfduck_bsd-0.17.8.dist-info/top_level.txt,sha256=503etRkoyeI1VYcAwe5KpD5Bamhx0R0y2ofkE8HpRDA,7
-tfduck_bsd-0.17.8.dist-info/RECORD,,
+tfduck_bsd-0.18.0.data/scripts/tfduck,sha256=UsuoAs4peJW4I-e6Gn91gEToP_YyuUp-rUUg3ObKneY,192
+tfduck_bsd-0.18.0.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+tfduck_bsd-0.18.0.dist-info/METADATA,sha256=kdiL2okuGWTB1xd643seSd0pXOGoND5IxL5z2PVLIEc,1003
+tfduck_bsd-0.18.0.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
+tfduck_bsd-0.18.0.dist-info/top_level.txt,sha256=503etRkoyeI1VYcAwe5KpD5Bamhx0R0y2ofkE8HpRDA,7
+tfduck_bsd-0.18.0.dist-info/RECORD,,