tfduck-bsd 0.16.1__tar.gz → 0.16.2__tar.gz

{tfduck-bsd-0.16.1/tfduck_bsd.egg-info → tfduck-bsd-0.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.16.1
+Version: 0.16.2
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/setup.py

@@ -8,7 +8,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="tfduck-bsd",
-    version="0.16.1",
+    version="0.16.2",
     author="yuanxiao",
     author_email="yuan6785@163.com",
     description="A small example package",

tfduck-bsd-0.16.2/tfduck/__init__.py

@@ -0,0 +1 @@
+__version__="0.16.2"

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck/pyspark_k8s/spark_manage.py

@@ -364,7 +364,7 @@ if __name__ == "__main__":
     # # "容器内path，没有被污染也可以不设置"
     # os.environ["PYSPARK_DRIVER_PYTHON"] = "/usr/bin/python342342342342"
 
-    if 0:  # 同步--天天--测试上传代码路径
+    if 1:  # 同步--天天--测试上传代码路径
         sm = SparkManage(
             ctx={},
             # code_path='/Users/yuanxiao/workspace/djcelery44/djcelery44/scripts/tools/debug/test_presto_s3',
@@ -380,7 +380,7 @@ if __name__ == "__main__":
         )
         sm.upload_code()
         sm.submit_spark_task()
-    if 1:  # 同步--天天---测试直接上传代码内容
+    if 0:  # 同步--天天---测试直接上传代码内容
         with open('/Users/yuanxiao/workspace/djcelery44/djcelery44/scripts/tools/debug/test_hello/main_script.py', 'r') as f:
             code_content = f.read()
         #

tfduck-bsd-0.16.2/tfduck/tga/predict_sql_ltv.py

@@ -0,0 +1,90 @@
+'''
+@des: tga获取批量转换特征值数据拉取的sql基类
+
+
+准备工作: 
+--------------------------------------
+gconf的属性有这些:
+########
+   part_date_start  # 特征值取新增用户的开始日期, 例如"2022-05-20", 字符串类型
+   part_date_end    # 特征值取新增用户的结束日期, 例如"2022-05-25", 字符串类型
+   level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
+   tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
+########
+
+如何使用(需要自己定义一个子类继承这个类， 代码如下): 
+#######
+from magic_number_train_tf_2.src.common.config import Config as GAConfig # 这里换成你自己的项目
+from tfduck.tga.predict_sql_ltv import PredictFeatureSql
+
+def call_method(self, ctx=None, **kwargs):
+    """
+    @des: 每个具体项目的训练特征数据拉取 
+    """
+    gconf = GAConfig.getInstance()  # 保证gconf有上面准备工作的属性
+    """
+    设置项目属性-----下面的都需要自己配置------
+    """
+    # 需要的事件
+    need_events = (
+        'new_device', 'new_session', 'act_level_path'
+    )
+    # 需要的属性
+    need_event_attrs = (
+        "#lib", "#country_code", "$part_event", "#user_id",  "sdk_session_time",
+        "object_type", "act", "object_number", "act_object",
+    )
+    # 特征值名称(不要用字母a作为key)---
+    feature_names = {
+        'b': '常规关卡通关',
+        'c': '冒险关卡通关'
+    }
+    # 特征值sql---模板保持不变，变里面的内容即可---
+    sub_sql_fs = {
+        'b':            """
+                        --%(real_des)s
+                        sum(
+                            if(
+                            "$part_event"='act_level_path' and object_type='normal' and act='win' and rt_dur<%(level_dur)s,
+                            1, 
+                            0)
+                        ) as %(sub_field_name)s
+                        """,
+        'c':            """
+                        --%(real_des)s
+                        sum(
+                            if(
+                            "$part_event"='act_level_path' and object_type='adventure' and act='win' and rt_dur<%(level_dur)s,
+                            1, 
+                            0)
+                        ) as %(sub_field_name)s
+                        """
+    }       
+    # 额外的用户属性比如(可选参数)
+    addon_attrs = ["#screen_width", "#screen_height"]         
+    """
+    创建调用实例---最好通过dict创建实例
+    """         
+    pf_sql_obj = PredictFeatureSql(
+            ctx = ctx, 
+            gconf = gconf, 
+            need_events = need_events, 
+            need_event_attrs = need_event_attrs, 
+            feature_names = feature_names,
+            sub_sql_fs = sub_sql_fs,
+            addon_attrs = addon_attrs # 可选参数
+    )  
+    sql = pf_sql_obj.get_sql() # 这个sql就是拉取特征值的sql
+#######
+'''
+from tfduck.common.defines import BMOBJ, Et
+from tfduck.tga.predict_sql_retain import PredictFeatureSql as BasePredictFeatureSql
+import arrow
+
+
+class PredictFeatureSql(BasePredictFeatureSql):
+    """
+    @des:sql批量转换模板基类
+    """
+    pass

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck/tga/predict_sql_retain.py

@@ -10,7 +10,7 @@ gconf的属性有这些:
    part_date_end    # 特征值取新增用户的结束日期, 例如"2022-05-25", 字符串类型
    level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
    tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
-   tga_event_table  # tga用户表名, 例如 "v_user_7", 字符串 
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
 ########
 
 如何使用(需要自己定义一个子类继承这个类， 代码如下): 

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck/tga/predict_sql_yh.py

@@ -10,7 +10,7 @@ gconf的属性有这些:
    part_date_end    # 特征值取新增用户的结束日期, 例如"2022-05-25", 字符串类型
    level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
    tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
-   tga_event_table  # tga用户表名, 例如 "v_user_7", 字符串 
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
 ########
 
 如何使用(需要自己定义一个子类继承这个类， 代码如下): 

tfduck-bsd-0.16.2/tfduck/tga/tga_test.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+from train_sql_ltv import TrainFeatureSql
+
+
+def test():
+    class GConf(object):
+        def __init__(self):
+            self.part_date_start = "2023-01-15"
+            self.part_date_end = "2023-01-17"
+            # 注册后0-14天的所有ltv作为标签
+            self.label_durs = [int(86400*0), int(86400*14)]
+            self.level_durs = [86400*0.25, 86400*0.5,
+                               86400, 86400*2, 86400*3]  # 由小到大排列
+            self.tga_user_table = "v_user_30"
+            self.tga_event_table = "v_event_30"
+
+    gconf = GConf()
+
+    need_events = (
+        'new_device',
+        'new_session',
+        'g_push',
+        'sdk_close_session'
+    )
+    need_event_attrs = (
+        "#lib",
+        "pn",
+        "#device_model",
+        "#screen_width",
+        "#screen_height",
+        "#country_code",
+        "$part_event",
+        "#user_id"
+    )
+    feature_names = {
+        'b': '打开游戏次数',
+        'c': '本地推送打开游戏次数'
+    }
+    addon_attrs = [
+        "#device_model",
+        "#screen_width",
+        "#screen_height"
+    ]
+    sub_sql_fs = {
+        'b':
+        # 打开游戏次数
+        """
+                                --%(real_des)s
+                                sum(
+                                    if(
+                                    "$part_event"='new_session' and rt_dur<%(level_dur)s,
+                                    1, 
+                                    0)
+                                ) as %(sub_field_name)s
+                                """, 'c':
+        # 本地推送打开游戏次数
+        """
+                                --%(real_des)s
+                                sum(
+                                    if(
+                                    "$part_event"='g_push' and  rt_dur<%(level_dur)s,
+                                    1,
+                                    0)
+                                ) as %(sub_field_name)s
+                                """
+    }
+    user_cols2 = """ "#lib"='Android' """
+    tf_sql_obj = TrainFeatureSql(
+        ctx={},
+        gconf=gconf,
+        need_events=need_events,
+        need_event_attrs=need_event_attrs,
+        feature_names=feature_names,
+        sub_sql_fs=sub_sql_fs,
+        user_cols2=user_cols2,
+        addon_attrs=addon_attrs,  # 可选参数
+        line_value_rd=0.5,  # 可选参数
+        mode="iaa+iap"
+    )
+    sql = tf_sql_obj.get_sql()  # 这个sql就是拉取特征值的sql
+    # 将sql复制到剪贴板
+    import pyperclip
+    pyperclip.copy(sql)
+    return sql
+
+
+if __name__ == "__main__":
+    test()

tfduck-bsd-0.16.2/tfduck/tga/train_sql_ltv.py

@@ -0,0 +1,603 @@
+'''
+@des: tga获取训练特征值数据拉取的sql基类
+
+
+准备工作: 
+--------------------------------------
+gconf的属性有这些:
+########
+   part_date_start  # 特征值取新增用户的开始日期, 例如"2022-05-20", 字符串类型
+   part_date_end    # 特征值取新增用户的结束日期, 例如"2022-05-25", 字符串类型
+   level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
+   label_durs       # 取多少边界算ltv标签值，例如[int(86400*14), int(86400*17)]，整型数组；这里是取14到17天发生过事件的作为正向用户
+   tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
+########
+
+如何使用(需要自己定义一个子类继承这个类， 代码如下): 
+#######
+from magic_number_train_tf_2.src.common.config import Config as GAConfig # 这里换成你自己的项目
+from tfduck.tga.train_sql_ltv import TrainFeatureSql
+
+def call_method(self, ctx=None, **kwargs):
+    """
+    @des: 每个具体项目的训练特征数据拉取 
+    """
+    gconf = GAConfig.getInstance()  # 保证gconf有上面准备工作的属性
+    """
+    设置项目属性-----下面的都需要自己配置------
+    """
+    # 需要的事件
+    need_events = (
+        'new_device', 'new_session', 'act_level_path'
+    )
+    # 需要的属性
+    need_event_attrs = (
+        "#lib", "#country_code", "$part_event", "#user_id",  "sdk_session_time",
+        "object_type", "act", "object_number", "act_object",
+    )
+    # 特征值名称--不要用字母a作为key---
+    feature_names = {
+        'b': '常规关卡通关',
+        'c': '冒险关卡通关'
+    }
+    # 特征值sql---模板保持不变，变里面的内容即可---
+    sub_sql_fs = {
+        'b':            """
+                        --%(real_des)s
+                        sum(
+                            if(
+                            "$part_event"='act_level_path' and object_type='normal' and act='win' and rt_dur<%(level_dur)s,
+                            1, 
+                            0)
+                        ) as %(sub_field_name)s
+                        """,
+        'c':            """
+                        --%(real_des)s
+                        sum(
+                            if(
+                            "$part_event"='act_level_path' and object_type='adventure' and act='win' and rt_dur<%(level_dur)s,
+                            1, 
+                            0)
+                        ) as %(sub_field_name)s
+                        """
+    }  
+    # 额外的用户属性比如(可选参数)
+    addon_attrs = ["#screen_width", "#screen_height"] 
+    # line_value_rd  ltv的线性值大于多少算正样本，小于多少算负样本，这样可以控制正负样本比例     
+    line_value_rd = 0.5
+    # mode 标签值iaa(只计算iaa), iap(只计算iap), iaa+iap(计算iaa+iap)
+    mode = 'iaa+iap'
+    """
+    创建调用实例---最好通过dict创建实例
+    注意： 针对一个tga项目属于多个项目的打点情况,例如3dfish， 可以适时启用user_cols1,user_cols2或者event_cols1
+    a.  user_cols1 参数为用户过滤条件(user表)
+        如果有的话, 例如: user_cols1 = """ "afrawip__meida_source"='FaceBook' OR "afrawip__app_id"='123456' """
+    b.  user_cols2 参数为用户过滤条件(event表的new_device事件)
+        如果有的话, 例如: user_cols2 = """ "#lib"='Android' OR "#lib"='iOS' """
+    c.  event_cols1 参数为事件过滤条件(event表的所有计算特征值的事件)
+        如果有的话, 例如: event_cols1 = """ "#lib"='Android' OR "#lib"='iOS' """
+    """         
+    tf_sql_obj = TrainFeatureSql(
+            ctx = ctx, 
+            gconf = gconf, 
+            need_events = need_events, 
+            need_event_attrs = need_event_attrs, 
+            feature_names = feature_names,
+            sub_sql_fs = sub_sql_fs,
+            addon_attrs = addon_attrs # 可选参数
+    )  
+    sql = tf_sql_obj.get_sql() # 这个sql就是拉取特征值的sql
+
+#######
+'''
+from tfduck.tga.base_tga import BaseTga
+from tfduck.common.defines import BMOBJ, Et
+import arrow
+
+
+class TrainFeatureSql(BaseTga):
+    """
+    @des:sql训练模板基类
+    """
+
+    def __init__(self,
+                 ctx=None,
+                 gconf=None,
+                 need_events=None,
+                 need_event_attrs=None,
+                 feature_names=None,
+                 sub_sql_fs=None,
+                 user_cols1='1=1',
+                 user_cols2='1=1',
+                 event_cols1='1=1',
+                 addon_attrs=[], 
+                 line_value_rd=0.5, 
+                 mode='iaa+iap', 
+                 **kwargs):
+        """
+        @des: 参数说明看上面的文档说明
+        """
+        self.gconf = gconf
+        self.ctx = ctx
+        # 基础属性
+        self.label_col = "is_rd"
+        self.base_event_table = 'all_need_event'
+        self.base_user_table = 'a'
+        self.base_feature_table = 'features_t'
+        self.nav_neg_multi = 2  # 负正样本比例， 线性值无正负样本比例的说法, 但可以给一个中间值，大于多少，小于多少即可
+        self.line_value_rd = line_value_rd # ltv的线性值大于多少算正样本，小于多少算负样本，这样可以控制正负样本比例
+        self.mode = mode # mode 标签值iaa(只计算iaa), iap(只计算iap), iaa+iap(计算iaa+iap)
+
+        # 项目属性
+        self.need_events = need_events
+        self.need_event_attrs = need_event_attrs
+        self.feature_names = feature_names
+        if self.base_user_table in self.feature_names:
+            raise Et(2, f"attr name '{self.base_user_table}' cannt be used")
+        feature_names_all = {
+            self.base_user_table: '用户属性和标签',  # 此属性固定sql，不需要拼接
+        }
+        feature_names_all.update(self.feature_names)
+        self.feature_names = feature_names_all
+        self.sub_sql_fs = sub_sql_fs
+        #
+        self.user_cols1 = user_cols1
+        self.user_cols2 = user_cols2
+        self.event_cols1 = event_cols1
+        self.addon_attrs = addon_attrs if isinstance(addon_attrs, list) else []
+        # 其他属性
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def get_first_col_name(self):
+        """
+        @des: 获取第一个特征属性列
+        """
+        return f"{list(self.feature_names.keys())[1]}_0_v"
+
+    def get_real_feature_names(self, mode='d'):
+        """
+        @des: 获取真正的特征值列表
+        """
+        real_features_names = {}
+        sub_len = len(self.gconf.level_durs)
+        for k, v in self.feature_names.items():
+            if k != self.base_user_table:
+                for i in range(sub_len):
+                    if mode == 'h':
+                        rv = f"{v}_{int(self.gconf.level_durs[i]/3600)}h"
+                    else:
+                        js = self.gconf.level_durs[i]/86400
+                        if js == int(js):
+                            js = int(js)
+                        else:
+                            js = round(js, 1)
+                        rv = f"{v}_{js}d"
+                    real_features_names[f"{k}_{i}_v"] = rv
+        return real_features_names
+
+    def get_threshold_sql(self, col_name):
+        """
+        @des: 根据col_name例如 g_1_v获取特征值计算的sql
+        """
+        first_col_name, second_col_index = col_name.split(
+            "_")[0], col_name.split("_")[1]
+        sub_field_name = "%s_%s_v" % (first_col_name, second_col_index)
+        real_des = "%s %s" % (self.feature_names.get(
+            first_col_name), second_col_index)
+        level_dur = self.gconf.level_durs[int(second_col_index)]
+        f_sub_sql = self.sub_sql_fs.get(first_col_name)
+        if not f_sub_sql:
+            raise Et(2, f"error first_col_name {first_col_name}")
+        sub_sql = f_sub_sql % {
+            "real_des": real_des, "level_dur": level_dur, 'sub_field_name': sub_field_name}
+        return sub_sql
+
+    def get_sql_config(self):
+        """
+        @des: 计算各种时间间隔和配置，统一从一个方法读取，后面方便改
+        """
+        gconf = self.gconf
+        #
+        part_date_start = gconf.part_date_start
+        part_date_end = gconf.part_date_end
+        days = (arrow.get(part_date_end)-arrow.get(part_date_start)).days
+        # 取用户新增后的N天作为事件池数据来取特征值，比如7日预测14日，就取8-9比较合适，根据level_durs[-1]决定
+        after_feature_day = int(gconf.level_durs[-1]/86400)+2  # 多取两天即可
+        feature_date_end = arrow.get(part_date_end).shift(days=after_feature_day).format(
+            "YYYY-MM-DD")  # 增加到8天的事件数据，因为现在有7日预测30日
+        label_durs = gconf.label_durs
+        label_date_start = arrow.get(part_date_start).shift(
+            days=int(label_durs[0]/86400)-2).format("YYYY-MM-DD")
+        label_date_end = arrow.get(part_date_end).shift(
+            days=int(label_durs[1]/86400)+2).format("YYYY-MM-DD")
+        #
+        new_device_start = arrow.get(part_date_start).format("YYYY-MM-DD")
+        new_device_end = arrow.get(part_date_end).shift(
+            days=1).format("YYYY-MM-DD")
+        comm_cc =  {
+            "days": days,
+            'part_date_start': part_date_start,
+            'part_date_end': part_date_end,
+            'feature_date_end': feature_date_end,
+            'label_date_start': label_date_start,
+            'label_date_end': label_date_end,
+            'new_device_start':  new_device_start,
+            'new_device_end':  new_device_end
+        }
+        # print(111111, comm_cc)
+        return comm_cc
+
+    def get_event_sql(self):
+        """
+        @des: 获取N日ltv用户数据
+        """
+        gconf = self.gconf
+        comm_cc = self.get_sql_config()
+        #
+        sql = f"""
+    -- des: add by yuanxiao for machine learn train
+    with new_user as (
+                -- 获取指定日期的注册用户 连接 这些用户在7-10天后的触发的事件
+                SELECT
+                    a1.user_register_time as user_register_time,
+                    a1."#user_id" as user_user_id,
+                    b1."#user_id" as event_user_id,
+                    CASE WHEN b1."yiap__itemrevenue" is NULL THEN 0 ELSE b1."yiap__itemrevenue" END yiap__itemrevenue,    
+                    CASE WHEN b1."sdk_ad_price" is NULL THEN 0 ELSE b1."sdk_ad_price" END sdk_ad_price,
+                    floor(to_unixtime(b1.event_time_utc))-floor(to_unixtime(a1.user_register_time)) as pd_rt_dur
+                FROM
+                (
+                    -- 获取指定日期的注册用户,以防万一，使用窗口函数去个重
+                    SELECT
+                        *
+                    FROM
+                    (
+                        SELECT
+                            *,
+                            row_number()  OVER  (PARTITION  BY  "#user_id" ORDER BY user_register_time ) AS  row_no
+                        FROM
+                        (
+                            SELECT
+                                b.event_time_utc as user_register_time, a."#user_id"
+                            FROM
+                            (
+                                SELECT
+                                    "#user_id"
+                                FROM
+                                    {gconf.tga_user_table}
+                                WHERE
+                                    {self.user_cols1}
+                            ) a
+                            INNER JOIN
+                            (
+                                SELECT
+                                    *
+                                FROM (
+                                    SELECT
+                                        "#user_id",
+                                        {self.tran_dt_by_zone("#event_time", "#zone_offset", 0)} as event_time_utc,
+                                        "$part_event"
+                                    FROM
+                                        {gconf.tga_event_table}
+                                    WHERE
+                                        "$part_event" = 'new_device'
+                                        AND "$part_date" >= '{arrow.get(comm_cc["new_device_start"]).shift(days=-2).format("YYYY-MM-DD")}'
+                                        AND "$part_date" <= '{arrow.get(comm_cc["new_device_end"]).shift(days=2).format("YYYY-MM-DD")}'
+                                        AND {self.user_cols2}
+                                )
+                                WHERE
+                                    event_time_utc >= timestamp '{comm_cc["new_device_start"]}'
+                                    AND event_time_utc < timestamp '{comm_cc["new_device_end"]}'
+                            ) b
+                            ON a."#user_id" = b."#user_id"
+                        )
+                    )
+                    WHERE row_no=1
+                ) a1
+                LEFT JOIN
+                (
+                    -- 获取指定日期的7-10天后的所有事件
+                    SELECT
+                        *
+                    FROM (
+                        SELECT
+                            "#user_id", 
+                            "yiap__itemrevenue",  --- 美元
+                            "sdk_ad_price", -- 美分
+                            "$part_event",
+                            {self.tran_dt_by_zone("#event_time", "#zone_offset", 0)} as event_time_utc
+                        FROM
+                            {gconf.tga_event_table}
+                        WHERE
+                            "$part_event" in ('server_iap', 'impression_ad')
+                            AND "$part_date" >= '{arrow.get(comm_cc["label_date_start"]).shift(days=-2).format("YYYY-MM-DD")}'
+                            AND "$part_date" <= '{arrow.get(comm_cc["label_date_end"]).shift(days=2).format("YYYY-MM-DD")}'
+                    )
+                    WHERE
+                        event_time_utc >= timestamp '{comm_cc["label_date_start"]}'
+                        AND event_time_utc <= timestamp '{comm_cc["label_date_end"]}' 
+                ) b1
+                ON a1."#user_id" = b1."#user_id"
+    )
+    , user_label_table as (
+        -- 获取指定日期的注册用户 连接 这些用户在7-10天后的触发的事件 的 数量 为0[流失用户] 和 大于0[留存优质用户] 的数量
+        SELECT
+            -- SUM(CASE WHEN a3.event_count=0 THEN 1 ELSE 0 END) as miss_count,
+            -- SUM(CASE WHEN a3.event_count>0 THEN 1 ELSE 0 END) as high_value_count
+            a3.revenue_price as is_rd, -- 将指定mode的ltv值作为标签
+            a3.user_user_id,
+            a3.user_register_time,
+            -- 计算固定随机值，打乱顺序
+            (abs(from_ieee754_64(xxhash64(cast(cast(user_user_id as varchar) as varbinary)))) % 100) / 100. as tt_stable_rand 
+        FROM
+        (
+            -- 获取指定日期的注册用户 连接 这些用户在7-10天后的触发的事件 的 数量
+            SELECT
+               a2_1.user_user_id,
+               a2_1.user_register_time,
+               {
+                [
+                    "cast((a2_1.itemrevenue + a2_1.ad_price) as double) as revenue_price -- 将iaa和iap的值相加，作为标签",
+                    "cast(a2_1.itemrevenue as double) as revenue_price -- 将iap作为标签",
+                    "cast(a2_1.ad_price as double) as revenue_price -- 将iaa作为标签",
+                ][['iaa+iap','iap','iaa'].index(self.mode)]
+               }
+            FROM (
+                SELECT
+                    a2.user_user_id as user_user_id,
+                    a2.user_register_time as user_register_time,
+                    SUM(
+                        CASE
+                            WHEN  a2.pd_rt_dur IS NULL THEN 0
+                            WHEN a2.pd_rt_dur<{gconf.label_durs[0]} OR a2.pd_rt_dur>={gconf.label_durs[1]} THEN 0
+                            ELSE a2."yiap__itemrevenue"
+                        END
+                    ) AS itemrevenue,  -- 美元
+                    SUM(
+                        CASE
+                            WHEN  a2.pd_rt_dur IS NULL THEN 0
+                            WHEN a2.pd_rt_dur<{gconf.label_durs[0]} OR a2.pd_rt_dur>={gconf.label_durs[1]} THEN 0
+                            ELSE a2."sdk_ad_price"
+                        END
+                    )/1000 AS ad_price  -- 这里除以1000变成美元
+                FROM
+                (
+                    new_user
+                ) a2
+                GROUP BY a2.user_user_id, a2.user_register_time
+            ) a2_1
+        ) a3
+    )
+    , nav_table as (
+        select  
+            *
+        from 
+            user_label_table  
+        where 
+            is_rd>={self.line_value_rd}
+        -- 固定排序，防止找不到用户, 不适用于超大数据量
+        order by tt_stable_rand
+        limit {6000* comm_cc['days']}
+    )
+    , neg_table as (
+        select  
+            *
+        from 
+            user_label_table  
+        where 
+            is_rd<={self.line_value_rd}
+        -- 固定排序，防止找不到用户, 不适用于超大数据量
+        order by tt_stable_rand
+        limit {12000* comm_cc['days']}
+    )
+    , union_all as (
+        -- 保持正负样本固定比例1:2
+        select
+            *
+        from (
+            (
+                select is_rd,user_register_time, user_user_id  from nav_table
+            )
+            UNION ALL
+            (
+                select
+                    is_rd,user_register_time, user_user_id
+                from (
+                    SELECT *, row_number() OVER (
+                        PARTITION BY is_rd 
+                        ORDER BY tt_stable_rand
+                    ) AS kere_nopoa_end_0
+                    FROM neg_table
+                ) where kere_nopoa_end_0 < (select count(1)*{self.nav_neg_multi} from nav_table)
+            ) 
+        ) st
+    )
+    , all_need_event as (
+        SELECT
+            *
+        FROM (
+            -- 获取正负标签样本需要的事件
+            select 
+                a5.is_rd,
+                a5.user_register_time,
+                a5.user_user_id,
+                floor(to_unixtime(b5.event_time_utc))-floor(to_unixtime(a5.user_register_time)) as rt_dur,
+                c5."afrawip__meida_source" as "afrawip__meida_source_ikfdssausercommonend",
+                c5."#account_id" as "#account_id_ikfdssausercommonend",
+                c5."#distinct_id" as "#distinct_id_ikfdssausercommonend",
+                b5.*,
+                b5.event_time_utc as "#event_time"
+            from 
+                union_all a5
+            INNER JOIN
+            (
+                SELECT
+                    *
+                FROM (
+                    SELECT
+                        -- *
+                        {','.join(['"%s"'%x for x in self.need_event_attrs])},
+                        {self.tran_dt_by_zone("#event_time", "#zone_offset", 0)} as event_time_utc
+                    FROM
+                        {gconf.tga_event_table}
+                    WHERE
+                        "$part_date" >= '{arrow.get(comm_cc["part_date_start"]).shift(days=-2).format("YYYY-MM-DD")}'
+                        AND "$part_date" <= '{arrow.get(comm_cc["feature_date_end"]).shift(days=2).format("YYYY-MM-DD")}'
+                        AND "$part_event" in ({','.join(["'%s'"%x for x in self.need_events])})
+                        AND {self.event_cols1}        
+                )
+                WHERE
+                    event_time_utc >= timestamp '{comm_cc["part_date_start"]}'
+                    AND event_time_utc <= timestamp '{comm_cc["feature_date_end"]}'   
+            )
+            as b5
+            ON a5.user_user_id = b5."#user_id"
+            INNER JOIN
+            (
+                SELECT
+                    "#user_id","afrawip__meida_source","#account_id","#distinct_id"
+                FROM
+                    {gconf.tga_user_table}
+            )
+            as c5
+            ON a5.user_user_id = c5."#user_id"
+        )  a6
+        WHERE
+        rt_dur>=0 AND rt_dur<={gconf.level_durs[-1]}  -- N小时内的事件
+    ) 
+    """
+        return sql
+
+    def get_sub_sql_i(self, sub_sql_f, col_name, col_des, base_event_table='all_need_event', base_user_table='a'):
+        """
+        @des: 内部调用
+        """
+        gconf = self.gconf
+        sub_field_f = "%(col)s_%(dur_i)s"
+        real_des_f = "%(des)s %(dur_i)s"
+        sub_field_names = []
+        sub_sqls = []
+        for _dur_i, level_dur in enumerate(gconf.level_durs):
+            real_des = real_des_f % {'des': col_des, 'dur_i': _dur_i}
+            sub_field_name = sub_field_f % {'col': col_name, 'dur_i': _dur_i}
+            sub_field_names.append(sub_field_name)
+            # sub_sql_f格式如下
+            # sub_sql_f = """
+            #     --%(real_des)s
+            #     sum(
+            #         if(
+            #             "$part_event"=='act_level_path' and object_type='normal' and act='win' and rt_dur<%(level_dur)s,
+            #             1,
+            #             0)
+            #     ) as %(sub_field_name)s
+            # """
+            sub_sql = sub_sql_f % {
+                'real_des': real_des,
+                'level_dur': level_dur,
+                'sub_field_name': sub_field_name
+            }
+            sub_sqls.append(sub_sql)
+        return sub_field_names, sub_sqls
+
+    def get_feature_sql(self):
+        """
+        @des: 提取特征值的sql
+        """
+        gconf = self.gconf
+        """
+        构建level_durs的sql
+        """
+        base_event_table = self.base_event_table
+        base_user_table = self.base_user_table
+        base_feature_table = self.base_feature_table
+        feature_names = self.feature_names
+        comm_cc = self.get_sql_config()
+        #
+        sub_sqls = []
+        sub_field_names = []
+        for col_name, col_des in feature_names.items():
+            sub_sql_f = ""
+            if col_name == 'a':
+                continue
+            else:
+                sub_sql_f = self.sub_sql_fs.get(col_name)
+                if not sub_sql_f:
+                    raise Et(2, f"error col_name: {col_name} ")
+            # 拼接结果
+            if sub_sql_f:
+                _sub_field_names, _sub_sqls = self.get_sub_sql_i(
+                    sub_sql_f, col_name, col_des)
+                sub_sqls.extend(_sub_sqls)
+                sub_field_names.extend(_sub_field_names)
+
+        """
+        构建select选项
+        """
+        # 带表名的字段 表名.field
+        base_user_table_fields = [
+            "is_rd", "user_user_id", "afrawip__meida_source_ikfdssausercommonend", "#lib", "#country_code"] 
+        for addon_attr in self.addon_attrs:
+            _addon_attr = addon_attr.strip()
+            if _addon_attr not in base_user_table_fields:
+                base_user_table_fields.append(_addon_attr)
+        #
+        sub_selects = ",".join(
+            [f"{base_feature_table}.{item} as {item}_v" for item in sub_field_names])
+        sub_selects = f"""{",".join(['%s."%s"'%(base_user_table, ptf) for ptf in base_user_table_fields])},{sub_selects}"""
+        # 不带表名的字段
+        # sub_selects_unique = ",".join(
+        #     [f'"{item}_v"' for item in sub_field_names])
+        # sub_selects_unique = f"""{",".join(['"%s"'%ptf for ptf in base_user_table_fields])},{sub_selects_unique}"""
+        """
+        构建sub_sqls
+        """
+        sub_join_sqls = ",".join(sub_sqls)
+        """
+        拼接最后的sql
+        """
+        sql = f"""
+    , user_tzz as (
+        select 
+            {sub_selects}
+        from (
+                --用户属性和标签
+                select 
+                    {",".join(['"%s"'%ptf for ptf in base_user_table_fields])}
+                from
+                (
+                    select
+                        {",".join(['"%s"'%ptf for ptf in base_user_table_fields])},
+                        row_number()  OVER  (PARTITION  BY  user_user_id ORDER  BY  "#event_time" asc) AS  row_no 
+                    from 
+                        {base_event_table} 
+                )
+                where
+                    row_no=1
+        ) {base_user_table} 
+        left join
+        (   --- 特征值计算
+            select
+                user_user_id,
+                {sub_join_sqls}
+            from
+                {base_event_table}
+            group by user_user_id
+        ) {base_feature_table}
+        on {base_user_table}.user_user_id={base_feature_table}.user_user_id
+    )
+    -- with结束没有逗号, 过滤左连接没有特征值的行，调试的时候取消where条件
+    select * from user_tzz where {self.get_first_col_name()} is not NULL
+    """
+        return sql
+
+    def get_sql(self):
+        # print("---------------------------------")
+        event_sql = self.get_event_sql()
+        feature_sql = self.get_feature_sql()
+        sql = f"""
+    {event_sql}
+    {feature_sql} 
+    """
+        return sql

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck/tga/train_sql_retain.py

@@ -11,7 +11,7 @@ gconf的属性有这些:
    level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
    label_durs       # 取多少边界算留存，例如[int(86400*14), int(86400*17)]，整型数组；这里是取14到17天发生过事件的作为正向用户
    tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
-   tga_event_table  # tga用户表名, 例如 "v_user_7", 字符串 
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
 ########
 
 如何使用(需要自己定义一个子类继承这个类， 代码如下): 

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck/tga/train_sql_yh.py

@@ -9,9 +9,9 @@ gconf的属性有这些:
    part_date_start  # 特征值取新增用户的开始日期, 例如"2022-05-20", 字符串类型
    part_date_end    # 特征值取新增用户的结束日期, 例如"2022-05-25", 字符串类型
    level_durs       # 特征取值段，由小到大排列, 例如 [43200, 43200*2], 这里是秒为单位，整型数组；多少时间段发生事件作为特征值
-   label_durs       # 取多少边界算留存，例如[int(86400*14), int(86400*17)]，整型数组；这里是取14到17天发生高粘度事件的作为正向用户
+   label_durs       # 取多少边界算硬核用户标签，例如[int(86400*14), int(86400*17)]，整型数组；这里是取14到17天发生高粘度事件的作为正向用户
    tga_user_table   # tga用户表名, 例如 "v_user_7", 字符串
-   tga_event_table  # tga用户表名, 例如 "v_user_7", 字符串 
+   tga_event_table  # tga事件表名, 例如 "v_event_7", 字符串 
 ########
 
 如何使用(需要自己定义一个子类继承这个类， 代码如下): 

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2/tfduck_bsd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.16.1
+Version: 0.16.2
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.16.1 → tfduck-bsd-0.16.2}/tfduck_bsd.egg-info/SOURCES.txt

@@ -26,9 +26,12 @@ tfduck/sagemaker/__init__.py
 tfduck/sagemaker/saoper.py
 tfduck/tga/__init__.py
 tfduck/tga/base_tga.py
+tfduck/tga/predict_sql_ltv.py
 tfduck/tga/predict_sql_retain.py
 tfduck/tga/predict_sql_yh.py
 tfduck/tga/tga.py
+tfduck/tga/tga_test.py
+tfduck/tga/train_sql_ltv.py
 tfduck/tga/train_sql_retain.py
 tfduck/tga/train_sql_yh.py
 tfduck/thinkdata/__init__.py

tfduck-bsd-0.16.1/tfduck/__init__.py

@@ -1 +0,0 @@
-__version__="0.16.1"