tfduck-bsd 0.18.5__tar.gz → 0.18.7__tar.gz

{tfduck-bsd-0.18.5/tfduck_bsd.egg-info → tfduck-bsd-0.18.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.5
+Version: 0.18.7
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.18.5 → tfduck-bsd-0.18.7}/setup.py

@@ -8,7 +8,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="tfduck-bsd",
-    version="0.18.5",
+    version="0.18.7",
     author="yuanxiao",
     author_email="yuan6785@163.com",
     description="A small example package",

tfduck-bsd-0.18.7/tfduck/__init__.py

@@ -0,0 +1 @@
+__version__="0.18.7"

tfduck-bsd-0.18.7/tfduck/serverless_k8s/k8s_manage.py

@@ -0,0 +1,228 @@
+"""
+auhtor: yuanxiao
+datetime: 20210423
+python3.8
+pyspark 提交阿里云ask集群管理器
+pip install kubernetes==12.0.1 oss2==2.19.1
+
+如果报错【CA_KEY_TOO_SMALL】则执行(因为k8s的老的秘钥创建是1024位的):
+参考: https://www.cnblogs.com/dingnosakura/p/17815410.html
+"""
+
+if 1:
+    # 解决 【CA_KEY_TOO_SMALL】的报错，利用猴子补丁[monkey Patching ] ，适用于urllib3==2.x版本
+    import urllib3
+    from urllib3.util.ssl_ import create_urllib3_context
+
+    # 创建自定义 SSL 上下文（复用 urllib3 默认配置基础上修改）
+    ctx = create_urllib3_context()
+    ctx.set_ciphers("DEFAULT@SECLEVEL=0")  # 例如降低安全级别
+    ctx.load_default_certs()  # 确保加载系统证书
+
+    # 保存原始 PoolManager 初始化方法（便于回滚）
+    _original_poolmanager_init = urllib3.PoolManager.__init__
+
+    def _patched_poolmanager_init(self, *args, **kwargs):
+        # 仅在未显式传递 ssl_context 时注入自定义上下文
+        if "ssl_context" not in kwargs:
+            kwargs["ssl_context"] = ctx
+        _original_poolmanager_init(self, *args, **kwargs)
+
+    # 应用补丁（确保只执行一次）
+    if urllib3.PoolManager.__init__ != _patched_poolmanager_init:
+        urllib3.PoolManager.__init__ = _patched_poolmanager_init
+
+import json
+
+# from tfduck.common.defines import BMOBJ, Et
+# from tfduck.oss.oss import AliyunOss
+from kubernetes import client, config, utils
+import uuid
+import arrow
+import os
+import base64
+import subprocess
+from io import BytesIO, StringIO
+import yaml
+
+
+class K8sManage(object):
+    """ """
+
+    def __init__(
+        self, ctx, k8s_conn_yaml=None, k8s_conn_json=None, namespace="default"
+    ):
+        """
+        @des: 初始化
+        k8s_conn_yaml ---- k8s的配置文件，从阿里云下载的，保存为yaml文件
+        k8s_conn_json ---- 从k8s的配置yaml文件解析成的json对象,解析方法: (list(yaml.safe_load_all(open(self.k8s_conn_config, "r")))[0])
+        """
+        """
+        全局初始化
+        """
+        self.ctx = ctx
+        self.k8s_conn_config = k8s_conn_yaml
+        self.k8s_conn_json = k8s_conn_json
+        self.namespace = namespace
+        """
+        生成k8s客户端
+        """
+        self.k8s_client = self.get_k8s_client(mode="v1")
+        self.k8s_api_client = self.get_k8s_client(mode="api")
+        #
+        # print(self.k8s_client, self.k8s_api_client)
+
+    def get_k8s_client(self, mode="v1"):
+        """
+        @des: 获取k8s操作客户端
+        例子  https://github.com/kubernetes-client/python/tree/master/examples
+        api文档 https://github.com/kubernetes-client/python/blob/master/kubernetes/README.md
+        """
+        # if 1:
+        #     # 从yaml生成json文件进行登录，方便后面程序配置
+        #     with open(self.k8s_conn_config, "r") as f:
+        #         config_jsons = yaml.safe_load_all(f)
+        #         config_json = list(config_jsons)[0]
+        #         print(111, config_json)
+        #         config.load_kube_config_from_dict(config_json)
+        # else:
+        #     config.load_kube_config(self.k8s_conn_config)
+        if self.k8s_conn_config:
+            config.load_kube_config(self.k8s_conn_config)
+        elif self.k8s_conn_json:
+            # 这样获取
+            # with open(self.k8s_conn_config, "r") as f:
+            #     config_jsons = yaml.safe_load_all(f)
+            #     k8s_conn_json = list(config_jsons)[0]
+            config.load_kube_config_from_dict(self.k8s_conn_json)
+        if mode == "v1":
+            c = client.CoreV1Api()
+        elif mode == "api":
+            c = client.ApiClient()
+        return c
+        # print("Listing pods with their IPs:")
+        # ret = v1.list_pod_for_all_namespaces(watch=False)
+        # for i in ret.items:
+        #     print("%s\t%s\t%s" %
+        #           (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
+
+    def get_all_pods(self):
+        """
+        @des: 获取所有的k8s的pod, 默认命名空间的
+        """
+        # print("Listing pods with their IPs:")
+        ret = self.k8s_client.list_pod_for_all_namespaces(watch=False)
+        pod_infos = []
+        for i in ret.items:
+            if i.metadata.namespace == self.namespace:
+                pod_infos.append(
+                    {
+                        "name": i.metadata.name,
+                        "status": i.status.phase,
+                        "create_time": i.metadata.creation_timestamp,
+                    }
+                )
+        return pod_infos
+
+    def get_pod_info(self, pod_name):
+        """
+        @des: 获取pod的基础信息，状态等信息
+        """
+        # resp = self.k8s_client.read_namespaced_pod_status(
+        #     name="tfspark-test-7200-driver-1e5ab8f2a41d11eb93c3acde48001122", namespace='default')
+        # resp = self.k8s_client.read_namespaced_pod(
+        #     name="tfspark-test-7200-driver-1e5ab8f2a41d11eb93c3acde48001122", namespace='default')
+        pod_obj = self.k8s_client.read_namespaced_pod(
+            name=pod_name, namespace=self.namespace
+        )
+        # print(resp)
+        # print(resp.metadata.name, resp.status.phase)
+        # return resp.status.phase # Pending,Succeeded, Running, Failed
+        return pod_obj
+
+    def get_pod_log(self, pod_name, lines_count=1000):
+        """
+        @des: 获取pod的日志--测试成功
+        """
+        # print(help(self.k8s_client.read_namespaced_pod_log))
+        # log_content = self.k8s_client.read_namespaced_pod_log(
+        #     'tfspark-test-7200-driver-c697aadea41d11eb9d2bacde48001122', 'default', pretty=True, tail_lines=1)  # container='spark-kubernetes-driver'
+        log_content = self.k8s_client.read_namespaced_pod_log(
+            pod_name, self.namespace, pretty=True, tail_lines=lines_count
+        )  # tail_lines 表示从结尾开始获取日志的行数
+        # print(log_content)
+        return log_content
+
+    # def patch_pod_info(self, pod_name):
+    #     """
+    #     @des: 修改pod的信息，包括状态
+    #     """
+    #     pod_obj = self.get_pod_info(pod_name)
+    #     pod_obj.status.phase = ""
+    #     # print(help(self.k8s_client.patch_namespaced_pod))
+    #     self.k8s_client.patch_namespaced_pod(pod_name, 'default', pod_obj)
+    def create_by_yaml(self, yaml_file):
+        """ """
+        result = utils.create_from_yaml(
+            k8s_client=self.k8s_api_client,
+            yaml_file=yaml_file,
+            namespace=self.namespace,
+        )
+        # print(result)
+        # print("count: ", len(result))
+
+    def create_by_yaml_str(self, yaml_str):
+        """
+        @des: 通过yaml字符串创建pod
+        """
+        with StringIO(yaml_str) as f:
+            yml_document_all = yaml.safe_load_all(f)
+            failures = []
+            k8s_objects = []
+            for yml_document in yml_document_all:
+                # print(111, json.dumps(yml_document)) ---- 后面也可以编写json创建
+                try:
+                    created = utils.create_from_dict(
+                        k8s_client=self.k8s_api_client,
+                        data=yml_document,
+                        namespace=self.namespace,
+                    )
+                    k8s_objects.append(created)
+                except utils.FailToCreateError as failure:
+                    failures.extend(failure.api_exceptions)
+            if failures:
+                raise utils.FailToCreateError(failures)
+        # print(k8s_objects)
+        # print("count: ", len(k8s_objects))
+        return k8s_objects
+
+    def create_by_json(self, json_obj):
+        """
+        @des: 通过json对象创建pod
+        """
+        failures = []
+        try:
+            created = utils.create_from_dict(
+                k8s_client=self.k8s_api_client,
+                data=json_obj,
+                namespace=self.namespace,
+            )
+        except utils.FailToCreateError as failure:
+            failures.extend(failure.api_exceptions)
+        if failures:
+            raise utils.FailToCreateError(failures)
+        return created
+
+    def delete_pod(self, pod_name):
+        """
+        @des: 删除pod
+        delete是删除---
+        delete_namespaced_pod
+        """
+        # print(help(self.k8s_client.delete_namespaced_pod))
+        # pod_obj = self.get_pod_info(pod_name)
+        # print(pod_obj.metadata.name, pod_obj.metadata.creation_timestamp, pod_obj.status.phase)
+        # pod_create = pod_obj.metadata.creation_timestamp
+        # status = pod_obj.status.phase
+        r = self.k8s_client.delete_namespaced_pod(pod_name, self.namespace)
+        return r

tfduck-bsd-0.18.7/tfduck/serverless_k8s/k8s_task.py

@@ -0,0 +1,343 @@
+"""
+auhtor: yuanxiao
+datetime: 20210423
+python3.8
+ask serverless的提交任务
+pip install kubernetes==12.0.1
+"""
+
+import json
+# from tfduck.common.defines import BMOBJ, Et
+
+# from tfduck.oss.oss import AliyunOss
+import arrow
+import time
+import os
+import base64
+import subprocess
+import uuid
+
+if 1:
+    from k8s_manage import K8sManage
+    from k8s_upload_oss import AliyunOss
+
+
+# class YxBMOBJ(object):
+#     def clog(self, ctx, *args):
+#         print(*args)
+#         pass
+
+#     def get_file_path(self, path):
+#         """
+#         @des: 文件的真实路径
+#         """
+#         # self.dj44_base_dir = "/Users/yuanxiao/workspace/djcelery44/djcelery44"
+#         # return os.path.join(self.dj44_base_dir, os.path.join("dags/sptasks/p_code", path))
+#         return os.path.abspath(path)
+
+
+# BMOBJ = YxBMOBJ()
+
+if 1:
+    from tfduck.common.defines import BMOBJ, Et
+    from tfduck.oss.oss import AliyunOss
+
+
+class Et(Exception):
+    def __init__(self, code, msg):
+        self.code = code
+        self.msg = msg
+
+
+class ServerlessTaskManage(object):
+    """ """
+
+    def __init__(
+        self,
+        ctx,
+        code_path,
+        max_run_time=7200,
+        # 参考main的调用
+        task_config={
+            # pod的名称前缀-必填
+            "task_name": "yxtestpodtask",
+            # cpu的使用量-必填
+            "cpu": "500m",
+            # 内存的使用量-必填
+            "memory": "500Mi",
+            # 镜像地址-必填
+            "image_url": "registry-intl.cn-beijing.aliyuncs.com/talefun/python311:base",
+            # serverless集群挂载的oss的pvc的名称,必须在【duck-task】命名空间的pvc-必填
+            "pvc_name": "tfduck-k8s-pvc",
+            # pypi的源，如果为空字符串，则使用官方源
+            "pypi_mirror": "https://pypi.tuna.tsinghua.edu.cn/simple",
+        },
+        #
+        is_debug=False,
+        project_name="playdayy-bj",
+        pull_pod_success_log=True,
+        pull_pod_fail_log=True,
+        is_async=False,
+        project_configs={},
+    ):
+        """
+        ######################################特别注意############################################
+        code_path: 代码路径
+        project_configs: 所有项目配置
+        project_name: 项目名称，在project_configs里面找到对应的key
+        ######################################
+        """
+        """
+        校验
+        """
+        task_sub_name = task_config.get("task_name", "default")
+        if task_sub_name is None:
+            raise Et(2, "task name must not be none")
+        task_sub_name = task_sub_name.strip()
+        if task_sub_name == "":
+            raise Et(2, "task name must not be '' ")
+        if task_sub_name.find("-") != -1:
+            raise Et(2, "task name not clude - char")
+        """
+        工程配置
+        """
+        self.project_name = project_name  # 切换本地调试集群
+        self.project_configs = project_configs
+        self.project_conf = self.project_configs[self.project_name]
+        """
+        全局初始化
+        """
+        self.ctx = ctx
+        self.is_debug = is_debug  # 如果为true，就在mac本地调试
+        self.max_run_time = max_run_time  # pod task最大运行时间
+        self.code_path = code_path
+        self.pull_pod_success_log = pull_pod_success_log
+        self.pull_pod_fail_log = pull_pod_fail_log
+        self.task_config = task_config
+        self.pod_name = f"{task_sub_name}-{self.max_run_time}-{uuid.uuid4().hex[:8]}"
+        self.oss_root_name = "oss_data"
+        self.is_async = is_async
+        """
+        系统oss的配置
+        """
+        self.oss_config = self.project_conf["oss_pv"]
+        """
+        生成k8s客户端
+        """
+        self.name_space = "duck-task"
+        self.k8s_client = K8sManage(
+            ctx,
+            k8s_conn_json=self.project_conf["k8s_serverless_conn"],
+            namespace=self.name_space,
+        )
+
+    def upload_code(self):
+        """
+        上传执行代码到oss
+        """
+        now = arrow.utcnow()
+        now_str = now.format("YYYY-MM-DD")
+        oss_config = self.oss_config
+        ctx = self.ctx
+        uoss = AliyunOss(
+            oss_config["bucket"],
+            oss_config["access_key"],
+            oss_config["secret_key"],
+            oss_config["endpoint_public"],
+        )
+        oss_file_path = f"code/{now_str}/{uuid.uuid4().hex}/"  # 注意必须斜杠结尾
+        if self.code_path:
+            local_file_path = BMOBJ.get_file_path(self.code_path)
+            if not local_file_path.startswith("/"):
+                local_file_path = f"/{local_file_path}"
+            uoss.upload_oss(
+                ctx,
+                local_file_path,
+                oss_file_path,
+                False,
+                False,
+                isrm=True,
+                isdel=False,
+            )  # 递归查找, 不删除本地文件
+        else:
+            raise Et("code_path不能为空")
+        # uoss.download_oss(ctx, local_file_path+"xx", oss_file_path, isrm=True, isdel=False)  # 测试下载
+        self.real_code_path = os.path.join(f"/{self.oss_root_name}", oss_file_path)
+
+    def clean_code(self):
+        """
+        清理代码
+        """
+        ctx = self.ctx
+        oss_config = self.oss_config
+        uoss = AliyunOss(
+            oss_config["bucket"],
+            oss_config["access_key"],
+            oss_config["secret_key"],
+            oss_config["endpoint_public"],
+        )
+        # self.real_code_path 不要self.oss_root_name部分即可
+        oss_code_path = self.real_code_path.replace(f"/{self.oss_root_name}/", "")
+        BMOBJ.clog(ctx, "clean code path:", oss_code_path)
+        uoss.delete_prefix_oss(ctx, oss_code_path, isrm=True)
+
+    def get_submit_task_cmd(self):
+        """
+        构建
+        """
+        pypi_mirror = self.task_config.get("pypi_mirror", "")
+        pypi_mirror_host = (
+            pypi_mirror.split("://")[1].split("/")[0] if pypi_mirror else ""
+        )
+        pypi_i = (
+            f"-i {pypi_mirror} --trusted-host {pypi_mirror_host}" if pypi_mirror else ""
+        )
+        """
+        json命令，参考该目录的k8s_manage.py的如何根据yaml解析json的方式
+        参考yaml_to_json.py
+        """
+        cmd_json = {
+            "apiVersion": "v1",
+            "kind": "Pod",
+            "metadata": {"name": self.pod_name, "namespace": self.name_space},
+            "spec": {
+                "containers": [
+                    {
+                        "name": "python311-container",
+                        "image": self.task_config["image_url"],
+                        "command": ["/bin/bash", "-c"],
+                        "args": [
+                            f"""cd {self.real_code_path} && \
+                                python3 --version && \
+                                    pip install -U pip {pypi_i} && \
+                                    pip install arrow {pypi_i} && \
+                                    pip install -r requirements.txt {pypi_i} && \
+                                    python main.py
+                            """
+                        ],
+                        "resources": {
+                            "requests": {
+                                "memory": self.task_config["memory"],
+                                "cpu": self.task_config["cpu"],
+                            },
+                            "limits": {
+                                "memory": self.task_config["memory"],
+                                "cpu": self.task_config["cpu"],
+                            },
+                        },
+                        "volumeMounts": [
+                            {
+                                "name": "tfduck-k8s-pv",
+                                "mountPath": f"/{self.oss_root_name}",
+                            }
+                        ],
+                    }
+                ],
+                "volumes": [
+                    {
+                        "name": "tfduck-k8s-pv",
+                        "persistentVolumeClaim": {
+                            "claimName": self.task_config["pvc_name"]
+                        },
+                    }
+                ],
+                "restartPolicy": "Never",
+            },
+        }
+        return cmd_json
+
+    def submit_task(self):
+        """
+        提交任务
+        """
+        result = True
+        ctx = self.ctx
+        """
+        执行任务
+        """
+        submit_json = self.get_submit_task_cmd()
+        try:
+            self.k8s_client.create_by_json(submit_json)
+        except Exception as e:
+            raise e
+        """
+        判断任务是否完成---不从日志里面去判断，这样会拉回来很多日志
+        """
+        pod_obj = self.k8s_client.get_pod_info(self.pod_name)
+        if self.is_async:
+            BMOBJ.clog(ctx, pod_obj.status.phase)
+            if pod_obj.status.phase in ["Pending", "Running"]:
+                BMOBJ.clog(ctx, "task success runing")
+            else:
+                BMOBJ.clog(ctx, "task fail")
+                result = False
+        else:
+            start = time.time()
+            #
+            wait_time = 0
+            # 第一阶段判断--进入Running状态--
+            pd_1 = 0
+            pd_1_status = ["Running"]
+            max_pd_1 = 30  # 最大等待次数
+            while pod_obj.status.phase not in pd_1_status and pd_1 < max_pd_1:
+                time.sleep(10)
+                try:
+                    pod_obj = self.k8s_client.get_pod_info(self.pod_name)
+                except Exception as _:
+                    pass
+                pd_1 += 1
+            if pod_obj.status.phase not in pd_1_status:
+                BMOBJ.clog(
+                    ctx,
+                    f"task fail in {time.time() - start} seconds state: {pod_obj.status.phase}",
+                )
+                result = False
+                return result
+            # 第二阶段判断--进入Succeeded或者Failed状态--
+            while (
+                pod_obj.status.phase not in ["Succeeded", "Failed"]
+                or wait_time > self.max_run_time
+            ):
+                time.sleep(10)
+                try:
+                    pod_obj = self.k8s_client.get_pod_info(self.pod_name)
+                except Exception as _:
+                    pass
+                wait_time = time.time() - start
+            #
+            BMOBJ.clog(ctx, f"pod status: {pod_obj.status.phase}")
+            #
+            if pod_obj.status.phase == "Succeeded":
+                BMOBJ.clog(ctx, "task success complated")
+                if self.pull_pod_success_log:
+                    success_log = self.k8s_client.get_pod_log(self.pod_name, 500)
+                    BMOBJ.clog(ctx, "pod task success log ----------:", success_log)
+            else:
+                BMOBJ.clog(ctx, "task fail-------------------error log:")
+                if self.pull_pod_fail_log:
+                    error_log = self.k8s_client.get_pod_log(self.pod_name, 1000)
+                    BMOBJ.clog(ctx, "pod task fail log ----------:", error_log)
+                result = False
+        return result
+
+    def sync(self):
+        """
+        all in one
+        """
+        try:
+            self.upload_code()
+            result = self.submit_task()
+        finally:
+            if not self.is_async:
+                # 清理代码
+                try:
+                    self.clean_code()
+                except Exception as _:
+                    pass
+                # 清理pod(我觉得正式环境不需要在这里清理，定时调度清理即可)
+                if self.is_debug:
+                    try:
+                        self.k8s_client.delete_pod(self.pod_name)
+                    except Exception as _:
+                        pass
+        return result

{tfduck-bsd-0.18.5 → tfduck-bsd-0.18.7}/tfduck/tga/tga.py

@@ -196,6 +196,7 @@ class ThinkDataQuery(BaseTga):
         retry_count=2,
         conn_timeout=30,
         tga_data_timeout=600,
+        retry_wait=1,
     ):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
@@ -228,6 +229,7 @@ class ThinkDataQuery(BaseTga):
                 # else:
                 #     break
                 if 1:
+                    time.sleep(retry_wait)
                     continue
         if gol_e is not None:
             raise gol_e
@@ -322,7 +324,9 @@ class ThinkDataQuery(BaseTga):
                     BMOBJ.clog(ctx, i)
             BMOBJ.clog(ctx, f"total: {i}")
             if len(datas) > 0:  # 保存最后收尾的
-                df = pandas.DataFrame(data=datas, columns=cols, dtype=object)  # 保存表头
+                df = pandas.DataFrame(
+                    data=datas, columns=cols, dtype=object
+                )  # 保存表头
                 # 解决科学计数法的问题
                 df = df.astype(str)
                 df = df.astype("string")
@@ -361,6 +365,7 @@ class ThinkDataQuery(BaseTga):
         retry_count=2,
         conn_timeout=30,
         tga_data_timeout=600,
+        retry_wait=1,
     ):
         """
         @des:从thinkdata的openapi获取数据----流式，为了节省内存---配合下面的getquerycsv
@@ -393,6 +398,7 @@ class ThinkDataQuery(BaseTga):
                 # else:
                 #     break
                 if 1:
+                    time.sleep(retry_wait)
                     continue
         if gol_e is not None:
             raise gol_e
@@ -409,6 +415,7 @@ class ThinkDataQuery(BaseTga):
         print_size=100000,
         conn_timeout=30,
         tga_data_timeout=600,
+        retry_wait=1,
     ):
         """
         @des: 接口装饰器--修改为get_data_csv，防止全面修改代码
@@ -423,6 +430,7 @@ class ThinkDataQuery(BaseTga):
             retry_count,
             conn_timeout,
             tga_data_timeout,
+            retry_wait,
         )
         return result
 

{tfduck-bsd-0.18.5 → tfduck-bsd-0.18.7/tfduck_bsd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.5
+Version: 0.18.7
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.18.5 → tfduck-bsd-0.18.7}/tfduck_bsd.egg-info/SOURCES.txt

@@ -25,6 +25,9 @@ tfduck/s3/__init__.py
 tfduck/s3/s3oper.py
 tfduck/sagemaker/__init__.py
 tfduck/sagemaker/saoper.py
+tfduck/serverless_k8s/__init__.py
+tfduck/serverless_k8s/k8s_manage.py
+tfduck/serverless_k8s/k8s_task.py
 tfduck/tga/__init__.py
 tfduck/tga/base_tga.py
 tfduck/tga/predict_sql_ltv.py

tfduck-bsd-0.18.5/tfduck/__init__.py

@@ -1 +0,0 @@
-__version__="0.18.5"