tfduck-bsd 0.18.9__tar.gz → 0.19.1__tar.gz

{tfduck-bsd-0.18.9/tfduck_bsd.egg-info → tfduck-bsd-0.19.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.9
+Version: 0.19.1
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

{tfduck-bsd-0.18.9 → tfduck-bsd-0.19.1}/setup.py

@@ -8,7 +8,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="tfduck-bsd",
-    version="0.18.9",
+    version="0.19.1",
     author="yuanxiao",
     author_email="yuan6785@163.com",
     description="A small example package",

tfduck-bsd-0.19.1/tfduck/__init__.py

@@ -0,0 +1 @@
+__version__="0.19.1"

{tfduck-bsd-0.18.9 → tfduck-bsd-0.19.1}/tfduck/s3/s3oper.py

@@ -2,6 +2,7 @@
 s3的公共操作
 版本号见setup.py
 """
+
 # coding=utf-8
 import boto3
 import time
@@ -14,7 +15,13 @@ import gzip
 import pprint
 from tfduck.common.defines import BMOBJ, Et
 from botocore.exceptions import ClientError
-from concurrent.futures import ThreadPoolExecutor, as_completed, wait, ALL_COMPLETED, FIRST_COMPLETED
+from concurrent.futures import (
+    ThreadPoolExecutor,
+    as_completed,
+    wait,
+    ALL_COMPLETED,
+    FIRST_COMPLETED,
+)
 from io import StringIO, BytesIO
 from botocore.client import Config
 
@@ -24,7 +31,9 @@ class S3Oper(object):
     @des: S3的公共操作, 上传文件
     """
 
-    def __init__(self, ctx={}, ak="", sk="", bucket="", region_name="", print_files_info=False):
+    def __init__(
+        self, ctx={}, ak="", sk="", bucket="", region_name="", print_files_info=False
+    ):
         """
         @des: 初始化
         """
@@ -44,7 +53,6 @@ class S3Oper(object):
         初始化client
         """
         self.set_s3_client()
-        
 
     def get_s3_config(self):
         """
@@ -52,11 +60,14 @@ class S3Oper(object):
         """
         # connect_timeout 和 read_timeout默认值都是60秒
         # max_pool_connections 最大的连接池，默认10
-        config = Config(connect_timeout=60,
-                        read_timeout=600,
-                        retries={'max_attempts': 0},  # 将默认重试次数设为0，不重试
-                        # 如果是多线程共享client, upload_foler的max_workers必须比这个值小;  目前是每个线程一个client，所以这里不受影响
-                        max_pool_connections=10)
+        config = Config(
+            connect_timeout=60,
+            read_timeout=600,
+            retries={"max_attempts": 0},  # 将默认重试次数设为0，不重试
+            # retries={"max_attempts": 5, "mode": "standard"},  # 将默认重试次数设为5
+            # 如果是多线程共享client, upload_foler的max_workers必须比这个值小;  但是目前_upload_i是每个线程一个client，所以这里就算超过也不受影响
+            max_pool_connections=10,
+        )
         # s3 = boto3.client('s3', config=config)
         return config
 
@@ -67,7 +78,7 @@ class S3Oper(object):
         s3_session = boto3.Session(
             aws_access_key_id=self.access_key,
             aws_secret_access_key=self.secret_key,
-            region_name=self.region_name
+            region_name=self.region_name,
         )
         return s3_session
 
@@ -77,7 +88,7 @@ class S3Oper(object):
         """
         s3_session = self.get_s3_session()
         config = self.get_s3_config()
-        s3_resource = s3_session.resource('s3')
+        s3_resource = s3_session.resource("s3")
         # s3_resource = s3_session.resource('s3', config=config)
         return s3_resource
 
@@ -143,16 +154,16 @@ class S3Oper(object):
         可以超过1000
         """
         paginator = client.get_paginator(
-            'list_objects_v2')  # MaxKeys的最大值为1000， 不能超过1000,
+            "list_objects_v2"
+        )  # MaxKeys的最大值为1000， 不能超过1000,
         # 所以用分页的方式，这样就可以通过分很多页超过1000
-        pages = paginator.paginate(
-            Bucket=self.bucket, Prefix=remote_path, MaxKeys=1000)
+        pages = paginator.paginate(Bucket=self.bucket, Prefix=remote_path, MaxKeys=1000)
         remote_path_parents_count = len(pathlib.Path(remote_path).parents)
         for page in pages:
-            for obj in page['Contents']:
-                key = obj['Key']
+            for obj in page["Contents"]:
+                key = obj["Key"]
                 if not isrm:
-                    if len(pathlib.Path(key).parents)-1 == remote_path_parents_count:
+                    if len(pathlib.Path(key).parents) - 1 == remote_path_parents_count:
                         file_list.append(key)
                 else:
                     file_list.append(key)
@@ -166,23 +177,96 @@ class S3Oper(object):
         s3_client = self.get_s3_client()
         try:
             resp = s3_client.list_objects(
-                Bucket=self.bucket, Delimiter='/', Prefix=remote_path+'/')
+                Bucket=self.bucket, Delimiter="/", Prefix=remote_path + "/"
+            )
             return [d.get("Prefix") for d in resp.get("CommonPrefixes")]
         except ClientError as e:
             BMOBJ.clog(self.ctx, "get s3 folders error:", e)
             return []
 
-    def upload_file(self, local_path, remote_path):
+    # def upload_file(self, local_path, remote_path):
+    #     """
+    #     @des: 将本地文件上传到s3
+    #     """
+    #     s3_client = self.get_s3_client()
+    #     try:
+    #         _ = s3_client.upload_file(local_path, self.bucket, remote_path)
+    #     except ClientError as e:
+    #         BMOBJ.clog(self.ctx, "upload s3 file error:", e)
+    #         return False
+    #     return True
+
+    def upload_file(self, local_path, remote_path, max_attempts=5, initial_delay=1):
         """
-        @des: 将本地文件上传到s3
+        @des: 将本地文件上传到s3，增加了健壮的重试逻辑。
+        @param max_attempts: 最大尝试次数（包括第一次）。
+        @param initial_delay: 初始重试等待时间（秒）。
         """
         s3_client = self.get_s3_client()
-        try:
-            _ = s3_client.upload_file(local_path, self.bucket, remote_path)
-        except ClientError as e:
-            BMOBJ.clog(self.ctx, "upload s3 file error:", e)
-            return False
-        return True
+        last_exception = None
+
+        # 定义哪些S3错误代码是可重试的
+        # 'InternalError' 和 'SlowDown' 是最常见的。'ThrottlingException' 也可能出现。
+        retryable_error_codes = {"InternalError", "SlowDown", "ThrottlingException"}
+
+        for attempt in range(max_attempts):
+            try:
+                # 尝试上传文件
+                _ = s3_client.upload_file(local_path, self.bucket, remote_path)
+                # 如果上传成功，记录日志并直接返回True
+                if attempt > 0:  # 如果不是第一次尝试就成功了，可以加个日志
+                    BMOBJ.clog(
+                        self.ctx,
+                        f"Successfully uploaded {local_path} on attempt {attempt + 1}",
+                    )
+                return True
+            except ClientError as e:
+                last_exception = e
+                error_code = e.response.get("Error", {}).get("Code")
+
+                # 检查错误代码是否在我们的可重试列表中
+                if error_code in retryable_error_codes:
+                    # 如果是最后一次尝试，则不再等待，直接跳出循环去处理失败
+                    if attempt == max_attempts - 1:
+                        BMOBJ.clog(
+                            self.ctx,
+                            f"Upload of {local_path} failed on the final attempt ({max_attempts}). Error: {e}",
+                        )
+                        break
+
+                    # 计算下一次重试的等待时间（指数退避 + 随机抖动）
+                    # 等待时间 = initial_delay * 2^attempt + 随机数(0~1秒)
+                    # 例如：1s, 2s, 4s, 8s, 16s
+                    sleep_time = (initial_delay * (2**attempt)) + random.randint(0, 10)
+
+                    BMOBJ.clog(
+                        self.ctx,
+                        f"Attempt {attempt + 1}/{max_attempts} to upload {local_path} failed with a retryable error: {error_code}. "
+                        f"Retrying in {sleep_time:.2f} seconds...",
+                    )
+                    time.sleep(sleep_time)
+                else:
+                    # 如果是不可重试的错误（如权限问题），立即记录错误并返回False
+                    BMOBJ.clog(
+                        self.ctx,
+                        f"Upload of {local_path} failed with a non-retryable error: {e}",
+                    )
+                    return False
+            except Exception as e:
+                # 捕获其他可能的异常 (如网络问题)，并将其视为失败
+                BMOBJ.clog(
+                    self.ctx,
+                    f"An unexpected error occurred during upload of {local_path}: {e}",
+                )
+                last_exception = e
+                break  # 出现未知异常，终止重试
+
+        # 如果循环结束仍未成功，说明所有重试都失败了
+        BMOBJ.clog(
+            self.ctx,
+            f"Failed to upload {local_path} to {remote_path} after {max_attempts} attempts. Last error: {last_exception}",
+        )
+        return False
 
     def upload_fileobj(self, io_obj, remote_path):
         """
@@ -251,7 +335,7 @@ class S3Oper(object):
         try:
             s3_client.head_object(Bucket=self.bucket, Key=remote_path)
         except ClientError as e:
-            return int(e.response['Error']['Code']) != 404
+            return int(e.response["Error"]["Code"]) != 404
         return True
 
     def delete_file(self, remote_path):
@@ -274,7 +358,7 @@ class S3Oper(object):
         bucket.objects.filter(Prefix=remote_path).delete()
         return True
 
-    def _upload_i(self,  remote_file, local_file, retry_count):
+    def _upload_i(self, remote_file, local_file, retry_count):
         """
         @des: 多线程批量上传
         """
@@ -286,33 +370,39 @@ class S3Oper(object):
         """
         for i in range(retry_count):  # 最多重试三次，由于网络不稳定等问题
             try:
-
                 _s = time.time()
                 _ = s3_client.upload_file(  # 这个方法本来就是分块多线程上传，所以开一个和多个在大文件来说上传速度区别不大
-                    local_file, self.bucket, remote_file)  # 返回值为None
+                    local_file, self.bucket, remote_file
+                )  # 返回值为None
                 _e = time.time()
                 if self.print_files_info:
                     BMOBJ.clog(
-                        ctx, f"{local_file} upload success, sub time {_e-_s}", )
+                        ctx,
+                        f"{local_file} upload success, sub time {_e - _s}",
+                    )
                 break
             except Exception as e:
-                BMOBJ.clog(
-                    ctx, f"{local_file} upload  fail, repeat {i}, error: {e}")
+                BMOBJ.clog(ctx, f"{local_file} upload  fail, repeat {i}, error: {e}")
                 #
-                if i < retry_count-1:
+                if i < retry_count - 1:
                     sleep_time = random.randint(60, 120)
                     time.sleep(sleep_time)
                     continue
                 else:
-                    BMOBJ.clog(
-                        ctx, f"{local_file} upload  finally fail: {e}")
+                    BMOBJ.clog(ctx, f"{local_file} upload  finally fail: {e}")
                     raise Et(2, f"upload fail {remote_file} {local_file}")
 
-    def upload_folder(self, local_path, remote_path,
-                      add_success=False, add_empty=False,
-                      max_workers=50, isrm=True,
-                      isdel=True, retry_count=5
-                      ):
+    def upload_folder(
+        self,
+        local_path,
+        remote_path,
+        add_success=False,
+        add_empty=False,
+        max_workers=50,
+        isrm=True,
+        isdel=True,
+        retry_count=5,
+    ):
         """
         @des: 上传到s3---多线程上传---上传文件夹
         """
@@ -330,17 +420,22 @@ class S3Oper(object):
         total_files = []
         total_size = 0
         for subfile in subfiles:
-            size = round(subfile.stat().st_size/1024/1024, 4)
+            size = round(subfile.stat().st_size / 1024 / 1024, 4)
             total_size += size
             name = subfile.name
             total_files.append(f"{size}M {name}")
-        _infos = '\n'.join(total_files)
+        _infos = "\n".join(total_files)
         if self.print_files_info:
             BMOBJ.clog(
-                ctx, f"""upload file info *  file total count {len(subfiles)}  file total size {total_size}M""", _infos)
+                ctx,
+                f"""upload file info *  file total count {len(subfiles)}  file total size {total_size}M""",
+                _infos,
+            )
         else:
             BMOBJ.clog(
-                ctx, f"""upload file info *  file total count {len(subfiles)}  file total size {total_size}M""")
+                ctx,
+                f"""upload file info *  file total count {len(subfiles)}  file total size {total_size}M""",
+            )
 
         # 参考 https://www.jianshu.com/p/b9b3d66aa0be
         # 控制最大队列数200，记得修改settings.py的redis队列数必须大于这个
@@ -357,7 +452,8 @@ class S3Oper(object):
                 local_file = str(subfile)
                 # 通过submit函数提交执行的函数到线程池中，submit函数立即返回，不阻塞
                 task_i = executor.submit(
-                    self._upload_i, *(remote_file, local_file, retry_count))
+                    self._upload_i, *(remote_file, local_file, retry_count)
+                )
                 all_tasks.append(task_i)
             # 等待所有任务完成后
             # wait(all_tasks, timeout=timeout, return_when=ALL_COMPLETED)
@@ -367,17 +463,16 @@ class S3Oper(object):
             if add_success:
                 # 上传成功后，上传一个空文件代表成功
                 with BytesIO() as f:
-                    with gzip.open(f, 'wb') as r:
-                        r.write(b'')
+                    with gzip.open(f, "wb") as r:
+                        r.write(b"")
                     f.seek(0)
-                    self.upload_fileobj(
-                        f, os.path.join(remote_path, "_SUCCESS"))
+                    self.upload_fileobj(f, os.path.join(remote_path, "_SUCCESS"))
         else:
             if add_empty:
                 # 上传一个empty文件，代表没有数据
                 with BytesIO() as f:
-                    with gzip.open(f, 'wb') as r:
-                        r.write(b'')
+                    with gzip.open(f, "wb") as r:
+                        r.write(b"")
                     f.seek(0)
                     self.upload_fileobj(f, os.path.join(remote_path, "_EMPTY"))
         e = time.time()
@@ -387,8 +482,7 @@ class S3Oper(object):
                 raise Et(2, "cannt del root folder")
             BMOBJ.remove_folder(local_path)
         #
-        BMOBJ.clog(
-            ctx, f"{remote_path} upload all time", e-s)
+        BMOBJ.clog(ctx, f"{remote_path} upload all time", e - s)
 
     def _download_i(self, remote_file, local_file, retry_count):
         """
@@ -402,26 +496,33 @@ class S3Oper(object):
             try:
                 _s = time.time()
                 _ = s3_client.download_file(
-                    self.bucket, remote_file, local_file)  # 返回值为None
+                    self.bucket, remote_file, local_file
+                )  # 返回值为None
                 _e = time.time()
                 BMOBJ.clog(
-                    ctx, f"{local_file} download success, sub time {_e-_s}", )
+                    ctx,
+                    f"{local_file} download success, sub time {_e - _s}",
+                )
                 break
             except Exception as e:
-                BMOBJ.clog(
-                    ctx, f"{local_file} download  fail, repeat {i}, error: {e}")
-                if i < retry_count-1:
+                BMOBJ.clog(ctx, f"{local_file} download  fail, repeat {i}, error: {e}")
+                if i < retry_count - 1:
                     sleep_time = random.randint(60, 120)
                     time.sleep(sleep_time)
                     continue
                 else:
-                    BMOBJ.clog(
-                        ctx, f"{local_file} download  finally fail: {e}")
+                    BMOBJ.clog(ctx, f"{local_file} download  finally fail: {e}")
                     raise Et(2, f"download fail {remote_file} {local_file}")
 
-    def download_folder(self, local_path, remote_path,
-                        max_workers=50, isrm=True,
-                        isdel=True, retry_count=5):
+    def download_folder(
+        self,
+        local_path,
+        remote_path,
+        max_workers=50,
+        isrm=True,
+        isdel=True,
+        retry_count=5,
+    ):
         """
         @des: 下载到本地---多线程下载---下载文件夹--下载后删除s3的文件
         """
@@ -442,15 +543,16 @@ class S3Oper(object):
                 if not isrm:
                     local_file = os.path.join(local_path, subfile_name)
                 else:
-                    l_name = str(pathlib.PurePath(remote_file)
-                                 ).replace(remote_path, "")
+                    l_name = str(pathlib.PurePath(remote_file)).replace(remote_path, "")
                     l_name = l_name.lstrip("/")
                     local_file = os.path.join(local_path, l_name)
-                    os.makedirs(os.path.dirname(local_file),
-                                exist_ok=True)  # 创建不存在的子文件夹
+                    os.makedirs(
+                        os.path.dirname(local_file), exist_ok=True
+                    )  # 创建不存在的子文件夹
                 # 通过submit函数提交执行的函数到线程池中，submit函数立即返回，不阻塞
                 task_i = executor.submit(
-                    self._download_i, *(remote_file, local_file, retry_count))
+                    self._download_i, *(remote_file, local_file, retry_count)
+                )
                 all_tasks.append(task_i)
             # 等待所有任务完成后
             # wait(all_tasks, timeout=timeout, return_when=ALL_COMPLETED)
@@ -464,11 +566,10 @@ class S3Oper(object):
                 raise Et(2, "cannt del root folder")
             self.delete_folder(remote_path)
         #
-        BMOBJ.clog(
-            ctx, f"{remote_path} download all time", e-s)
+        BMOBJ.clog(ctx, f"{remote_path} download all time", e - s)
 
 
-if __name__ == '__main__':  # 打版本的时候一定记得记得脱敏
+if __name__ == "__main__":  # 打版本的时候一定记得记得脱敏
     pass
     # s3 = S3Oper(ctx = {}, ak="xx", sk="yy", bucket="xx", region_name="us-east-2")
     # s3.upload_folder(local_path="/Users/yuanxiao/Downloads/train/samples",

{tfduck-bsd-0.18.9 → tfduck-bsd-0.19.1}/tfduck/serverless_k8s/k8s_task.py

@@ -72,6 +72,8 @@ class ServerlessTaskManage(object):
             "pvc_name": "tfduck-k8s-pvc",
             # pypi的源，如果为空字符串，则使用官方源
             "pypi_mirror": "https://pypi.tuna.tsinghua.edu.cn/simple",
+            # 执行python脚本参数---最好是base64编码的json字符串
+            "params": "xxx",
         },
         #
         is_debug=False,
@@ -86,6 +88,11 @@ class ServerlessTaskManage(object):
         code_path: 代码路径
         project_configs: 所有项目配置
         project_name: 项目名称，在project_configs里面找到对应的key
+        params: 用法
+        param_b64_content = base64.b64encode(json.dumps(self.task_params).encode(
+            'utf8')).decode()  # 将参数编辑为base64，防止出现特殊字符分割参数
+        # 解码
+        # ds = json.loads(base64.b64decode(param_b64_content).decode('utf8'))
         ######################################
         """
         """
@@ -213,7 +220,7 @@ class ServerlessTaskManage(object):
                                     pip install -U pip {pypi_i} && \
                                     pip install arrow {pypi_i} && \
                                     pip install -r requirements.txt {pypi_i} && \
-                                    python main.py
+                                    python main.py --params {self.task_config["params"]}
                             """
                         ],
                         "resources": {

{tfduck-bsd-0.18.9 → tfduck-bsd-0.19.1/tfduck_bsd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tfduck-bsd
-Version: 0.18.9
+Version: 0.19.1
 Summary: A small example package
 Home-page: UNKNOWN
 Author: yuanxiao

tfduck-bsd-0.18.9/tfduck/__init__.py

@@ -1 +0,0 @@
-__version__="0.18.9"