re-common 10.0.18__tar.gz → 10.0.21__tar.gz

{re_common-10.0.18/re_common.egg-info → re_common-10.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.18
+Version: 10.0.21
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,14 +11,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-python
-Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.18 → re_common-10.0.21}/re_common/v2/baselibrary/tools/hdfs_data_processer.py

@@ -109,12 +109,12 @@ class HDFSDataProcessor:
         retry_count = 0
         while retry_count < self.retry_limit:
             try:
-                return await process_func(data) # 成功处理后退出
+                return await process_func(data)  # 成功处理后退出
             except Exception as e:
                 retry_count += 1
                 print(f"处理数据时发生错误: {e}, 正在重试 {retry_count}/{self.retry_limit}, data: {data}")
                 await asyncio.sleep(2**retry_count)
-        print(f"处理数据失败, 达到重试上限, data: {data}")
+        raise Exception(f"处理数据失败, 达到重试上限, data: {data}")
 
     async def process_file(self, hdfs_file_path, process_func, write_dir: str):
         """处理单个 gz 文件"""
@@ -132,7 +132,7 @@ class HDFSDataProcessor:
                     data = json.loads(line)
                     tasks.append(self.process_data(data, process_func))
                 except json.JSONDecodeError as e:
-                    print(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+                    raise Exception(f"解析JSON失败: {e}, 行内容: {line.strip()}")
 
             # await AsyncTaskPool(self.batch_size).run(tasks) # AsyncTaskPool 适用于一次提交所有任务, 限制并发数执行
             results.extend(await asyncio.gather(*tasks))
@@ -236,7 +236,7 @@ class HDFSDataProcessor:
                     data = json.loads(line)
                     batch_data.append(data)
                 except json.JSONDecodeError as e:
-                    print(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+                    raise Exception(f"解析JSON失败: {e}, 行内容: {line.strip()}")
 
             # 处理读取到的批次数据
             if batch_data:

{re_common-10.0.18 → re_common-10.0.21}/re_common/v2/baselibrary/utils/basehdfs.py

@@ -107,6 +107,40 @@ class HDFSUtils(object):
             writer.write(content)
             print(f"文件写入成功: {hdfs_path}")
 
+    def write_file_kwargs(self, hdfs_path, content, **kwargs):
+        """
+        向 HDFS 文件写入内容
+        自定义参数实现更大的灵活性
+        """
+        with self.client.write(hdfs_path, **kwargs) as writer:
+            writer.write(content)
+            print(f"文件写入成功: {hdfs_path}")
+
+    def safe_append_hdfs(self, hdfs_path, content):
+        """
+        更安全的追加写入方式，显式检查文件是否存在
+
+        :param content: 要写入的内容
+        :param hdfs_path: HDFS文件路径
+        """
+        try:
+            # 检查文件是否存在
+            file_exists = self.client.status(hdfs_path, strict=False) is not None
+
+            if not file_exists:
+                print(f"文件 {hdfs_path} 不存在，将创建新文件")
+                # 第一次写入不使用append模式
+                with self.client.write(hdfs_path, encoding='utf-8') as writer:
+                    writer.write(content)
+            else:
+                # 追加模式写入
+                with self.client.write(hdfs_path, encoding='utf-8', append=True) as writer:
+                    writer.write(content)
+
+        except Exception as e:
+            print(f"文件操作失败: {str(e)}")
+            raise
+
     def file_exists(self, hdfs_path):
         """
         检查 HDFS 文件或目录是否存在。

re_common-10.0.21/re_common/v2/baselibrary/utils/basetime.py

@@ -0,0 +1,77 @@
+from datetime import datetime, timezone, timedelta
+from typing import Optional
+
+
+class BaseTime(object):
+    @staticmethod
+    def get_utc_now() -> datetime:
+        """获取当前UTC时间"""
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def to_beijing_time(dt: datetime) -> datetime:
+        """将datetime对象转换为北京时间(UTC+8)"""
+        return dt.astimezone(timezone(timedelta(hours=8)))
+
+    @staticmethod
+    def get_current_beijing_time() -> datetime:
+        """获取当前北京时间"""
+        return BaseTime.to_beijing_time(BaseTime.get_utc_now())
+
+    @staticmethod
+    def get_beijing_time_str(format: str = "%Y-%m-%d %H:%M:%S") -> str:
+        """
+        获取当前北京时间的字符串表示
+        :param format: 时间字符串的格式 格式为'YYYY-MM-DD HH:MM:SS'
+        """
+        beijing_time = BaseTime.get_current_beijing_time()
+        return beijing_time.strftime(format)
+
+    @staticmethod
+    def get_current_month_str() -> str:
+        """获取当前月份的字符串表示，格式为'YYYYMM'"""
+        beijing_time = BaseTime.get_current_beijing_time()
+        return beijing_time.strftime("%Y%m")
+
+    @staticmethod
+    def parse_beijing_time(time_str: str, format: str = "%Y-%m-%d %H:%M:%S") -> datetime:
+        """
+        解析时间字符串为北京时间的datetime对象
+
+        :param time_str: 时间字符串
+        :param format: 时间字符串的格式
+        :return: 带有时区信息的datetime对象(UTC+8)
+        """
+        dt = datetime.strptime(time_str, format)
+        return dt.replace(tzinfo=timezone(timedelta(hours=8)))
+
+    @staticmethod
+    def get_time_difference(time_str: str) -> timedelta:
+        """
+        计算当前时间与给定时间字符串的时间差
+
+        :param time_str: 时间字符串，格式为"YYYY-MM-DD HH:MM:SS"
+        :return: timedelta对象，表示时间差
+        """
+        current_time = BaseTime.get_current_beijing_time()
+        target_time = BaseTime.parse_beijing_time(time_str)
+        difference = current_time - target_time
+        # print(f"时间差: {difference}")
+        # print(f"总秒数: {difference.total_seconds()}秒")
+        # print(f"天数: {difference.days}天")
+        # print(f"小时数: {difference.total_seconds() / 3600:.2f}小时")
+        # print(f"分钟数: {difference.total_seconds() / 60:.2f}分钟")
+
+        return difference
+
+    @staticmethod
+    def is_hour_changed(last_time_str: str) -> bool:
+        """
+        检查最后一次记录时间与当前时间是否跨越了小时边界
+
+        :param last_time_str: 最后一次记录的时间字符串，格式为"YYYY-MM-DD HH:MM:SS"
+        :return: True表示跨越了小时边界，False表示仍在同一小时内
+        """
+        current_time = BaseTime.get_current_beijing_time()
+        last_time = BaseTime.parse_beijing_time(last_time_str)
+        return current_time.hour != last_time.hour

{re_common-10.0.18 → re_common-10.0.21}/re_common/v2/baselibrary/utils/json_cls.py

@@ -1,6 +1,9 @@
 import base64
+import functools
 import json
 
+json_dumps = functools.partial(json.dumps, ensure_ascii=False)
+
 
 class BytesEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -8,4 +11,6 @@ class BytesEncoder(json.JSONEncoder):
             return base64.b64encode(obj).decode('utf-8')
         return super().default(obj)
 
-# json.dumps(x, ensure_ascii=False, cls=BytesEncoder)
+
+def base64_to_bytes(base64_str, encoding="utf-8") -> bytes:
+    return base64.b64decode(base64_str.encode(encoding))

{re_common-10.0.18 → re_common-10.0.21/re_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.18
+Version: 10.0.21
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,14 +11,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-python
-Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.18 → re_common-10.0.21}/re_common.egg-info/SOURCES.txt

@@ -193,6 +193,7 @@ re_common/v2/baselibrary/utils/base_string_similarity.py
 re_common/v2/baselibrary/utils/basedict.py
 re_common/v2/baselibrary/utils/basehdfs.py
 re_common/v2/baselibrary/utils/basepika.py
+re_common/v2/baselibrary/utils/basetime.py
 re_common/v2/baselibrary/utils/db.py
 re_common/v2/baselibrary/utils/json_cls.py
 re_common/v2/baselibrary/utils/mq.py

{re_common-10.0.18 → re_common-10.0.21}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.18",
+    version="10.0.21",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",