re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl

re_common/v2/baselibrary/tools/data_processer/data_reader.py

@@ -1,187 +1,187 @@
-import gzip
-import io
-import json
-from io import BytesIO
-from pathlib import Path
-from typing import List, Generator
-
-import pandas as pd
-from hdfs import InsecureClient
-
-from re_common.v2.baselibrary.tools.data_processer.base import BaseFileReader
-
-
-class HDFSFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
-        self.client = InsecureClient(hdfs_url, user=hdfs_user)
-
-    def list_files(self, path: str) -> List[str]:
-        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0] != '_SUCCESS']
-
-    def count_lines(self, file_path: str) -> int:
-        with self.client.read(file_path) as f:
-            return sum(1 for _ in f)
-
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        # 批量读取后 处理 缺点 连接可能会断
-        with self.client.read(file_path) as f:
-            while True:
-                batch = []
-                for _ in range(self.batch_size):
-                    try:
-                        line = next(f)
-                        line = line.decode('utf-8')
-                        if line.strip():
-                            batch.append(line.strip())
-                    except StopIteration:
-                        break
-                if not batch:
-                    break
-                yield batch
-
-    def read_all(self, file_path: str) -> List[List[str]]:
-        # 一次读取返回所有后批量处理缺点 内存占用
-        with self.client.read(file_path) as f:
-            lines = [line.decode('utf-8').strip() for line in f if line.decode('utf-8').strip()]
-            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
-
-
-class HDFSGZFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
-        self.hdfs_url = hdfs_url
-        self.hdfs_user = hdfs_user
-        self.client = None
-
-    def _init_client(self):
-        if self.client is None:
-            self.client = InsecureClient(self.hdfs_url, user=self.hdfs_user)
-        return self
-
-    def list_files(self, path: str) -> List[str]:
-        self._init_client()
-        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".gz")]
-
-    def count_lines(self, file_path: str) -> int:
-        self._init_client()
-        with self.client.read(file_path) as f:
-            with gzip.GzipFile(fileobj=f) as gz:
-                return sum(1 for _ in gz)
-
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        self._init_client()
-        # 批量读取后 处理 缺点 连接可能会断
-        with self.client.read(file_path) as f:
-            with gzip.GzipFile(fileobj=f) as gz:
-                while True:
-                    batch = []
-                    for _ in range(self.batch_size):
-                        try:
-                            line = next(gz)
-                            if line.strip():
-                                batch.append(line.decode("utf-8"))
-                        except StopIteration:
-                            break
-                    if not batch:
-                        break
-                    yield batch
-
-    def read_all(self, file_path: str) -> List[List[str]]:
-        self._init_client()
-        # 一次读取返回所有后批量处理缺点 内存占用
-        with self.client.read(file_path) as reader:
-            compressed_data = reader.read()
-            with gzip.GzipFile(fileobj=BytesIO(compressed_data)) as gz_file:
-                content = gz_file.read().decode("utf-8")
-                lines = [i for i in content.split("\n") if i.strip()]
-                return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
-
-
-class HDFSParquetFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
-        self.client = InsecureClient(hdfs_url, user=hdfs_user)
-
-    def list_files(self, path: str) -> List[str]:
-        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".parquet")]
-
-    def count_lines(self, file_path: str) -> int:
-        with self.client.read(file_path) as f:
-            data = f.read()
-            df = pd.read_parquet(io.BytesIO(data))
-        count = len(df)
-        return count
-
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        # 批量读取后 处理 缺点 连接可能会断
-        with self.client.read(file_path) as f:
-            data = f.read()
-            df = pd.read_parquet(io.BytesIO(data))
-            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
-            for i in range(0, len(records), self.batch_size):
-                yield records[i: i + self.batch_size]
-
-    def read_all(self, file_path: str) -> List[List[str]]:
-        # 一次读取返回所有后批量处理缺点 内存占用
-        with self.client.read(file_path) as f:
-            data = f.read()
-            df = pd.read_parquet(io.BytesIO(data))
-            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
-            return [records[i: i + self.batch_size] for i in range(0, len(records), self.batch_size)]
-
-
-class LocalGZFileReader(BaseFileReader):
-    def list_files(self, path: str) -> List[str]:
-        return [str(p) for p in Path(path).rglob("*.gz")]
-
-    def count_lines(self, file_path: str) -> int:
-        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
-            return sum(1 for _ in f)
-
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
-            while True:
-                batch = []
-                for _ in range(self.batch_size):
-                    line = f.readline()
-                    if not line:
-                        break
-                    if line.strip():
-                        batch.append(line.strip())
-                if not batch:
-                    break
-                yield batch
-
-    def read_all(self, file_path: str) -> List[List[str]]:
-        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
-            lines = [line.strip() for line in f if line.strip()]
-            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
-
-
-class LocalFileReader(BaseFileReader):
-    def list_files(self, path: str) -> List[str]:
-        return [str(p) for p in Path(path).rglob("*") if p.is_file()]
-
-    def count_lines(self, file_path: str) -> int:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            return sum(1 for _ in f)
-
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            while True:
-                batch = []
-                for _ in range(self.batch_size):
-                    line = f.readline()
-                    if not line:
-                        break
-                    if line.strip():
-                        batch.append(line.strip())
-                if not batch:
-                    break
-                yield batch
-
-    def read_all(self, file_path: str) -> List[List[str]]:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            lines = [line.strip() for line in f if line.strip()]
-            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+import gzip
+import io
+import json
+from io import BytesIO
+from pathlib import Path
+from typing import List, Generator
+
+import pandas as pd
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.tools.data_processer.base import BaseFileReader
+
+
+class HDFSFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def list_files(self, path: str) -> List[str]:
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0] != '_SUCCESS']
+
+    def count_lines(self, file_path: str) -> int:
+        with self.client.read(file_path) as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    try:
+                        line = next(f)
+                        line = line.decode('utf-8')
+                        if line.strip():
+                            batch.append(line.strip())
+                    except StopIteration:
+                        break
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as f:
+            lines = [line.decode('utf-8').strip() for line in f if line.decode('utf-8').strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class HDFSGZFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.hdfs_url = hdfs_url
+        self.hdfs_user = hdfs_user
+        self.client = None
+
+    def _init_client(self):
+        if self.client is None:
+            self.client = InsecureClient(self.hdfs_url, user=self.hdfs_user)
+        return self
+
+    def list_files(self, path: str) -> List[str]:
+        self._init_client()
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".gz")]
+
+    def count_lines(self, file_path: str) -> int:
+        self._init_client()
+        with self.client.read(file_path) as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                return sum(1 for _ in gz)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        self._init_client()
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                while True:
+                    batch = []
+                    for _ in range(self.batch_size):
+                        try:
+                            line = next(gz)
+                            if line.strip():
+                                batch.append(line.decode("utf-8"))
+                        except StopIteration:
+                            break
+                    if not batch:
+                        break
+                    yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        self._init_client()
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as reader:
+            compressed_data = reader.read()
+            with gzip.GzipFile(fileobj=BytesIO(compressed_data)) as gz_file:
+                content = gz_file.read().decode("utf-8")
+                lines = [i for i in content.split("\n") if i.strip()]
+                return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class HDFSParquetFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def list_files(self, path: str) -> List[str]:
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".parquet")]
+
+    def count_lines(self, file_path: str) -> int:
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+        count = len(df)
+        return count
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
+            for i in range(0, len(records), self.batch_size):
+                yield records[i: i + self.batch_size]
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
+            return [records[i: i + self.batch_size] for i in range(0, len(records), self.batch_size)]
+
+
+class LocalGZFileReader(BaseFileReader):
+    def list_files(self, path: str) -> List[str]:
+        return [str(p) for p in Path(path).rglob("*.gz")]
+
+    def count_lines(self, file_path: str) -> int:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    line = f.readline()
+                    if not line:
+                        break
+                    if line.strip():
+                        batch.append(line.strip())
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            lines = [line.strip() for line in f if line.strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class LocalFileReader(BaseFileReader):
+    def list_files(self, path: str) -> List[str]:
+        return [str(p) for p in Path(path).rglob("*") if p.is_file()]
+
+    def count_lines(self, file_path: str) -> int:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    line = f.readline()
+                    if not line:
+                        break
+                    if line.strip():
+                        batch.append(line.strip())
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = [line.strip() for line in f if line.strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]

re_common/v2/baselibrary/tools/data_processer/data_writer.py

@@ -1,38 +1,38 @@
-import gzip
-from io import BytesIO
-from typing import List
-
-from hdfs import InsecureClient
-
-from re_common.v2.baselibrary.tools.data_processer.base import BaseFileWriter
-
-
-class HDFSFileWriter(BaseFileWriter):
-    def __init__(self, file_path: str, hdfs_url: str, hdfs_user: str, *args, **kwargs):
-        super().__init__(file_path, *args, **kwargs)
-        self.client = InsecureClient(hdfs_url, user=hdfs_user)
-
-    def write_lines(self, lines: List[str], file_path: str = None):
-        if file_path is None:
-            file_path = self.file_path
-        data = "\n".join(lines).encode(self.encoding)
-        if self.compress:
-            buf = BytesIO()
-            with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
-                gz.write(data)
-            buf.seek(0)
-            self.client.write(file_path, data=buf, overwrite=self.overwrite)
-        else:
-            self.client.write(file_path, data=data, overwrite=self.overwrite)
-
-
-class LocalFileWriter(BaseFileWriter):
-    def write_lines(self, lines: List[str], file_path: str, compress: bool = True, encoding="utf-8"):
-        if compress:
-            with gzip.open(file_path, 'wt', encoding=encoding) as f:
-                for line in lines:
-                    f.write(f"{line}\n")
-        else:
-            with open(file_path, 'w', encoding=encoding) as f:
-                for line in lines:
-                    f.write(f"{line}\n")
+import gzip
+from io import BytesIO
+from typing import List
+
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.tools.data_processer.base import BaseFileWriter
+
+
+class HDFSFileWriter(BaseFileWriter):
+    def __init__(self, file_path: str, hdfs_url: str, hdfs_user: str, *args, **kwargs):
+        super().__init__(file_path, *args, **kwargs)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def write_lines(self, lines: List[str], file_path: str = None):
+        if file_path is None:
+            file_path = self.file_path
+        data = "\n".join(lines).encode(self.encoding)
+        if self.compress:
+            buf = BytesIO()
+            with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
+                gz.write(data)
+            buf.seek(0)
+            self.client.write(file_path, data=buf, overwrite=self.overwrite)
+        else:
+            self.client.write(file_path, data=data, overwrite=self.overwrite)
+
+
+class LocalFileWriter(BaseFileWriter):
+    def write_lines(self, lines: List[str], file_path: str, compress: bool = True, encoding="utf-8"):
+        if compress:
+            with gzip.open(file_path, 'wt', encoding=encoding) as f:
+                for line in lines:
+                    f.write(f"{line}\n")
+        else:
+            with open(file_path, 'w', encoding=encoding) as f:
+                for line in lines:
+                    f.write(f"{line}\n")

re_common/v2/baselibrary/tools/dict_tools.py

@@ -1,44 +1,44 @@
-class DotDict(dict):
-    """
-    让字典成为对象 既可以用字典方式访问 也可以用点访问key
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # 递归地将嵌套字典转换为 DotDict
-        for key, value in self.items():
-            if isinstance(value, dict):
-                self[key] = DotDict(value)
-
-    def __getattr__(self, key):
-        try:
-            value = self[key]
-            if isinstance(value, dict):  # 如果值是字典，继续转换为 DotDict
-                return DotDict(value)
-            return value
-        except KeyError:
-            raise AttributeError(f"'DotDict' object has no attribute '{key}'")
-
-    def __setattr__(self, key, value):
-        if isinstance(value, dict):  # 如果值是字典，转换为 DotDict
-            value = DotDict(value)
-        self[key] = value
-
-    def to_dict(self):
-        """
-        将 DotDict 实例转换为普通字典
-        """
-        result = {}
-        for key, value in self.items():
-            if isinstance(value, DotDict):
-                result[key] = value.to_dict()
-            else:
-                result[key] = value
-        return result
-
-
-def none_to_empty_str(d):
-    for k, v in d.items():
-        if v is None:
-            d[k] = ""
-    return d
+class DotDict(dict):
+    """
+    让字典成为对象 既可以用字典方式访问 也可以用点访问key
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 递归地将嵌套字典转换为 DotDict
+        for key, value in self.items():
+            if isinstance(value, dict):
+                self[key] = DotDict(value)
+
+    def __getattr__(self, key):
+        try:
+            value = self[key]
+            if isinstance(value, dict):  # 如果值是字典，继续转换为 DotDict
+                return DotDict(value)
+            return value
+        except KeyError:
+            raise AttributeError(f"'DotDict' object has no attribute '{key}'")
+
+    def __setattr__(self, key, value):
+        if isinstance(value, dict):  # 如果值是字典，转换为 DotDict
+            value = DotDict(value)
+        self[key] = value
+
+    def to_dict(self):
+        """
+        将 DotDict 实例转换为普通字典
+        """
+        result = {}
+        for key, value in self.items():
+            if isinstance(value, DotDict):
+                result[key] = value.to_dict()
+            else:
+                result[key] = value
+        return result
+
+
+def none_to_empty_str(d):
+    for k, v in d.items():
+        if v is None:
+            d[k] = ""
+    return d