nltfile 1.0.28__py3-none-any.whl

nltfile/__init__.py

@@ -0,0 +1,18 @@
+from .compress import tarfile, zipfile
+from .file import ConcurrentFile
+from .utils import file_size, file_tqdm_bar
+from .utils import file_hash, file_sha1, file_sha256, file_sha512, file_md5
+
+__all__ = [
+    "tarfile",
+    "zipfile",
+    "file_tqdm_bar",
+    "file_size",
+    "ConcurrentFile",
+    "file_size",
+    "file_hash",
+    "file_md5",
+    "file_sha1",
+    "file_sha512",
+    "file_sha256",
+]

nltfile/compress/__init__.py

@@ -0,0 +1,6 @@
+from . import tarfile, zipfile
+
+from nltfile.utils import file_tqdm_bar, file_size
+
+
+__all__ = ["file_tqdm_bar", "file_size", "tarfile", "zipfile"]

nltfile/compress/allfile.py

@@ -0,0 +1,12 @@
+from nltfile.compress import tarfile, zipfile
+
+_TAR_EXTENSIONS = (".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", ".txz")
+
+
+def extractall(archive_path: str, path: str = "."):
+    if archive_path.endswith(".zip"):
+        with zipfile.ZipFile(archive_path) as zf:
+            zf.extractall(path=path)
+    elif archive_path.endswith(_TAR_EXTENSIONS):
+        with tarfile.TarFile(archive_path) as tf:
+            tf.extractall(path=path)

nltfile/compress/tarfile.py

@@ -0,0 +1,105 @@
+import io
+import os
+import tarfile
+
+from nltfile.utils import file_tqdm_bar
+from tqdm import tqdm
+
+
+class ProgressFileIO(io.FileIO):
+    def __init__(self, path, progress=None, *args, **kwargs):
+        super(ProgressFileIO, self).__init__(path, *args, **kwargs)
+        self._progress = progress
+
+    def read(self, size=None) -> bytes:
+        self._progress.update(self.tell() - self._progress.n)
+        return io.FileIO.read(self, size)
+
+
+class FileWrapper(object):
+    def __init__(self, fileobj, progress: tqdm, *args, **kwargs):
+        super(FileWrapper, self).__init__(*args, **kwargs)
+        self._fileobj = fileobj
+        self._progress = progress
+
+    def _update(self, current):
+        if self._progress is not None:
+            if current > self._progress.total:
+                self._progress.total = current
+            self._progress.update(current - self._progress.n)
+
+    def read(self, size=-1):
+        self._update(self._fileobj.tell())
+        return self._fileobj.read(size)
+
+    def readline(self, size=-1):
+        self._update(self._fileobj.tell())
+        return self._fileobj.readline(size)
+
+    def __getattr__(self, name):
+        return getattr(self._fileobj, name)
+
+    def __del__(self):
+        if self._fileobj is not None:
+            self._update(self._fileobj.tell())
+
+
+class TarFile(tarfile.TarFile):
+    def __init__(self, name=None, mode="r", fileobj=None, **kwargs):
+        self._progress = None
+        if "r" in mode:
+            self._progress = file_tqdm_bar(name, prefix="解压")
+            if fileobj is not None:
+                fileobj = FileWrapper(fileobj, progress=self._progress)
+            else:
+                fileobj = ProgressFileIO(name, progress=self._progress)
+        super(TarFile, self).__init__(name=name, mode=mode, fileobj=fileobj, **kwargs)
+        if "r" in mode and self._progress is not None:
+            self._progress.total = self.tar_size()
+
+    def _check_progress_available(self) -> bool:
+        if self._progress is None:
+            return False
+        return self._progress.n < self._progress.total
+
+    def addfile(self, tarinfo, fileobj=None):
+        if fileobj is not None:
+            fileobj = FileWrapper(fileobj, progress=self._progress)
+        return super(TarFile, self).addfile(tarinfo, fileobj)
+
+    def add(
+        self,
+        name,
+        arcname=None,
+        recursive=True,
+        filter=None,
+        progress=None,
+        *args,
+        **kwargs,
+    ):
+        self._progress = progress or self._progress or file_tqdm_bar(name)
+        return super(TarFile, self).add(
+            name=name, arcname=arcname, recursive=recursive, filter=filter
+        )
+
+    def tar_size(self):
+        return sum(m.size for m in self.getmembers())
+
+
+tar_open: TarFile = TarFile.open
+
+
+def file_entar(src_path, dst_path=None):
+    if dst_path is None:
+        dst_path = src_path + ".tar"
+    with tar_open(dst_path, "w:xz") as fw:
+        fw.add(src_path, arcname=os.path.basename(src_path))
+    return dst_path
+
+
+def file_detar(src_path, dst_path=None):
+    if dst_path is None:
+        dst_path = os.path.dirname(src_path)
+    with tar_open(src_path, "r:xz") as fr:
+        fr.extractall(path=dst_path)
+    return os.path.join(dst_path, os.listdir(dst_path)[0])

nltfile/compress/utils.py

@@ -0,0 +1,31 @@
+import os
+
+from tqdm import tqdm
+
+
+def get_size(path, recursive=False) -> int:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Path not found: {path}")
+    if os.path.isfile(path):
+        return os.path.getsize(path)
+    size = 0
+    for filename in os.listdir(path):
+        path2 = os.path.join(path, filename)
+        if os.path.isfile(path2):
+            size += os.path.getsize(path2)
+        elif recursive:
+            size += get_size(path2, recursive=recursive)
+    return size
+
+
+def file_tqdm_bar(path, prefix="", total=None, ncols=120, recursive=False) -> tqdm:
+    prefix = f"{prefix}: " if prefix is not None and len(prefix) > 0 else ""
+    return tqdm(
+        total=total or get_size(path, recursive=recursive),
+        desc=f"{prefix}{os.path.basename(path)}"[:20],
+        ncols=ncols,
+        ascii=True,
+        unit="B",
+        unit_scale=True,
+        unit_divisor=1024,
+    )

nltfile/compress/zipfile.py

@@ -0,0 +1,6 @@
+import zipfile
+
+
+class ZipFile(zipfile.ZipFile):
+    def __init__(self, file, mode="r", *args, **kwargs):
+        super(ZipFile, self).__init__(file=file, mode=mode, *args, **kwargs)

nltfile/file/__init__.py

@@ -0,0 +1,4 @@
+from .concurrent import ConcurrentFile
+from .core import copy
+
+__all__ = ["copy", "ConcurrentFile"]

nltfile/file/concurrent.py

@@ -0,0 +1,113 @@
+import os.path
+import pickle
+import time
+import traceback
+from queue import Empty, Queue
+from threading import Thread
+
+from funlog import getLogger
+
+logger = getLogger("nltfile")
+
+
+class ConcurrentWriteFile:
+    def __init__(self, filepath, mode="w", capacity=200, timeout=3):
+        self.filepath = filepath
+        self.mode = mode
+        self.timeout = timeout
+        self._write_queue = Queue(capacity)
+        self._close = False
+        self._writen_data = []
+        self._writen_size = 0
+        self._flush_size = 0
+        thread = Thread(target=self._write)
+        thread.start()
+
+    def write(self, chunk, offset=None):
+        self._write_queue.put((offset, chunk))
+        size = len(chunk)
+        self.curser_add(offset, size)
+        return size
+
+    def _write(self):
+        with open(self.filepath, self.mode) as fw:
+            while True:
+                try:
+                    try:
+                        offset, chunk = self._write_queue.get(timeout=self.timeout)
+                    except Empty:
+                        continue
+                    if chunk is None:
+                        self._write_queue.task_done()
+                        continue
+                    if offset is not None:
+                        fw.seek(offset)
+                    self._writen_size += fw.write(chunk)
+                    self._write_queue.task_done()
+                    if self._writen_size > self._flush_size + 50 * 1024 * 1024:
+                        fw.flush()
+                        self._flush_size = self._writen_size
+                        self.curser_merge()
+                except Exception as e:
+                    logger.error(f"write error: {e}:{traceback.format_exc()}")
+                if self._close:
+                    break
+            fw.flush()
+            self._flush_size = self._writen_size
+            self.curser_merge()
+
+    def close(self):
+        self._close = True
+
+    def wait_for_all_done(self):
+        self._write_queue.join()
+
+    def empty(self):
+        return self._write_queue.empty()
+
+    def curser_add(self, offset, size):
+        if offset is None:
+            return
+        for record in self._writen_data:
+            if record[0] <= offset <= record[1]:
+                if record[1] < offset + size:
+                    record[1] = offset + size
+                return
+        self._writen_data.append([offset, offset + size])
+        return
+
+    def _process_filepath(self):
+        return f"{self.filepath}.process"
+
+    def curser_merge(self):
+        self._writen_data.sort(key=lambda x: x[0])
+        merged = []
+        for interval in self._writen_data:
+            if not merged or interval[0] > merged[-1][1]:
+                merged.append(interval)
+            else:
+                merged[-1][1] = max(merged[-1][1], interval[1])
+        self._writen_data = merged
+        with open(self._process_filepath(), "wb") as fw:
+            pickle.dump(self._writen_data, fw)
+
+    def __enter__(self):
+        self._handle = self
+        if os.path.exists(self._process_filepath()):
+            with open(self._process_filepath(), "rb") as fr:
+                self._writen_data = pickle.load(fr)
+        return self._handle
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        while not self.empty():
+            time.sleep(1)
+        self.wait_for_all_done()
+        self.close()
+        if os.path.exists(self._process_filepath()):
+            os.remove(self._process_filepath())
+        return False
+
+
+class ConcurrentFile(ConcurrentWriteFile):
+    def __init__(self, *args, **kwargs):
+        super(ConcurrentFile, self).__init__(*args, **kwargs)

nltfile/file/core.py

@@ -0,0 +1,5 @@
+import shutil
+
+
+def copy(src, dst, follow_symlinks=True):
+    shutil.copy(src, dst, follow_symlinks=follow_symlinks)

nltfile/funos.py

@@ -0,0 +1,11 @@
+import os
+import shutil
+
+
+def makedirs(path):
+    os.makedirs(path, exist_ok=True)
+
+
+def delete(path):
+    if os.path.exists(path):
+        shutil.rmtree(path, ignore_errors=True)

nltfile/pickle/__init__.py

@@ -0,0 +1,3 @@
+from .core import dump, dumps, load, loads
+
+__all__ = ["load", "dump", "loads", "dumps"]

nltfile/pickle/core.py

@@ -0,0 +1,19 @@
+import pickle
+
+
+def dump(data, path):
+    with open(path, "wb") as fw:
+        pickle.dump(data, fw)
+
+
+def load(path):
+    with open(path, "rb") as fr:
+        return pickle.load(fr)
+
+
+def dumps(obj):
+    return pickle.dumps(obj)
+
+
+def loads(obj):
+    return pickle.loads(obj)

nltfile/utils/__init__.py

@@ -0,0 +1,13 @@
+from .tqdm_bar import file_tqdm_bar
+from .size import file_size
+from .hash import file_hash, file_sha1, file_sha256, file_sha512, file_md5
+
+__all__ = [
+    "file_tqdm_bar",
+    "file_size",
+    "file_hash",
+    "file_md5",
+    "file_sha1",
+    "file_sha512",
+    "file_sha256",
+]

nltfile/utils/hash.py

@@ -0,0 +1,33 @@
+import hashlib
+
+
+def file_hash(file_path, algorithm):
+    """
+    计算文件的哈希值
+    :param file_path: 文件路径
+    :param algorithm: 哈希算法（'md5', 'sha1', 'sha256', 'sha512'）
+    :return: 文件的哈希值
+    """
+    hash_func = hashlib.new(algorithm)
+    with open(file_path, "rb") as f:
+        chunk = f.read(8192)
+        while chunk:
+            hash_func.update(chunk)
+            chunk = f.read(8192)
+    return hash_func.hexdigest()
+
+
+def file_md5(filepath):
+    return file_hash(filepath, "md5")
+
+
+def file_sha1(filepath):
+    return file_hash(filepath, "sha1")
+
+
+def file_sha256(filepath):
+    return file_hash(filepath, "sha256")
+
+
+def file_sha512(filepath):
+    return file_hash(filepath, "sha512")

nltfile/utils/size.py

@@ -0,0 +1,16 @@
+import os
+
+
+def file_size(path, recursive=False) -> int:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Path not found: {path}")
+    if os.path.isfile(path):
+        return os.path.getsize(path)
+    size = 0
+    for filename in os.listdir(path):
+        path2 = os.path.join(path, filename)
+        if os.path.isfile(path2):
+            size += os.path.getsize(path2)
+        elif recursive:
+            size += file_size(path2, recursive=recursive)
+    return size

nltfile/utils/tqdm_bar.py

@@ -0,0 +1,17 @@
+import os
+
+from tqdm import tqdm
+from .size import file_size
+
+
+def file_tqdm_bar(path, prefix="", total=None, ncols=120, recursive=False) -> tqdm:
+    prefix = f"{prefix}: " if prefix is not None and len(prefix) > 0 else ""
+    return tqdm(
+        total=total or file_size(path, recursive=recursive),
+        desc=f"{prefix}{os.path.basename(path)}"[:20],
+        ncols=ncols,
+        ascii=True,
+        unit="B",
+        unit_scale=True,
+        unit_divisor=1024,
+    )

nltfile-1.0.28.dist-info/METADATA

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: nltfile
+Version: 1.0.28
+Summary: nltfile
+Author-email: 牛哥 <niuliangtao@qq.com>, farfarfun <farfarfun@qq.com>
+Maintainer-email: 牛哥 <niuliangtao@qq.com>, farfarfun <farfarfun@qq.com>
+Project-URL: Organization, https://github.com/farfarfun
+Project-URL: Repository, https://github.com/farfarfun/nltfile
+Project-URL: Releases, https://github.com/farfarfun/nltfile/releases
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: funlog-tau>=1.0.0
+Requires-Dist: tqdm>=4.66.5
+
+# nltfile
+
+一个实用的 Python 文件操作工具库，提供带进度条的压缩/解压、并发写入、pickle 序列化等功能的增强封装。
+
+## 特性
+
+- 带进度条的 tar/zip 压缩与解压，用法与标准库一致
+- 线程安全的并发文件写入，支持断点续写
+- pickle 序列化/反序列化的便捷封装
+- 常用文件系统操作（创建目录、删除、复制）
+
+## 安装
+
+```bash
+pip install nltfile
+```
+
+## 依赖
+
+- Python >= 3.7
+- [funutil](https://pypi.org/project/funutil/) >= 1.0.15
+- [tqdm](https://pypi.org/project/tqdm/) >= 4.66.5
+
+## 使用
+
+### tar 压缩/解压（带进度条）
+
+用法与标准库 `tarfile` 一致，自动显示进度条：
+
+```python
+from nltfile import tarfile
+
+# 压缩
+with tarfile.open("results.tar", "w|xz") as tar:
+    tar.add("a.txt")
+
+# 解压
+with tarfile.open("results.tar", "r|xz") as tar:
+    tar.extractall("local")
+```
+
+也可以使用快捷函数：
+
+```python
+from nltfile.compress.tarfile import file_entar, file_detar
+
+# 一键压缩
+file_entar("mydir", "mydir.tar")
+
+# 一键解压
+file_detar("mydir.tar", "output")
+```
+
+### zip 解压
+
+```python
+from nltfile import zipfile
+
+with zipfile.ZipFile("archive.zip") as zf:
+    zf.extractall("output")
+```
+
+### 通用解压
+
+根据文件后缀自动选择解压方式，支持 `.zip`、`.tar`、`.tar.gz`、`.tgz`、`.tar.bz2`、`.tar.xz`、`.txz`：
+
+```python
+from nltfile.compress.allfile import extractall
+
+extractall("archive.tar.gz", "output")
+```
+
+### 并发文件写入
+
+线程安全的异步写入，适用于多线程场景。支持指定偏移量写入，并在写入过程中自动记录进度，异常中断后可断点续写：
+
+```python
+from nltfile import ConcurrentFile
+
+with ConcurrentFile("output.txt", mode="w") as fw:
+    fw.write("hello, nltfile.")
+    fw.write("another line.")
+```
+
+支持指定偏移量的随机写入：
+
+```python
+with ConcurrentFile("output.bin", mode="wb") as fw:
+    fw.write(b"chunk1", offset=0)
+    fw.write(b"chunk2", offset=1024)
+```
+
+### pickle 序列化
+
+便捷的 pickle 读写封装：
+
+```python
+from nltfile.pickle import dump, load, dumps, loads
+
+# 写入文件
+dump({"key": "value"}, "data.pkl")
+
+# 从文件读取
+data = load("data.pkl")
+
+# 序列化为 bytes
+raw = dumps({"key": "value"})
+
+# 从 bytes 反序列化
+obj = loads(raw)
+```
+
+### 文件系统工具
+
+```python
+from nltfile.funos import makedirs, delete
+
+# 递归创建目录（已存在不报错）
+makedirs("path/to/dir")
+
+# 递归删除目录
+delete("path/to/dir")
+```
+
+```python
+from nltfile.file import copy
+
+# 复制文件
+copy("src.txt", "dst.txt")
+```
+
+### 获取文件/目录大小
+
+```python
+from nltfile import get_size
+
+# 获取文件大小（字节）
+size = get_size("large_file.bin")
+
+# 递归获取目录大小
+size = get_size("mydir", recursive=True)
+```
+
+## 项目结构
+
+```
+src/nltfile/
+├── __init__.py            # 顶层导出
+├── funos.py               # 文件系统工具（makedirs, delete）
+├── file/
+│   ├── core.py            # 文件复制
+│   └── concurrent.py      # 并发文件写入
+├── compress/
+│   ├── utils.py           # 进度条、文件大小计算
+│   ├── tarfile.py         # 带进度条的 tar 操作
+│   ├── zipfile.py         # zip 操作封装
+│   └── allfile.py         # 通用解压
+└── pickle/
+    └── core.py            # pickle 序列化封装
+```
+
+## 许可证
+
+[Apache License 2.0](LICENSE)

nltfile-1.0.28.dist-info/RECORD

@@ -0,0 +1,20 @@
+nltfile/__init__.py,sha256=kAHph8CeliQQGUqHepjLhmlPJ3jVd7GS49ZRBcaBX8E,402
+nltfile/funos.py,sha256=qQK2qJuMEwbUR1D2i6saY6uNQ-_To6dnuITTfMox5Lg,180
+nltfile/compress/__init__.py,sha256=-vQPrErKZ5OWqmCepQ0a0mumkxiDL7jUMoMvQbpZxGo,148
+nltfile/compress/allfile.py,sha256=TbH0Wp0U9RNIgtq_7Wb5ver2Yfrd3hlDz8U0WVvCzBc,439
+nltfile/compress/tarfile.py,sha256=llS5Upid4s5hU0APCVVhh8iRTQ3NAETvciPHFwZ1_r8,3267
+nltfile/compress/utils.py,sha256=D3O4-Ek90WKE8wBhVnQXdnW0DE7snzeAqYh3Q7eIwTw,919
+nltfile/compress/zipfile.py,sha256=DZexJ-tv05FgM2tu-pjM2knevz5cK04iFsh8HMrjIcI,183
+nltfile/file/__init__.py,sha256=j1z19wQkVnVvQsaP9OELFx4hquLFERWdFR3OF1rwA7g,100
+nltfile/file/concurrent.py,sha256=UOiGp4PVdL9C3w3vf-vY4AEcuqIQZQ8cyvl4KijePLo,3642
+nltfile/file/core.py,sha256=mTyS1VF5vTH22gjdea2WJ3qFu9Smc1MU8oNB8nkn6Yw,117
+nltfile/pickle/__init__.py,sha256=s_T_2MuvRjlIsZGlRDmkSTjimljB32K588fPWrBbIgw,89
+nltfile/pickle/core.py,sha256=PWmIfrk__OWaNn_7k1G65bb2oXPZMhMUZ0hU0vr3c34,277
+nltfile/utils/__init__.py,sha256=JWIXl95pOFwRyH_T3hqy8xevdswH148wEhHNLwipJ0Q,280
+nltfile/utils/hash.py,sha256=soV1QgOLth6oJ5R_YLH3JyLKKdDr8arnoX8KK31jU_4,738
+nltfile/utils/size.py,sha256=ABgEbXYUhrxOcU1IagoFPvkoo41oPgw1dRn2w1ULVvY,486
+nltfile/utils/tqdm_bar.py,sha256=QsicPBLB13y34e3kHyGME6vIWe_tTXJdtECVDLDlHpc,474
+nltfile-1.0.28.dist-info/METADATA,sha256=u9vuPEv5nSyWbCzV6tjbpNEa55A2u5PSRVyYqYzmJv4,4023
+nltfile-1.0.28.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+nltfile-1.0.28.dist-info/top_level.txt,sha256=HRKWMQx8AAhq80WKdtJAqks4xvFtiqWpHKa7qH0eT9k,8
+nltfile-1.0.28.dist-info/RECORD,,

nltfile-1.0.28.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nltfile-1.0.28.dist-info/top_level.txt

@@ -0,0 +1 @@
+nltfile