pewutils 0.0.1__tar.gz

pewutils-0.0.1/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: pewutils
+Version: 0.0.1
+Summary: Lightweight utils for windmill scripts (migrated from peutils).
+Author-email: rxu <rxu@appen.com>
+License: MIT
+Project-URL: Homepage, https://gitee.com/yunsansheng/pewutils
+Keywords: pewutils,windmill,pe
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: oss2>=2.15
+Requires-Dist: requests>=2.25
+
+# pewutils
+
+PE Windmill Utils —— 为 Windmill 脚本平台提供的轻量级 Python 工具包。
+
+由 `peutils` 迁移重构而来，适配 windmill 脚本平台。
+
+## 已迁移模块
+
+| 模块 | 说明 | 依赖 |
+|------|------|------|
+| `url_util` | URL 处理工具（解析、存储 URL、签名 URL 清理、查询参数等） | 仅标准库 |
+| `oss_util` | 阿里云 OSS 工具（STS 鉴权、列举、复制、搜索） | oss2, requests |
+| `wmill_util` | Windmill 平台工具（zip 打包上传 S3） | wmill（windmill runtime 内置） |
+
+## 安装
+
+```bash
+# 核心包（含 oss2/requests 依赖）
+pip install pewutils
+
+# 本地开发（editable）
+git clone <repo-url> pewutils
+cd pewutils
+pip install -e .
+```
+
+## 使用示例
+
+### URL 工具
+
+```python
+from pewutils.url_util import (
+    get_clean_url,
+    parse_storage_url,
+    parse_oss_full_path,
+    get_relative_path,
+)
+
+# 去除 OSS 签名 URL 的查询参数
+clean = get_clean_url("https://bucket.aliyuncs.com/path/file.pcd?Expires=123&Signature=xxx")
+# -> "https://bucket.aliyuncs.com/path/file.pcd"
+
+# 解析存储 URL（宽松）
+bucket, path = parse_storage_url("oss://my-bucket/folder/file.txt")
+# -> ("my-bucket", "folder/file.txt")
+
+# 解析 OSS 目录路径（强约束，要求以 / 结尾）
+bucket, folder = parse_oss_full_path("oss://my-bucket/folder/")
+```
+
+### OSS 工具
+
+```python
+from pewutils.oss_util import OSSClientFactory
+
+# 创建带自动刷新的 OSS client（auth_str 12h 过期，11h 自动刷新）
+client = OSSClientFactory.create_client(bucket_name="my-bucket")
+
+# 读取 OSS 对象为字节流
+data = client.read_bytes("path/to/file.pcd")
+
+# 列举目录下所有文件
+files = client.list_bucket_files_deep("path/to/folder/")
+
+# BFS 查找最浅的目标文件
+path = client.find_shallowest_target("root/", "target.json", "file")
+```
+
+### Windmill 平台工具
+
+```python
+from pewutils.wmill_util import zip_to_s3
+
+# with 块内生成文件，退出时自动打包 zip 上传到 Windmill S3
+with zip_to_s3("output/zip", suffix=".json") as out:
+    Path("a.json").write_text('{"k": 1}')
+    Path("b.json").write_text('{"k": 2}')
+# out.result -> S3Object(s3="output/zip/20260729/uuid.zip")
+# out.files  -> [Path("a.json"), Path("b.json")]
+```
+
+## 目录结构
+
+```
+pewutils/
+├── README.md
+├── pyproject.toml              # 打包配置（PEP 517/518，requires-python>=3.10）
+├── .gitignore
+└── pewutils/
+    ├── __init__.py             # 默认导出 url_util
+    ├── url_util.py             # URL 处理（纯标准库）
+    ├── oss_util.py             # OSS 工具（oss2, requests）
+    └── wmill_util.py           # Windmill 平台工具（wmill lazy import）
+```
+
+## 构建
+
+```bash
+pip install build
+python -m build
+# 产物：dist/pewutils-<version>-py3-none-any.whl
+```

pewutils-0.0.1/README.md

@@ -0,0 +1,102 @@
+# pewutils
+
+PE Windmill Utils —— 为 Windmill 脚本平台提供的轻量级 Python 工具包。
+
+由 `peutils` 迁移重构而来，适配 windmill 脚本平台。
+
+## 已迁移模块
+
+| 模块 | 说明 | 依赖 |
+|------|------|------|
+| `url_util` | URL 处理工具（解析、存储 URL、签名 URL 清理、查询参数等） | 仅标准库 |
+| `oss_util` | 阿里云 OSS 工具（STS 鉴权、列举、复制、搜索） | oss2, requests |
+| `wmill_util` | Windmill 平台工具（zip 打包上传 S3） | wmill（windmill runtime 内置） |
+
+## 安装
+
+```bash
+# 核心包（含 oss2/requests 依赖）
+pip install pewutils
+
+# 本地开发（editable）
+git clone <repo-url> pewutils
+cd pewutils
+pip install -e .
+```
+
+## 使用示例
+
+### URL 工具
+
+```python
+from pewutils.url_util import (
+    get_clean_url,
+    parse_storage_url,
+    parse_oss_full_path,
+    get_relative_path,
+)
+
+# 去除 OSS 签名 URL 的查询参数
+clean = get_clean_url("https://bucket.aliyuncs.com/path/file.pcd?Expires=123&Signature=xxx")
+# -> "https://bucket.aliyuncs.com/path/file.pcd"
+
+# 解析存储 URL（宽松）
+bucket, path = parse_storage_url("oss://my-bucket/folder/file.txt")
+# -> ("my-bucket", "folder/file.txt")
+
+# 解析 OSS 目录路径（强约束，要求以 / 结尾）
+bucket, folder = parse_oss_full_path("oss://my-bucket/folder/")
+```
+
+### OSS 工具
+
+```python
+from pewutils.oss_util import OSSClientFactory
+
+# 创建带自动刷新的 OSS client（auth_str 12h 过期，11h 自动刷新）
+client = OSSClientFactory.create_client(bucket_name="my-bucket")
+
+# 读取 OSS 对象为字节流
+data = client.read_bytes("path/to/file.pcd")
+
+# 列举目录下所有文件
+files = client.list_bucket_files_deep("path/to/folder/")
+
+# BFS 查找最浅的目标文件
+path = client.find_shallowest_target("root/", "target.json", "file")
+```
+
+### Windmill 平台工具
+
+```python
+from pewutils.wmill_util import zip_to_s3
+
+# with 块内生成文件，退出时自动打包 zip 上传到 Windmill S3
+with zip_to_s3("output/zip", suffix=".json") as out:
+    Path("a.json").write_text('{"k": 1}')
+    Path("b.json").write_text('{"k": 2}')
+# out.result -> S3Object(s3="output/zip/20260729/uuid.zip")
+# out.files  -> [Path("a.json"), Path("b.json")]
+```
+
+## 目录结构
+
+```
+pewutils/
+├── README.md
+├── pyproject.toml              # 打包配置（PEP 517/518，requires-python>=3.10）
+├── .gitignore
+└── pewutils/
+    ├── __init__.py             # 默认导出 url_util
+    ├── url_util.py             # URL 处理（纯标准库）
+    ├── oss_util.py             # OSS 工具（oss2, requests）
+    └── wmill_util.py           # Windmill 平台工具（wmill lazy import）
+```
+
+## 构建
+
+```bash
+pip install build
+python -m build
+# 产物：dist/pewutils-<version>-py3-none-any.whl
+```

pewutils-0.0.1/pewutils/__init__.py

@@ -0,0 +1,23 @@
+# -*- coding: UTF-8 -*-
+
+"""
+pewutils - PE Windmill Utils
+为 Windmill 脚本平台提供的轻量级 Python 工具包。
+
+设计原则：
+- 核心包零重依赖，保证 windmill 脚本冷启动快
+- 重依赖（oss2/open3d/opencv/pandas 等）按需拆分到可选子包或独立模块
+- 由 wind_scripts 脚本通过 `# requirements: pewutils==x.x.x` 引用
+"""
+
+__version__ = "0.0.1"
+__author__ = "rxu"
+
+# 默认导出轻量模块（纯标准库）
+from . import url_util
+
+# 其他模块按需导入（依赖已在 dependencies 里，直接 import 即可）
+# from . import oss_util
+# from . import wmill_util
+
+__all__ = ["url_util"]

pewutils-0.0.1/pewutils/oss_util.py

@@ -0,0 +1,524 @@
+# -*- coding: UTF-8 -*-
+
+"""
+pewutils.oss_util - 阿里云 OSS 工具
+从 peutils.ossutil 迁移，基于 oss2 + STS 鉴权。
+
+Change History:
+  2023-11-02: 初始版本（peutils ossutil.py, Henry.Wang）
+  2026-07-29: 迁移至 pewutils，去除 textutil/comutil 依赖
+"""
+
+import base64
+import json
+import os
+import platform
+import subprocess
+import time
+from collections import deque
+from pathlib import Path
+from typing import Optional
+
+import oss2
+from oss2 import determine_part_size
+from oss2.models import PartInfo
+
+import requests
+
+__all__ = [
+    "get_oss_auth_str",
+    "get_long_object_link",
+    "parse_auth_token",
+    "OSS_STS_API",
+    "OSSClientProxy",
+    "OSSClientFactory",
+]
+
+
+# ============================================================
+# 辅助函数
+# ============================================================
+
+def _get_device_sn() -> str:
+    """获取设备序列号，用于 Linux VPC 鉴权。"""
+    sys_kind = platform.system()
+    if sys_kind == "Darwin":
+        cmd = "ioreg -l | grep IOPlatformSerialNumber"
+        result = subprocess.check_output(cmd, shell=True).decode("utf-8")
+        return result.split(" ")[-1].strip('\n').strip('"')
+    elif sys_kind == "Windows":
+        try:
+            cmd = 'powershell -Command "Get-WmiObject Win32_BIOS | Select-Object -ExpandProperty SerialNumber"'
+            return subprocess.check_output(cmd, shell=True).decode("utf-8").strip()
+        except Exception:
+            cmd = "wmic bios get serialnumber"
+            result = subprocess.check_output(cmd, shell=True).decode("utf-8")
+            return result.split("\n")[1].strip()
+    else:
+        raise Exception(f"unknown system kind: {sys_kind}")
+
+
+def parse_auth_token(token: str) -> dict:
+    """解析 STS auth_str，返回包含 id/secret/stoken/osspath/region 的 dict。
+
+    替代原 peutils.textutil.parse_info_from_token。
+    """
+    return json.loads(base64.b64decode(token).decode())
+
+
+# ============================================================
+# 鉴权接口
+# ============================================================
+
+def get_oss_auth_str(oss_path: str, auth_type: str = "read", is_print: bool = True) -> str:
+    """调用鉴权服务获取 STS auth_str。
+
+    Args:
+        oss_path: OSS 路径，如 "oss://bucket/"
+        auth_type: 权限类型，"read" 或 "re_up"
+        is_print: 是否打印 auth_str（12 小时过期）
+    """
+    headers = {"Content-Type": "application/json"}
+    payload = {"oss_path": oss_path, "auth_type": auth_type}
+
+    sys_kind = platform.system()
+    if sys_kind == "Linux":
+        url = "https://dataflow.appen.com.cn/oss_auth/other/oss_auth_from_vpc/"
+    else:
+        url = "https://dataflow.appen.com.cn/oss_auth/other/oss_auth_from_debug/"
+        payload["sn_no"] = _get_device_sn()
+
+    r = requests.post(url=url, json=payload, headers=headers)
+    if r.status_code == 403:
+        ip_info = requests.get("https://jsonip.com").json().get("ip")
+        raise Exception(f"{url.split('/')[-2]} 当前未开通白名单 {ip_info}")
+
+    data = r.json()
+    if data["code"] != 200:
+        raise Exception(f"授权接口调用失败，具体原因: {data['message']}")
+    auth_str = data["data"]["auth_str"]
+    if is_print:
+        print("请复制下方的授权码，使用oss登陆，授权码将在12小时后过期，请尽快使用!")
+        print(auth_str)
+    return auth_str
+
+
+def get_long_object_link(
+    auth_str: str, bucket_name: str, file_key: str, duration: int = 604800
+) -> str:
+    """获取 OSS 对象的长期访问 URL（默认 7 天）。
+
+    Args:
+        auth_str: STS auth_str
+        bucket_name: bucket 名称
+        file_key: 对象 key
+        duration: 有效期（秒）
+    """
+    headers = {"Content-Type": "application/json"}
+    payload = {
+        "authToken": auth_str,
+        "filePath": file_key,
+        "bucketName": bucket_name,
+        "expiration": duration,
+    }
+
+    r = requests.post(
+        url="https://dataflow.appen.com.cn/oss_app/other/get_oss_file_url/",
+        json=payload,
+        headers=headers,
+    )
+    rsp = r.json()
+    if rsp["code"] != 200:
+        raise Exception(rsp["message"])
+    return rsp["data"]["url"]
+
+
+# ============================================================
+# OSS STS API
+# ============================================================
+
+class OSS_STS_API:
+    """阿里云 OSS STS 鉴权客户端。
+
+    自动通过鉴权服务获取 STS 凭证，支持 Linux VPC 内网访问。
+    """
+
+    def __init__(
+        self,
+        bucket_name: Optional[str] = None,
+        time_out: int = 60,
+        region: Optional[str] = None,
+        always_public: bool = False,
+        auth_str: Optional[str] = None,
+    ):
+        if auth_str:
+            self.auth_str = auth_str
+            self.auth_dict = parse_auth_token(self.auth_str)
+            from urllib.parse import urlparse
+            self.bucket_name = urlparse(self.auth_dict["osspath"]).hostname
+        else:
+            assert bucket_name, "bucket_name 和 auth_str 必须二选一"
+            self.bucket_name = bucket_name
+            oss_path = f"oss://{self.bucket_name}/"
+            self.auth_str = get_oss_auth_str(oss_path, auth_type="re_up", is_print=False)
+            self.auth_dict = parse_auth_token(self.auth_str)
+
+        self.auth = oss2.StsAuth(
+            self.auth_dict["id"],
+            self.auth_dict["secret"],
+            self.auth_dict["stoken"],
+        )
+        self.short_region = self.auth_dict["region"]
+
+        # 默认根据操作系统选择 endpoint
+        sys_kind = platform.system()
+        if region is None:
+            if sys_kind == "Linux":
+                region = f"http://{self.short_region}-internal.aliyuncs.com"
+            else:
+                region = f"http://{self.short_region}.aliyuncs.com"
+
+        if always_public:
+            region = f"http://{self.short_region}.aliyuncs.com"
+
+        self.region = region
+        self.bucket = oss2.Bucket(self.auth, region, self.bucket_name, connect_timeout=time_out)
+
+    # ------------------------------------------------------------
+    # 读取
+    # ------------------------------------------------------------
+
+    def read_bytes(self, oss_key: str) -> bytes:
+        """读取 OSS 对象为字节流。
+
+        替代散落的 `bucket.get_object(key).read()` 调用，方便配合 pcd_util.read_pcd_from_bytes 等使用。
+        """
+        return self.bucket.get_object(oss_key).read()
+
+    def check_is_empty_oss_folder(self, oss_key: str) -> bool:
+        """检查 OSS 文件夹是否为空（oss_key 必须以 / 结尾）。"""
+        assert oss_key.endswith("/"), "oss_key必须以/结尾"
+        assert not oss_key.startswith("/"), "oss_key不能以/开始"
+        is_empty = True
+        count = 1
+        for obj in oss2.ObjectIterator(self.bucket, prefix=oss_key):
+            if count > 2:
+                break
+            if obj.key != oss_key:
+                is_empty = False
+            count += 1
+        return is_empty
+
+    # ------------------------------------------------------------
+    # 列举
+    # ------------------------------------------------------------
+
+    def list_bucket_files_deep(
+        self, oss_path: str, suffix: str = "", with_bucket_name: bool = False
+    ) -> list[str]:
+        """深度列举 bucket 下所有文件，忽略临时/隐藏文件。"""
+        if not oss_path.endswith("/"):
+            oss_path = oss_path + "/"
+
+        path_list = []
+        ignore_list = []
+        for obj in oss2.ObjectIterator(self.bucket, prefix=oss_path):
+            if not obj.key.endswith("/") and obj.key != oss_path and obj.key.endswith(suffix):
+                basename = obj.key.split("/")[-1]
+                if basename.startswith((".", "~")):
+                    ignore_list.append(obj.key)
+                else:
+                    path_list.append(obj.key)
+
+        if ignore_list:
+            print(f"!请注意:{oss_path}下已自动忽略文件: {ignore_list}")
+
+        if with_bucket_name:
+            return [f"/{self.bucket_name}/" + x for x in path_list]
+        return path_list
+
+    def list_bucket_folders_deep(
+        self, oss_path: str, with_bucket_name: bool = False
+    ) -> list[str]:
+        """深度列举 bucket 下所有空叶子目录。"""
+        if not oss_path.endswith("/"):
+            oss_path = oss_path + "/"
+
+        path_list = []
+        for obj in oss2.ObjectIterator(self.bucket, prefix=oss_path):
+            if obj.key.endswith("/") and obj.key != oss_path:
+                fd_list = self.list_bucket_current(oss_path=obj.key, list_type="folder")
+                if len(fd_list) == 0:
+                    path_list.append(obj.key)
+        assert len(path_list) == len(set(path_list)), "folder出现重复"
+
+        if with_bucket_name:
+            return [f"/{self.bucket_name}/" + x for x in path_list]
+        return path_list
+
+    def list_bucket_current(
+        self,
+        oss_path: str,
+        list_type: str,
+        suffix: str = "",
+        full_path: bool = True,
+    ) -> list[str]:
+        """列举当前层级（非递归）的文件或文件夹。
+
+        Args:
+            list_type: "folder" 或 "file"
+            suffix: 文件后缀过滤（仅 list_type='file' 生效）
+            full_path: True 返回完整路径，False 返回 basename
+        """
+        if not oss_path.endswith("/"):
+            oss_path = oss_path + "/"
+
+        if list_type == "folder":
+            fd_list = [
+                obj.key
+                for obj in oss2.ObjectIterator(self.bucket, prefix=oss_path, delimiter="/")
+                if obj.key.endswith("/") and obj.key != oss_path
+            ]
+            if full_path:
+                return fd_list
+            return [os.path.basename(x[:-1]) for x in fd_list]
+
+        elif list_type == "file":
+            path_list = []
+            ignore_list = []
+            for obj in oss2.ObjectIterator(self.bucket, prefix=oss_path, delimiter="/"):
+                if not obj.key.endswith("/") and obj.key != oss_path and obj.key.endswith(suffix):
+                    basename = obj.key.split("/")[-1]
+                    if basename.startswith((".", "~")):
+                        ignore_list.append(obj.key)
+                    else:
+                        path_list.append(obj.key)
+            if ignore_list:
+                print(f"!请注意:{oss_path}下已自动忽略文件: {ignore_list}")
+            return path_list
+
+        else:
+            raise Exception("list_type 只支持 'folder' 或 'file'")
+
+    # ------------------------------------------------------------
+    # 元信息 / 复制
+    # ------------------------------------------------------------
+
+    def set_obj_meta(self, filename_list: list[str], meta_header: dict):
+        """批量更新对象元信息（Content-Type / Cache-Control 等）。"""
+        print(f"即将更新meta信息 {meta_header} ,文件数量{len(filename_list)},过程可能较长")
+        for fl in filename_list:
+            self.bucket.update_object_meta(fl, headers=meta_header)
+        print("meta信息更新完成.")
+
+    def copy_big_file(self, src_key: str, dest_key: str, src_bucket_name: str) -> int:
+        """分片复制大文件（>1G）。
+
+        注意：初始化的 bucket 必须是目标 bucket。
+        参考 https://help.aliyun.com/document_detail/88465.html
+        """
+        if src_bucket_name == self.bucket_name:
+            src_bucket = self.bucket
+        else:
+            src_bucket = oss2.Bucket(self.auth, self.region, src_bucket_name)
+
+        head_info = src_bucket.head_object(src_key)
+        total_size = head_info.content_length
+        part_size = determine_part_size(total_size, preferred_size=100 * 1024)
+
+        upload_id = self.bucket.init_multipart_upload(dest_key).upload_id
+        parts = []
+
+        part_number = 1
+        offset = 0
+        while offset < total_size:
+            num_to_upload = min(part_size, total_size - offset)
+            end = offset + num_to_upload - 1
+            result = self.bucket.upload_part_copy(
+                src_bucket_name, src_key, (offset, end), dest_key, upload_id, part_number
+            )
+            parts.append(PartInfo(part_number, result.etag))
+            offset += num_to_upload
+            part_number += 1
+
+        result = self.bucket.complete_multipart_upload(dest_key, upload_id, parts)
+        head_info = self.bucket.head_object(dest_key)
+        assert head_info.content_length == total_size
+        return result.status
+
+    # ------------------------------------------------------------
+    # 搜索
+    # ------------------------------------------------------------
+
+    def find_shallowest_target(
+        self, root: str, target_name: str, target_type: str
+    ) -> Optional[str]:
+        """BFS 查找路径最浅的目标文件/文件夹。"""
+        if target_type not in ("folder", "file"):
+            raise Exception("target_type只支持 folder or file")
+        if not root.endswith("/"):
+            root = f"{root}/"
+
+        queue = deque([root])
+        while queue:
+            current_prefix = queue.popleft()
+            for obj in oss2.ObjectIterator(self.bucket, prefix=current_prefix, delimiter="/"):
+                if obj.key == current_prefix:
+                    continue
+                if obj.is_prefix():
+                    if target_type == "folder" and Path(obj.key).name == target_name:
+                        return obj.key
+                    queue.append(obj.key)
+                else:
+                    if target_type == "file" and Path(obj.key).name == target_name:
+                        return obj.key
+        return None
+
+    def find_deepest_target(
+        self, root: str, target_name: str, target_type: str, current_depth: int = 0
+    ) -> tuple[Optional[str], int]:
+        """DFS 查找路径最深的目标文件/文件夹。"""
+        if target_type not in ("folder", "file"):
+            raise Exception("target_type只支持 folder or file")
+        if not root.endswith("/"):
+            root = f"{root}/"
+
+        deepest_path = None
+        max_depth = current_depth
+
+        for obj in oss2.ObjectIterator(self.bucket, prefix=root, delimiter="/"):
+            if obj.key == root:
+                continue
+            if obj.is_prefix():
+                if target_type == "folder" and Path(obj.key).name == target_name:
+                    deepest_path, max_depth = obj.key, current_depth
+                sub_path, sub_depth = self.find_deepest_target(
+                    obj.key, target_name, target_type, current_depth + 1
+                )
+                if sub_path and sub_depth > max_depth:
+                    deepest_path, max_depth = sub_path, sub_depth
+            else:
+                if target_type == "file" and Path(obj.key).name == target_name:
+                    deepest_path, max_depth = obj.key, current_depth
+
+        return deepest_path, max_depth
+
+    def find_paths_with_files(
+        self, oss_path: str, suffixes=None, debug: bool = False
+    ) -> list[str]:
+        """深度遍历 OSS 路径，找到包含匹配后缀文件的所有目录。"""
+        if not oss_path.endswith("/"):
+            oss_path = oss_path + "/"
+
+        if suffixes is None:
+            suffix_list = None
+        elif isinstance(suffixes, str):
+            suffix_list = [suffixes if suffixes.startswith(".") else f".{suffixes}"]
+        else:
+            suffix_list = [s if s.startswith(".") else f".{s}" for s in suffixes]
+
+        if debug:
+            print(f"搜索路径: {oss_path}")
+            print(f"目标后缀: {suffix_list}")
+
+        result_paths = set()
+        processed_count = 0
+        matched_count = 0
+
+        for obj in oss2.ObjectIterator(self.bucket, prefix=oss_path):
+            processed_count += 1
+            if obj.key.endswith("/"):
+                continue
+
+            obj_path = Path(obj.key)
+            if suffix_list is not None and obj_path.suffix not in suffix_list:
+                continue
+
+            matched_count += 1
+            if debug and matched_count <= 10:
+                print(f"匹配文件: {obj.key} (后缀: {obj_path.suffix})")
+
+            parent_str = str(obj_path.parent)
+            if not parent_str.endswith("/"):
+                parent_str += "/"
+            result_paths.add(parent_str)
+
+        if debug:
+            print(f"\n总计: 处理 {processed_count} 个对象, 匹配 {matched_count} 个文件")
+            print(f"找到 {len(result_paths)} 个目录")
+
+        return sorted(result_paths)
+
+
+# ============================================================
+# 自动刷新代理
+# ============================================================
+
+class OSSClientProxy:
+    """OSS client 代理，每次调用前检查凭证是否过期，超过 11 小时自动刷新。
+
+    auth_str 有效期 12 小时，阈值设为 11 小时。使用外部 auth_str 时不自动刷新。
+    """
+
+    _ttl_seconds: int = 11 * 60 * 60
+
+    def __init__(
+        self,
+        client: OSS_STS_API,
+        created_time: float,
+        bucket_name: str,
+        time_out: int = 60,
+        region: Optional[str] = None,
+        always_public: bool = False,
+        auth_str: Optional[str] = None,
+    ):
+        self._client = client
+        self._created_time = created_time
+        self._init_params = {
+            "bucket_name": bucket_name,
+            "time_out": time_out,
+            "region": region,
+            "always_public": always_public,
+            "auth_str": auth_str,
+        }
+        self._auto_refresh = auth_str is None
+
+    def _check_and_refresh(self):
+        if not self._auto_refresh:
+            return
+        if time.time() - self._created_time >= self._ttl_seconds:
+            self._client = OSS_STS_API(**self._init_params)
+            self._created_time = time.time()
+
+    def __getattr__(self, name):
+        self._check_and_refresh()
+        return getattr(self._client, name)
+
+
+class OSSClientFactory:
+    """OSS Client 工厂，创建带自动刷新的 OSS 客户端代理。"""
+
+    @staticmethod
+    def create_client(
+        bucket_name: str,
+        time_out: int = 60,
+        region: Optional[str] = None,
+        always_public: bool = False,
+        auth_str: Optional[str] = None,
+    ) -> OSSClientProxy:
+        client = OSS_STS_API(
+            bucket_name=bucket_name,
+            time_out=time_out,
+            region=region,
+            always_public=always_public,
+            auth_str=auth_str,
+        )
+        return OSSClientProxy(
+            client=client,
+            created_time=time.time(),
+            bucket_name=bucket_name,
+            time_out=time_out,
+            region=region,
+            always_public=always_public,
+            auth_str=auth_str,
+        )

pewutils-0.0.1/pewutils/url_util.py

@@ -0,0 +1,231 @@
+# -*- coding: UTF-8 -*-
+
+"""
+Author: rxu
+Date: 2024-07-17 16:38
+Short Description: URL处理工具模块（从 peutils 迁移至 pewutils）
+
+Change History:
+2024-07-17: 初始版本（peutils）
+2026-07-29: 迁移至 pewutils，适配 windmill 脚本平台；精简低频函数
+
+模块特点：仅依赖 Python 标准库，无第三方依赖。
+聚焦 windmill 脚本场景：OSS/存储 URL 解析、签名 URL 清理、查询参数操作。
+"""
+
+import urllib.parse
+
+
+# ----------------------------------------------------------------------------
+# 存储系统 URL（oss://, obs://, s3://, appen://）
+# ----------------------------------------------------------------------------
+
+_STORAGE_PREFIXES = ('oss://', 'obs://', 's3://', 'appen://')
+
+
+def is_storage_url(url: str) -> bool:
+    """
+    检查是否为存储系统URL (oss://, obs://, s3://, appen://)
+
+    Example:
+        >>> is_storage_url("oss://bucket/key")
+        True
+        >>> is_storage_url("https://bucket/key")
+        False
+    """
+    if not isinstance(url, str):
+        return False
+    return url.startswith(_STORAGE_PREFIXES)
+
+
+def parse_storage_url(storage_url: str) -> tuple[str, str]:
+    """
+    解析存储URL，返回(bucket, path)
+
+    Args:
+        storage_url (str): 存储URL (如: oss://bucket-name/path/to/file)
+
+    Returns:
+        tuple[str, str]: (bucket_name, path) 元组，path 可能为空字符串
+
+    Raises:
+        ValueError: 当URL格式不正确时
+
+    Example:
+        >>> parse_storage_url("oss://my-bucket/path/file.txt")
+        ('my-bucket', 'path/file.txt')
+        >>> parse_storage_url("oss://my-bucket/")
+        ('my-bucket', '')
+    """
+    if not isinstance(storage_url, str):
+        raise ValueError("存储URL必须是字符串")
+    if not is_storage_url(storage_url):
+        raise ValueError("不是有效的存储URL格式")
+
+    # 移除协议前缀
+    for prefix in _STORAGE_PREFIXES:
+        if storage_url.startswith(prefix):
+            path_without_prefix = storage_url[len(prefix):]
+            break
+
+    # 分割bucket和路径
+    parts = path_without_prefix.split('/', 1)
+    if len(parts) == 1:
+        return parts[0], ""
+    bucket_name, path = parts
+    return bucket_name, path
+
+
+def parse_oss_full_path(oss_full_path: str) -> tuple[str, str]:
+    """
+    解析 oss:// 路径，要求以 / 结尾，bucket/path 非空
+
+    与 parse_storage_url 的区别：强约束，适合脚本入参校验场景。
+    - 要求路径必须以 / 结尾（避免把文件当目录处理）
+    - bucket 和 path 都不能为空
+
+    Args:
+        oss_full_path (str): OSS 目录路径，如 "oss://bucket/folder/"
+
+    Returns:
+        tuple[str, str]: (bucket_name, oss_path)，oss_path 保留结尾的 "/"
+
+    Raises:
+        AssertionError: 当路径不以 oss:// 开头、不以 / 结尾，或 bucket/path 为空
+
+    Example:
+        >>> parse_oss_full_path("oss://my-bucket/folder/")
+        ('my-bucket', 'folder/')
+    """
+    oss_full_path = oss_full_path.strip()  # 去掉前后空格
+    assert oss_full_path.startswith("oss://"), "路径必须以oss://开始"
+    assert oss_full_path.endswith("/"), "路径必须以/结尾"
+    bucket_name, oss_path = parse_storage_url(oss_full_path)
+    assert bucket_name != '', "bucket不能为空，请检查"
+    assert oss_path != '', "路径不能为空，请检查"
+    return bucket_name, oss_path
+
+
+# ----------------------------------------------------------------------------
+# HTTP/HTTPS URL 清理与查询参数
+# ----------------------------------------------------------------------------
+
+def get_clean_url(url: str) -> str:
+    """
+    获取纯净url（移除查询参数并解码百分号）
+
+    典型场景：处理 OSS 临时签名 URL，去掉 Expires/OSSAccessKeyId/Signature 等参数，
+    并把 %2F 等解码回正常字符。
+
+    Example:
+        >>> get_clean_url("https://bucket.aliyuncs.com/path%2Ffile.pcd?Expires=123&Signature=xxx")
+        'https://bucket.aliyuncs.com/path/file.pcd'
+    """
+    if not isinstance(url, str):
+        raise ValueError("URL必须是字符串类型")
+    if not url.strip():
+        return ""
+
+    url = urllib.parse.unquote(url)
+    parsed_url = urllib.parse.urlparse(url)
+    return urllib.parse.urlunparse(parsed_url._replace(query=""))
+
+
+def is_private_url(url: str) -> bool:
+    """
+    判断是否为私有访问URL（含 OSS/AWS 签名参数）
+
+    Example:
+        >>> is_private_url("http://example.com/file.txt?Expires=123&OSSAccessKeyId=abc")
+        True
+        >>> is_private_url("http://example.com/file.txt")
+        False
+    """
+    if not isinstance(url, str):
+        return False
+    private_indicators = ('Expires=', 'OSSAccessKeyId=', 'Signature=', 'AWSAccessKeyId=')
+    return any(indicator in url for indicator in private_indicators)
+
+
+def extract_query_params(url: str) -> dict[str, list[str]]:
+    """
+    提取URL中的查询参数为字典
+
+    Example:
+        >>> extract_query_params("http://example.com?a=1&b=2&a=3")
+        {'a': ['1', '3'], 'b': ['2']}
+    """
+    if not isinstance(url, str):
+        return {}
+    parsed = urllib.parse.urlparse(url)
+    return urllib.parse.parse_qs(parsed.query)
+
+
+def add_query_params(url: str, params: dict[str, str | list[str]]) -> str:
+    """
+    向URL添加查询参数，已存在的参数会合并
+
+    Example:
+        >>> add_query_params("http://example.com?a=1", {'b': '2'})
+        'http://example.com?a=1&b=2'
+        >>> add_query_params("http://example.com?a=1", {'a': '2'})
+        'http://example.com?a=1&a=2'
+    """
+    if not isinstance(url, str) or not isinstance(params, dict):
+        return url
+
+    parsed = urllib.parse.urlparse(url)
+    existing_params = urllib.parse.parse_qs(parsed.query)
+
+    # 合并参数
+    for key, value in params.items():
+        new_values = value if isinstance(value, list) else [str(value)]
+        new_values = [str(v) for v in new_values]
+        if key in existing_params:
+            existing_params[key].extend(new_values)
+        else:
+            existing_params[key] = new_values
+
+    new_query = urllib.parse.urlencode(existing_params, doseq=True)
+    return urllib.parse.urlunparse(parsed._replace(query=new_query))
+
+
+# ----------------------------------------------------------------------------
+# URL 类型判断与路径提取
+# ----------------------------------------------------------------------------
+
+def is_http_url(url: str) -> bool:
+    """检查是否为HTTP/HTTPS协议的URL"""
+    if not isinstance(url, str):
+        return False
+    parsed = urllib.parse.urlparse(url)
+    return parsed.scheme.lower() in ('http', 'https')
+
+
+def get_relative_path(url: str) -> str:
+    """
+    获取URL中域名或bucket后的相对路径（去掉开头斜杠）
+
+    支持HTTP/HTTPS URL和存储系统URL。
+
+    Examples:
+        >>> get_relative_path("https://appen-data.aliyun.com/xc/occ/anno.json")
+        'xc/occ/anno.json'
+        >>> get_relative_path("appen://appen-data/xc/occ/anno.json")
+        'xc/occ/anno.json'
+        >>> get_relative_path("oss://my-bucket/folder/file.txt")
+        'folder/file.txt'
+    """
+    if not isinstance(url, str) or not url.strip():
+        return ""
+
+    try:
+        if is_storage_url(url):
+            _, path = parse_storage_url(url)
+            return path
+        if is_http_url(url):
+            parsed = urllib.parse.urlparse(url)
+            return parsed.path.lstrip('/')
+        return ""
+    except Exception:
+        return ""

pewutils-0.0.1/pewutils/wmill_util.py

@@ -0,0 +1,108 @@
+# -*- coding: UTF-8 -*-
+
+"""
+wmill_util - Windmill 平台专有工具
+
+提供与 windmill runtime（wmill SDK / S3Object）交互的封装。
+本地开发时若不调用相关方法，模块导入不会触发 wmill 加载。
+
+Change History:
+    2026-07-29: 从 wind_scripts/ropedia-video-after-parse-11245.py 迁移 s3_output
+                重命名为 zip_to_s3；修复 suffix 不带点时匹配失败的 bug
+"""
+
+import io
+import logging
+import uuid as _uuid
+import zipfile
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional, Union, List
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["zip_to_s3"]
+
+
+def _gen_zip_key(prefix: str) -> str:
+    """生成 S3 key：{prefix}/{YYYYMMDD}/{uuid}.zip"""
+    date_str = datetime.now(timezone.utc).strftime("%Y%m%d")
+    return f"{prefix}/{date_str}/{_uuid.uuid4()}.zip"
+
+
+def _normalize_suffixes(suffix: Union[str, List[str], None]) -> Union[set, None]:
+    """统一 suffix 为带点的集合：'json' / '.json' / ['json', '.txt'] → {'.json', '.txt'}"""
+    if suffix is None:
+        return None
+    if isinstance(suffix, str):
+        suffix = [suffix]
+    return {s if s.startswith(".") else f".{s}" for s in suffix}
+
+
+@contextmanager
+def zip_to_s3(
+    key_prefix: str = "zip_files",
+    suffix: Optional[Union[str, List[str]]] = None,
+):
+    """
+    上下文管理器：跟踪 with 块内新增的本地文件，退出时打包成 zip 上传到 Windmill S3。
+    S3 key 自动生成为：{key_prefix}/{YYYYMMDD}/{uuid}.zip，无需手动指定。
+
+    :param key_prefix: S3 key 前缀
+    :param suffix: 仅打包指定后缀的新文件；可为 str 或 list。
+                   'json' 与 '.json' 均可（自动补点）
+
+    用法::
+
+        with zip_to_s3("output/zip", suffix=".json") as out:
+            Path("a.json").write_text("...")
+        # 退出 with 时打包 → 上传，out.result 为 S3Object，out.files 为本地文件列表
+
+    本地开发若未安装 wmill，调用本函数会抛 ImportError。
+    """
+    try:
+        import wmill
+        from wmill import S3Object
+    except ImportError:
+        raise ImportError(
+            "wmill 未安装。windmill 脚本运行时无需手动安装；"
+            "本地开发请运行: pip install wmill"
+        )
+
+    suffixes = _normalize_suffixes(suffix)
+
+    watch_dir = Path.cwd()
+    existing = {p.name for p in watch_dir.iterdir() if p.is_file()}
+
+    class _Result:
+        def __init__(self):
+            self.result = None  # Optional[S3Object]，运行时赋值
+            self.files: List[Path] = []
+
+    out = _Result()
+
+    try:
+        yield out
+    finally:
+        new_files = [
+            p for p in watch_dir.iterdir()
+            if p.is_file()
+            and p.name not in existing
+            and (suffixes is None or p.suffix in suffixes)
+        ]
+        out.files = new_files
+
+        if not new_files:
+            logger.warning("zip_to_s3: 没有新增文件可上传")
+            return
+
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+            for f in new_files:
+                zf.write(f, f.name)
+        s3_key = _gen_zip_key(key_prefix)
+        obj = S3Object(s3=s3_key)
+        wmill.write_s3_file(obj, buf.getvalue())
+        out.result = obj
+        logger.info(f"zip_to_s3: 打包 {len(new_files)} 个文件 → {s3_key}")

pewutils-0.0.1/pewutils.egg-info/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: pewutils
+Version: 0.0.1
+Summary: Lightweight utils for windmill scripts (migrated from peutils).
+Author-email: rxu <rxu@appen.com>
+License: MIT
+Project-URL: Homepage, https://gitee.com/yunsansheng/pewutils
+Keywords: pewutils,windmill,pe
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: oss2>=2.15
+Requires-Dist: requests>=2.25
+
+# pewutils
+
+PE Windmill Utils —— 为 Windmill 脚本平台提供的轻量级 Python 工具包。
+
+由 `peutils` 迁移重构而来，适配 windmill 脚本平台。
+
+## 已迁移模块
+
+| 模块 | 说明 | 依赖 |
+|------|------|------|
+| `url_util` | URL 处理工具（解析、存储 URL、签名 URL 清理、查询参数等） | 仅标准库 |
+| `oss_util` | 阿里云 OSS 工具（STS 鉴权、列举、复制、搜索） | oss2, requests |
+| `wmill_util` | Windmill 平台工具（zip 打包上传 S3） | wmill（windmill runtime 内置） |
+
+## 安装
+
+```bash
+# 核心包（含 oss2/requests 依赖）
+pip install pewutils
+
+# 本地开发（editable）
+git clone <repo-url> pewutils
+cd pewutils
+pip install -e .
+```
+
+## 使用示例
+
+### URL 工具
+
+```python
+from pewutils.url_util import (
+    get_clean_url,
+    parse_storage_url,
+    parse_oss_full_path,
+    get_relative_path,
+)
+
+# 去除 OSS 签名 URL 的查询参数
+clean = get_clean_url("https://bucket.aliyuncs.com/path/file.pcd?Expires=123&Signature=xxx")
+# -> "https://bucket.aliyuncs.com/path/file.pcd"
+
+# 解析存储 URL（宽松）
+bucket, path = parse_storage_url("oss://my-bucket/folder/file.txt")
+# -> ("my-bucket", "folder/file.txt")
+
+# 解析 OSS 目录路径（强约束，要求以 / 结尾）
+bucket, folder = parse_oss_full_path("oss://my-bucket/folder/")
+```
+
+### OSS 工具
+
+```python
+from pewutils.oss_util import OSSClientFactory
+
+# 创建带自动刷新的 OSS client（auth_str 12h 过期，11h 自动刷新）
+client = OSSClientFactory.create_client(bucket_name="my-bucket")
+
+# 读取 OSS 对象为字节流
+data = client.read_bytes("path/to/file.pcd")
+
+# 列举目录下所有文件
+files = client.list_bucket_files_deep("path/to/folder/")
+
+# BFS 查找最浅的目标文件
+path = client.find_shallowest_target("root/", "target.json", "file")
+```
+
+### Windmill 平台工具
+
+```python
+from pewutils.wmill_util import zip_to_s3
+
+# with 块内生成文件，退出时自动打包 zip 上传到 Windmill S3
+with zip_to_s3("output/zip", suffix=".json") as out:
+    Path("a.json").write_text('{"k": 1}')
+    Path("b.json").write_text('{"k": 2}')
+# out.result -> S3Object(s3="output/zip/20260729/uuid.zip")
+# out.files  -> [Path("a.json"), Path("b.json")]
+```
+
+## 目录结构
+
+```
+pewutils/
+├── README.md
+├── pyproject.toml              # 打包配置（PEP 517/518，requires-python>=3.10）
+├── .gitignore
+└── pewutils/
+    ├── __init__.py             # 默认导出 url_util
+    ├── url_util.py             # URL 处理（纯标准库）
+    ├── oss_util.py             # OSS 工具（oss2, requests）
+    └── wmill_util.py           # Windmill 平台工具（wmill lazy import）
+```
+
+## 构建
+
+```bash
+pip install build
+python -m build
+# 产物：dist/pewutils-<version>-py3-none-any.whl
+```

pewutils-0.0.1/pewutils.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+pewutils/__init__.py
+pewutils/oss_util.py
+pewutils/url_util.py
+pewutils/wmill_util.py
+pewutils.egg-info/PKG-INFO
+pewutils.egg-info/SOURCES.txt
+pewutils.egg-info/dependency_links.txt
+pewutils.egg-info/requires.txt
+pewutils.egg-info/top_level.txt

pewutils-0.0.1/pewutils.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pewutils-0.0.1/pewutils.egg-info/requires.txt

@@ -0,0 +1,2 @@
+oss2>=2.15
+requests>=2.25

pewutils-0.0.1/pewutils.egg-info/top_level.txt

@@ -0,0 +1 @@
+pewutils

pewutils-0.0.1/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pewutils"
+version = "0.0.1"
+description = "Lightweight utils for windmill scripts (migrated from peutils)."
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+authors = [
+    { name = "rxu", email = "rxu@appen.com" },
+]
+keywords = ["pewutils", "windmill", "pe"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+# 默认安装核心依赖（轻量）
+dependencies = [
+    "oss2>=2.15",
+    "requests>=2.25",
+]
+
+# 可选重依赖：按需安装
+# pcd = ["pypcd4>=1.0", "plyfile>=0.7.4", "open3d>=0.19"]
+# image = ["opencv-python", "Pillow", "numpy"]
+# excel = ["openpyxl", "pandas"]
+
+[project.urls]
+Homepage = "https://gitee.com/yunsansheng/pewutils"
+
+[tool.setuptools.packages.find]
+include = ["pewutils*"]

pewutils-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+