dexverse-uploader 0.1.4a1__tar.gz

dexverse_uploader-0.1.4a1/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.5
+Name: dexverse-uploader
+Version: 0.1.4a1
+Summary: DexVerse edge-device data uploader
+Requires-Python: <3.11,>=3.10
+Requires-Dist: paho-mqtt<3,>=2.0
+Requires-Dist: requests<3,>=2.25
+Provides-Extra: hdf5
+Requires-Dist: h5py<4,>=3; extra == 'hdf5'
+Provides-Extra: piper
+Requires-Dist: h5py<4,>=3.8; extra == 'piper'
+Requires-Dist: numpy<3,>=1.24; extra == 'piper'
+Requires-Dist: pillow<13,>=10; extra == 'piper'
+Provides-Extra: tianji
+Requires-Dist: av<19,>=8.1; extra == 'tianji'
+Requires-Dist: h5ffmpeg==2.4.0; extra == 'tianji'
+Requires-Dist: h5py<4,>=3.6; extra == 'tianji'
+Requires-Dist: imgaug==0.4.0; extra == 'tianji'
+Requires-Dist: lz4<5,>=4.3; extra == 'tianji'
+Requires-Dist: mcap-ros2-support==0.5.7; extra == 'tianji'
+Requires-Dist: mcap==1.4.0; extra == 'tianji'
+Requires-Dist: numpy<2,>=1.24; extra == 'tianji'
+Requires-Dist: pillow<12,>=9; extra == 'tianji'
+Requires-Dist: pyyaml<7,>=6; extra == 'tianji'
+Requires-Dist: zstandard<1,>=0.23; extra == 'tianji'
+Provides-Extra: umi
+Requires-Dist: h5ffmpeg==2.4.0; extra == 'umi'
+Requires-Dist: h5py==3.14.0; extra == 'umi'
+Requires-Dist: hdfdict==0.3.1; extra == 'umi'
+Requires-Dist: numpy==1.26.4; extra == 'umi'
+Requires-Dist: opencv-python-headless<5,>=4.8; extra == 'umi'
+Requires-Dist: scipy<2,>=1.11; extra == 'umi'
+Requires-Dist: tqdm<5,>=4.66; extra == 'umi'
+Provides-Extra: w1
+Requires-Dist: h5py<4,>=3; extra == 'w1'
+Requires-Dist: numpy<3,>=1.24; extra == 'w1'
+Requires-Dist: opencv-python-headless<5,>=4.8; extra == 'w1'
+Requires-Dist: pillow<13,>=10; extra == 'w1'
+Requires-Dist: pyyaml<7,>=5.1; extra == 'w1'
+Requires-Dist: tqdm<5,>=4.66; extra == 'w1'
+Description-Content-Type: text/markdown
+
+# DexVerse Uploader
+
+The uploader is a standard `src`-layout Python package. Install the base
+wheel for the MQTT/HTTP service and select a protocol extra when the target
+runtime needs its Python-side conversion or quality-check dependencies:
+
+```text
+pip install dexverse-uploader
+pip install dexverse-uploader[w1]
+pip install dexverse-uploader[umi]
+```
+
+The profile requirements under `requirements/` remain the offline deployment
+manifests. They are kept separate from package metadata because W1/GStreamer,
+UMI/native codecs, and other protocol components can be supplied by the edge
+image or a dedicated runtime. The service keeps the existing MQTT RPC
+contracts and can be started with `python -m dexverse_uploader.device_main`.

dexverse_uploader-0.1.4a1/README.md

@@ -0,0 +1,17 @@
+# DexVerse Uploader
+
+The uploader is a standard `src`-layout Python package. Install the base
+wheel for the MQTT/HTTP service and select a protocol extra when the target
+runtime needs its Python-side conversion or quality-check dependencies:
+
+```text
+pip install dexverse-uploader
+pip install dexverse-uploader[w1]
+pip install dexverse-uploader[umi]
+```
+
+The profile requirements under `requirements/` remain the offline deployment
+manifests. They are kept separate from package metadata because W1/GStreamer,
+UMI/native codecs, and other protocol components can be supplied by the edge
+image or a dedicated runtime. The service keeps the existing MQTT RPC
+contracts and can be started with `python -m dexverse_uploader.device_main`.

dexverse_uploader-0.1.4a1/cloud_upload.sh

@@ -0,0 +1,266 @@
+#!/bin/bash
+# ----------------------------------------------------------------------------
+# Copyright (c) 2021-2025 DexForce Technology Co., Ltd.
+#
+# All rights reserved.
+# ----------------------------------------------------------------------------
+# cloud_upload.sh — 云端上传服务启动/重启/停止脚本
+#
+# 用法:
+#   bash dexverse/dexverse-uploader/cloud_upload.sh [start|restart|stop]
+#
+# 说明:
+#   1) Python 包位于 src/dexverse_uploader/，外层目录名不参与导入。
+#   2) base 主进程与 UMI 转换进程可使用不同 Python 环境。
+#   3) 支持 start / restart / stop 三种模式。
+#   4) 单实例锁，防止重复拉起。
+#   5) 日志统一输出到模块内 log/ 目录，模块日志由 get_logger() 自动写入。
+# ----------------------------------------------------------------------------
+
+set -u
+
+MODE="${1:-start}"
+if [[ "${MODE}" != "start" && "${MODE}" != "restart" && "${MODE}" != "stop" ]]; then
+    echo "用法: bash $(basename "$0") [start|restart|stop]"
+    exit 1
+fi
+
+# 发布目录；外层目录名不参与模块导入。
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RUNTIME_DIR="${CLOUD_UPLOAD_RUNTIME_DIR:-${SCRIPT_DIR}}"
+CONFIG_PATH="${CLOUD_UPLOAD_CONFIG:-${RUNTIME_DIR}/config.json}"
+PACKAGE_PARENT="${SCRIPT_DIR}/src"
+
+# 日志目录
+LOG_DIR="${CLOUD_UPLOAD_LOG_DIR:-${RUNTIME_DIR}/log}"
+mkdir -p "${LOG_DIR}"
+
+# base 主进程只需要扫描、MQTT、HTTP 和日志依赖。
+PYTHON_BIN="${CLOUD_UPLOAD_BASE_PYTHON:-python3}"
+BASE_DEPS_DIR="${CLOUD_UPLOAD_BASE_DEPS_DIR:-${RUNTIME_DIR}/.deps/base}"
+W1_DEPS_DIR="${CLOUD_UPLOAD_W1_DEPS_DIR:-${RUNTIME_DIR}/.deps/w1}"
+PIP_ZIPAPP="${CLOUD_UPLOAD_PIP_ZIPAPP:-${RUNTIME_DIR}/.deps/pip.pyz}"
+BASE_REQUIREMENTS="${SCRIPT_DIR}/requirements/base.txt"
+HDF5_REQUIREMENTS="${SCRIPT_DIR}/requirements/hdf5.txt"
+W1_REQUIREMENTS="${SCRIPT_DIR}/requirements/w1.txt"
+# UMI worker 使用独立环境；未配置时回退到 base Python，便于兼容现有部署。
+UMI_PYTHON_BIN="${CLOUD_UPLOAD_UMI_PYTHON:-${PYTHON_BIN}}"
+START_MODULE="device_main"
+SERVICE_PATTERN="([p]ython[^ ]* -m [d]exverse_uploader.device_main|[p]ackage_runner.py device_main|python3 -m [d]exe_cloud_upload.device_main)"
+
+# ── 单实例锁 ──────────────────────────────────────────────────────────
+LAUNCHER_LOCK_FILE="${LOG_DIR}/dexverse_uploader.launcher.pid"
+SERVICE_PID_FILE="${LOG_DIR}/dexverse_uploader.pid"
+
+acquire_launcher_lock() {
+    local current_pid="$$"
+
+    if [[ -f "${LAUNCHER_LOCK_FILE}" ]]; then
+        local old_pid
+        old_pid="$(cat "${LAUNCHER_LOCK_FILE}" 2>/dev/null || true)"
+
+        if [[ -n "${old_pid}" && "${old_pid}" =~ ^[0-9]+$ ]] && kill -0 "${old_pid}" 2>/dev/null; then
+            if [[ "${MODE}" == "restart" || "${MODE}" == "stop" ]]; then
+                echo "[WARN] 检测到已有启动脚本实例 (pid=${old_pid})，先停止旧实例。"
+                kill -TERM "${old_pid}" 2>/dev/null || true
+                for _ in 1 2 3 4 5; do
+                    if ! kill -0 "${old_pid}" 2>/dev/null; then
+                        break
+                    fi
+                    sleep 1
+                done
+                if kill -0 "${old_pid}" 2>/dev/null; then
+                    echo "[WARN] 旧实例未退出，执行强制停止: pid=${old_pid}"
+                    kill -KILL "${old_pid}" 2>/dev/null || true
+                fi
+            else
+                echo "[ERROR] 已有启动脚本实例在运行 (pid=${old_pid})。"
+                echo "[ERROR] 请使用: bash $(basename "$0") restart"
+                exit 1
+            fi
+        fi
+    fi
+
+    echo "${current_pid}" > "${LAUNCHER_LOCK_FILE}"
+    trap 'rm -f "${LAUNCHER_LOCK_FILE}"' EXIT
+}
+
+# ── 环境检查 ──────────────────────────────────────────────────────────
+
+check_env() {
+    if [[ ! -f "${PACKAGE_PARENT}/dexverse_uploader/device_main.py" ]]; then
+        echo "[ERROR] Python 包不存在: ${PACKAGE_PARENT}/dexverse_uploader"
+        exit 1
+    fi
+
+    if [[ ! -f "${CONFIG_PATH}" ]]; then
+        echo "[ERROR] 配置文件不存在: ${CONFIG_PATH}"
+        exit 1
+    fi
+
+    if [[ ! -f "${BASE_REQUIREMENTS}" ]]; then
+        echo "[ERROR] base 依赖清单不存在: ${BASE_REQUIREMENTS}"
+        exit 1
+    fi
+    if [[ ! -f "${HDF5_REQUIREMENTS}" ]]; then
+        echo "[ERROR] HDF5 依赖清单不存在: ${HDF5_REQUIREMENTS}"
+        exit 1
+    fi
+    if [[ ! -f "${W1_REQUIREMENTS}" ]]; then
+        echo "[ERROR] W1 依赖清单不存在: ${W1_REQUIREMENTS}"
+        exit 1
+    fi
+}
+
+ensure_base_dependencies() {
+    local import_check="import numpy, paho.mqtt.client, requests, h5py"
+    if PYTHONPATH="${BASE_DEPS_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \
+        "${PYTHON_BIN}" -c "${import_check}" >/dev/null 2>&1; then
+        return 0
+    fi
+
+    echo "[INFO] 检测到 base 依赖缺失，安装到模块私有目录: ${BASE_DEPS_DIR}"
+    mkdir -p "${BASE_DEPS_DIR}" "$(dirname "${PIP_ZIPAPP}")"
+
+    if [[ ! -f "${PIP_ZIPAPP}" ]]; then
+        echo "[INFO] 下载 pip 启动器..."
+        "${PYTHON_BIN}" - "${PIP_ZIPAPP}" <<'PY'
+import pathlib
+import sys
+import urllib.request
+
+target = pathlib.Path(sys.argv[1])
+urllib.request.urlretrieve("https://bootstrap.pypa.io/pip/pip.pyz", target)
+PY
+        if [[ $? -ne 0 ]]; then
+            echo "[ERROR] pip 启动器下载失败，请检查网络或设置 CLOUD_UPLOAD_PIP_ZIPAPP。"
+            exit 1
+        fi
+    fi
+
+    "${PYTHON_BIN}" "${PIP_ZIPAPP}" install \
+        --disable-pip-version-check \
+        --target "${BASE_DEPS_DIR}" \
+        -r "${BASE_REQUIREMENTS}"
+    if [[ $? -ne 0 ]]; then
+        echo "[ERROR] base 依赖安装失败: ${BASE_REQUIREMENTS}"
+        exit 1
+    fi
+
+    "${PYTHON_BIN}" "${PIP_ZIPAPP}" install \
+        --disable-pip-version-check \
+        --no-deps \
+        --target "${BASE_DEPS_DIR}" \
+        -r "${HDF5_REQUIREMENTS}"
+    if [[ $? -ne 0 ]]; then
+        echo "[ERROR] HDF5 依赖安装失败: ${HDF5_REQUIREMENTS}"
+        exit 1
+    fi
+
+    if ! PYTHONPATH="${BASE_DEPS_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \
+        "${PYTHON_BIN}" -c "${import_check}" >/dev/null 2>&1; then
+        echo "[ERROR] base 依赖安装后仍无法导入，请确认设备 Python 已提供兼容的 NumPy。"
+        exit 1
+    fi
+}
+
+# ── 停止已有服务 ──────────────────────────────────────────────────────
+
+stop_existing_services() {
+    local pids
+    pids="$(pgrep -f "${SERVICE_PATTERN}" || true)"
+    if [[ -z "${pids}" ]]; then
+        rm -f "${SERVICE_PID_FILE}"
+        echo "[INFO] 未发现运行中的云端上传服务。"
+        return 0
+    fi
+
+    echo "[INFO] 准备停止云端上传服务 (pid=${pids//$'\n'/, })"
+    for pid in ${pids}; do
+        kill -TERM "${pid}" 2>/dev/null || true
+    done
+
+    # 等待优雅退出
+    for _ in 1 2 3 4 5; do
+        local any_alive=0
+        for pid in ${pids}; do
+            if kill -0 "${pid}" 2>/dev/null; then
+                any_alive=1
+            fi
+        done
+        [[ "${any_alive}" -eq 0 ]] && break
+        sleep 1
+    done
+
+    # 强制停止
+    for pid in ${pids}; do
+        if kill -0 "${pid}" 2>/dev/null; then
+            echo "[WARN] 进程 ${pid} 未在超时内退出，执行强制停止。"
+            kill -KILL "${pid}" 2>/dev/null || true
+        fi
+    done
+
+    rm -f "${SERVICE_PID_FILE}"
+    echo "[INFO] 云端上传服务已停止。"
+}
+
+# ── 主流程 ────────────────────────────────────────────────────────────
+
+check_env
+
+if [[ "${MODE}" == "stop" ]]; then
+    echo "[INFO] 模式: stop"
+    stop_existing_services
+    exit 0
+fi
+
+if [[ "${MODE}" == "restart" ]]; then
+    echo "[INFO] 模式: restart（先停后起）"
+    stop_existing_services
+else
+    echo "[INFO] 模式: start"
+    existing_pids="$(pgrep -f "${SERVICE_PATTERN}" || true)"
+    if [[ -n "${existing_pids}" ]]; then
+        echo "[ERROR] 已有云端上传服务运行中 (pid=${existing_pids//$'\n'/, })。"
+        echo "[ERROR] 如需切换到当前模块，请使用: bash $(basename "$0") restart"
+        exit 1
+    fi
+fi
+
+acquire_launcher_lock
+ensure_base_dependencies
+
+cd "${RUNTIME_DIR}" || exit 1
+
+echo "[INFO] 运行目录: ${RUNTIME_DIR}"
+echo "[INFO] 启动云端上传服务..."
+
+# 启动服务（后台运行，模块方式）
+CONSOLE_LOG="${LOG_DIR}/service_console.log"
+nohup env PYTHONPATH="${PACKAGE_PARENT}:${BASE_DEPS_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \
+    CLOUD_UPLOAD_LOG_DIR="${LOG_DIR}" \
+    CLOUD_UPLOAD_RUNTIME_DIR="${RUNTIME_DIR}" \
+    CLOUD_UPLOAD_CONFIG="${CONFIG_PATH}" \
+    CLOUD_UPLOAD_W1_DEPS_DIR="${W1_DEPS_DIR}" \
+    CLOUD_UPLOAD_W1_REQUIREMENTS="${W1_REQUIREMENTS}" \
+    CLOUD_UPLOAD_PIP_ZIPAPP="${PIP_ZIPAPP}" \
+    CLOUD_UPLOAD_UMI_PYTHON="${UMI_PYTHON_BIN}" \
+    PYTHONUNBUFFERED=1 \
+    "${PYTHON_BIN}" -m "dexverse_uploader.${START_MODULE}" >> "${CONSOLE_LOG}" 2>&1 &
+
+SERVICE_PID=$!
+echo "${SERVICE_PID}" > "${SERVICE_PID_FILE}"
+sleep 3
+if ! kill -0 "${SERVICE_PID}" 2>/dev/null; then
+    rm -f "${SERVICE_PID_FILE}"
+    echo "[ERROR] 云端上传服务启动后立即退出。"
+    echo "[ERROR] 最近日志: ${CONSOLE_LOG}"
+    tail -n 20 "${CONSOLE_LOG}" 2>/dev/null || true
+    exit 1
+fi
+echo "[INFO] 云端上传服务已启动 (pid=${SERVICE_PID})"
+echo "[INFO] 日志:"
+echo "       tail -f ${CONSOLE_LOG}"
+echo "       tail -f ${LOG_DIR}/mqtt_client.log"
+echo "       tail -f ${LOG_DIR}/file_scanner.log"
+echo "       tail -f ${LOG_DIR}/upload_manager.log"

dexverse_uploader-0.1.4a1/package_runner.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""从任意名称的发布目录运行 dexverse_uploader 模块。"""
+
+from __future__ import annotations
+
+import importlib.util
+import runpy
+import sys
+from pathlib import Path
+
+
+PACKAGE_NAME = "dexverse_uploader"
+PACKAGE_PARENT = Path(__file__).resolve().parent / "src"
+PACKAGE_ROOT = PACKAGE_PARENT / PACKAGE_NAME
+
+
+def load_package() -> None:
+    """将源码布局中的固定包名注册给测试和源码运行器。"""
+    if PACKAGE_NAME in sys.modules:
+        return
+    spec = importlib.util.spec_from_file_location(
+        PACKAGE_NAME,
+        PACKAGE_ROOT / "__init__.py",
+        submodule_search_locations=[str(PACKAGE_ROOT)],
+    )
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"无法加载上传模块: {PACKAGE_ROOT}")
+    package = importlib.util.module_from_spec(spec)
+    sys.modules[PACKAGE_NAME] = package
+    spec.loader.exec_module(package)
+
+
+def main() -> None:
+    """运行命令行指定的子模块（支持点分隔的协议路径）。"""
+    if len(sys.argv) < 2:
+        raise SystemExit(f"用法: {Path(sys.argv[0]).name} <module> [args...]")
+    module_name = sys.argv.pop(1)
+    load_package()
+    sys.argv[0] = str(PACKAGE_ROOT.joinpath(*module_name.split(".")).with_suffix(".py"))
+    runpy.run_module(f"{PACKAGE_NAME}.{module_name}", run_name="__main__")
+
+
+if __name__ == "__main__":
+    main()

dexverse_uploader-0.1.4a1/pyproject.toml

@@ -0,0 +1,105 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dexverse-uploader"
+version = "0.1.4a1"
+description = "DexVerse edge-device data uploader"
+readme = "README.md"
+requires-python = ">=3.10,<3.11"
+dependencies = [
+  "paho-mqtt>=2.0,<3",
+  "requests>=2.25,<3",
+]
+
+[project.scripts]
+dexverse-uploader = "dexverse_uploader.device_main:main"
+dexverse-uploader-converter = "dexverse_uploader.converter_worker:main"
+
+[project.optional-dependencies]
+# These profiles are intentionally separate: edge images may provide some of
+# the native/system components themselves, while wheel-based environments can
+# opt into the corresponding Python dependencies.
+hdf5 = [
+  "h5py>=3,<4",
+]
+w1 = [
+  "h5py>=3,<4",
+  "numpy>=1.24,<3",
+  "opencv-python-headless>=4.8,<5",
+  "Pillow>=10,<13",
+  "PyYAML>=5.1,<7",
+  "tqdm>=4.66,<5",
+]
+piper = [
+  "numpy>=1.24,<3",
+  "h5py>=3.8,<4",
+  "Pillow>=10,<13",
+]
+tianji = [
+  "mcap==1.4.0",
+  "mcap-ros2-support==0.5.7",
+  "lz4>=4.3,<5",
+  "zstandard>=0.23,<1",
+  "av>=8.1,<19",
+  "h5py>=3.6,<4",
+  "numpy>=1.24,<2",
+  "h5ffmpeg==2.4.0",
+  "imgaug==0.4.0",
+  "Pillow>=9,<12",
+  "PyYAML>=6,<7",
+]
+umi = [
+  "h5ffmpeg==2.4.0",
+  "h5py==3.14.0",
+  "numpy==1.26.4",
+  "scipy>=1.11,<2",
+  "opencv-python-headless>=4.8,<5",
+  "hdfdict==0.3.1",
+  "tqdm>=4.66,<5",
+]
+
+[dependency-groups]
+dev = [
+  "mypy>=1.15,<2",
+  "pytest>=8.3,<10",
+  "ruff>=0.11,<1",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dexverse_uploader"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/pyproject.toml",
+  "/README.md",
+  "/src/dexverse_uploader",
+  "/requirements",
+  "/cloud_upload.sh",
+  "/package_runner.py",
+]
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F"]
+ignore = ["E501"]
+
+[tool.mypy]
+python_version = "3.10"
+check_untyped_defs = true
+disallow_any_generics = false
+disallow_incomplete_defs = false
+disallow_untyped_defs = false
+warn_return_any = false
+warn_unreachable = true
+ignore_missing_imports = true
+files = ["src"]

dexverse_uploader-0.1.4a1/requirements/base.txt

@@ -0,0 +1,3 @@
+# dexverse_uploader 主进程：扫描、MQTT、HTTP 上传。
+paho-mqtt>=2.0,<3
+requests>=2.25,<3

dexverse_uploader-0.1.4a1/requirements/dev.txt

@@ -0,0 +1,7 @@
+# dexverse_uploader 开发与测试工具；不进入生产主进程环境。
+-r base.txt
+pytest>=8,<9
+ruff>=0.6,<1
+# 合成 HDF5 checker 测试所需；不引入 h5ffmpeg/转换 worker。
+h5py>=3,<4
+numpy>=1.24,<2

dexverse_uploader-0.1.4a1/requirements/hdf5.txt

@@ -0,0 +1,2 @@
+# HDF5 元信息写入；启动脚本使用 --no-deps 安装，复用设备已有的 NumPy。
+h5py>=3,<4

dexverse_uploader-0.1.4a1/requirements/piper.txt

@@ -0,0 +1,4 @@
+# PIPER HDF5 质检私有依赖，不进入 base 主进程。
+numpy>=1.24,<3
+h5py>=3.8,<4
+Pillow>=10,<13

dexverse_uploader-0.1.4a1/requirements/tianji.txt

@@ -0,0 +1,13 @@
+# 天机 converter worker；首次转换时按需安装到模块私有目录。
+mcap==1.4.0
+mcap-ros2-support==0.5.7
+lz4>=4.3,<5
+zstandard>=0.23,<1
+av>=8.1,<19
+h5py>=3.6,<4
+numpy>=1.24,<2
+h5ffmpeg==2.4.0
+# MR-compatible optional resize path (NumPy must remain <2 for imgaug).
+imgaug==0.4.0
+Pillow>=9,<12
+PyYAML>=6,<7

dexverse_uploader-0.1.4a1/requirements/umi.txt

@@ -0,0 +1,8 @@
+# UMI converter worker；建议安装在容器或独立 venv 中。
+h5ffmpeg==2.4.0
+h5py==3.14.0
+numpy==1.26.4
+scipy>=1.11,<2
+opencv-python-headless>=4.8,<5
+hdfdict==0.3.1
+tqdm>=4.66,<5

dexverse_uploader-0.1.4a1/requirements/w1.txt

@@ -0,0 +1,2 @@
+# W1 converter 按需安装；其余 NumPy/OpenCV/GStreamer 由 Jetson 系统环境提供。
+h5py>=3,<4

dexverse_uploader-0.1.4a1/src/dexverse_uploader/__init__.py

@@ -0,0 +1,49 @@
+# ----------------------------------------------------------------------------
+# Copyright (c) 2021-2025 DexForce Technology Co., Ltd.
+#
+# All rights reserved.
+# ----------------------------------------------------------------------------
+"""云端上传模块 — 将录制的遥操数据上传到云端服务器。
+
+控制面：MQTT RPC（get_data / upload）
+数据面：HTTP PUT 到 MINIO presigned-url
+"""
+
+from dexverse_uploader.chunk_uploader import ChunkUploader
+from dexverse_uploader.converter import (
+    Converter,
+    ConverterRegistry,
+    TianjiSubprocessConverter,
+    UmiSubprocessConverter,
+    W1Converter,
+    default_registry,
+)
+from dexverse_uploader.file_scanner import FileScanner
+from dexverse_uploader.quality_profiles import (
+    QualityProfileError,
+    build_default_profile,
+    quality_templates,
+    validate_quality_profile,
+)
+from dexverse_uploader.quality_result import build_quality_result, compact_quality_result
+from dexverse_uploader.quality_report_store import QualityReportStore
+from dexverse_uploader.upload_manager import UploadManager
+
+__all__ = [
+    "ChunkUploader",
+    "Converter",
+    "ConverterRegistry",
+    "FileScanner",
+    "QualityProfileError",
+    "QualityReportStore",
+    "TianjiSubprocessConverter",
+    "UmiSubprocessConverter",
+    "UploadManager",
+    "W1Converter",
+    "default_registry",
+    "build_default_profile",
+    "build_quality_result",
+    "compact_quality_result",
+    "quality_templates",
+    "validate_quality_profile",
+]

dexverse_uploader-0.1.4a1/src/dexverse_uploader/chunk_uploader.py

@@ -0,0 +1,76 @@
+# ----------------------------------------------------------------------------
+# Copyright (c) 2021-2025 DexForce Technology Co., Ltd.
+#
+# All rights reserved.
+# ----------------------------------------------------------------------------
+"""chunk_uploader.py — 文件上传器：抽样摘要计算 + HTTP PUT 到 presigned-url。"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Callable, Dict
+
+from dexverse_uploader.logger import get_logger
+from dexverse_uploader.sample_hash import HASH_TYPE, compute_sample_hash_bytes, file_signature
+
+logger = get_logger("chunk_uploader")
+
+
+class ChunkUploader:
+    """文件上传器。
+
+    云端接口模式：接收 presigned-url → HTTP PUT 整个文件 → 返回抽样摘要。
+    不再做分片。
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def upload_file(
+        self,
+        file_path: Path,
+        presigned_url: str,
+        put_cb: Callable[[str, bytes], bool],
+    ) -> Dict:
+        """上传单个文件到 MINIO presigned-url。
+
+        Args:
+            file_path: 文件路径
+            presigned_url: MINIO presigned upload URL
+            put_cb: HTTP PUT 回调 (url, file_bytes) -> bool
+
+        Returns:
+            {"success": bool, "content_hash": str, "hash_type": str, "size": int, "error": str}
+        """
+        try:
+            with open(file_path, "rb") as f:
+                before = os.fstat(f.fileno())
+                data = f.read()
+                after = os.fstat(f.fileno())
+            if (file_signature(before) != file_signature(after)
+                    or file_signature(after) != file_signature(Path(file_path).stat())
+                    or len(data) != before.st_size):
+                raise OSError("文件在上传读取期间发生变化，请重试")
+        except Exception as e:
+            return {"success": False, "content_hash": "", "hash_type": "", "size": 0, "error": str(e)}
+
+        content_hash = compute_sample_hash_bytes(data)
+        file_size = len(data)
+
+        try:
+            ok = put_cb(presigned_url, data)
+            if ok:
+                return {"success": True, "content_hash": content_hash, "hash_type": HASH_TYPE, "size": file_size, "error": ""}
+            else:
+                return {"success": False, "content_hash": content_hash, "hash_type": HASH_TYPE, "size": file_size, "error": "PUT failed"}
+        except Exception as e:
+            error_type = type(e).__name__
+            logger.error("上传回调异常: %s", error_type)
+            return {
+                "success": False,
+                "content_hash": content_hash,
+                "hash_type": HASH_TYPE,
+                "size": file_size,
+                "error": f"PUT callback exception: {error_type}",
+            }