cdt-data-uploader 1.0.1__py3-none-any.whl

cdt_data_uploader-1.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 VLA Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cdt_data_uploader-1.0.1.dist-info/METADATA

@@ -0,0 +1,395 @@
+Metadata-Version: 2.1
+Name: cdt-data-uploader
+Version: 1.0.1
+Summary: Python SDK for uploading CDT datasets (VLA, etc.)
+Home-page: https://github.com/yourorg/cdt-data-uploader
+Author: CDT Team
+Author-email: cdt@example.com
+License: UNKNOWN
+Project-URL: Bug Reports, https://github.com/yourorg/cdt-data-uploader/issues
+Project-URL: Source, https://github.com/yourorg/cdt-data-uploader
+Keywords: cdt data uploader sdk machine learning ai vla
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests (>=2.20.0)
+Requires-Dist: tos (>=2.5.0)
+Requires-Dist: attrs (>=21.0.0)
+Provides-Extra: dev
+Requires-Dist: pytest (>=7.0.0) ; extra == 'dev'
+Requires-Dist: pytest-cov (>=4.0.0) ; extra == 'dev'
+Requires-Dist: black (>=23.0.0) ; extra == 'dev'
+Requires-Dist: flake8 (>=6.0.0) ; extra == 'dev'
+Requires-Dist: mypy (>=1.0.0) ; extra == 'dev'
+
+# CDT Data Uploader SDK
+
+[![Python 3.6+](https://img.shields.io/badge/python-3.6+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+一个用于上传CDT数据集的Python SDK，支持VLA(Visual Language Action)等多种数据格式，提供并发上传、自动重试和失败恢复功能。
+
+## 特性
+
+- ✅ **并发上传** - 支持多线程并发上传,提高效率
+- ✅ **自动重试** - 上传失败时自动重试,支持可配置的重试次数和延迟
+- ✅ **失败恢复** - 记录失败的上传任务,支持后续重试
+- ✅ **灵活配置** - 支持多种配置参数,满足不同场景需求
+- ✅ **完整日志** - 提供详细的日志输出,便于调试和监控
+- ✅ **类型提示** - 完整的类型注解,提高代码可读性
+- ✅ **Python 3.6+兼容** - 使用attrs库兼容Python 3.6及以上版本
+
+## 安装
+
+```bash
+pip install cdt_data_uploader
+```
+
+## 依赖
+
+- Python 3.6+
+- `requests>=2.20.0` - HTTP请求库
+- `tos>=2.5.0` - 腾讯云对象存储SDK
+- `attrs>=21.0.0` - 数据类库(兼容Python 3.6)
+
+**兼容性:** Windows/Linux/macOS
+
+## 模块说明
+
+SDK包含以下数据上传模块：
+
+- **vla_uploader** - VLA (Visual Language Action) 数据上传
+- 更多数据格式支持即将推出...
+
+## 快速开始
+
+### 1. 导入SDK
+
+```python
+from vla_uploader import VLAUploader
+```
+
+### 2. 初始化上传器
+
+```python
+uploader = VLAUploader(
+    api_key="your_api_key",              # 从系统管理员获取
+    dataset_id="your_dataset_id",         # 数据集ID
+    api_base="https://nebula.core-dt.com/api/v1",  # API地址
+    workers=4                             # 并发线程数
+)
+```
+
+### 3. 上传数据
+
+#### 上传目录
+
+```python
+# 上传整个目录
+result = uploader.upload_directory("./dataset")
+
+# 查看结果
+print(f"总数: {result.total_rows}")
+print(f"成功: {result.success_count}")
+print(f"失败: {result.failure_count}")
+print(f"成功率: {result.success_rate:.2f}%")
+```
+
+#### 上传ZIP文件
+
+```python
+# 上传ZIP压缩包
+result = uploader.upload_zip(
+    zip_path="./dataset.zip",
+    workers=4
+)
+```
+
+#### 重试失败的上传
+
+```python
+# SDK自动记录失败的上传，可以重试
+result = uploader.retry_failed("./.vla_upload_work/run-xxx/failed_rows.json")
+```
+
+## 使用场景
+
+### 场景1: 首次上传VLA数据集
+
+```python
+from vla_uploader import VLAUploader
+
+# 1. 初始化
+uploader = VLAUploader(
+    api_key="your_api_key",
+    dataset_id="your_dataset_id"
+)
+
+# 2. 准备数据目录结构
+# dataset/
+# ├── episode_0001/
+# │   ├── data/
+# │   │   ├── video/front.mp4
+# │   │   ├── actions/actions.jsonl
+# │   │   └── depth/frame_000001.png
+# └── episode_0002/
+
+# 3. 上传
+result = uploader.upload_directory("./dataset")
+
+# 4. 检查结果
+if result.success_rate == 100.0:
+    print("✅ 所有数据上传成功!")
+else:
+    print(f"⚠️ 部分失败: {result.failure_count} 行")
+    for failed in result.failed_rows:
+        print(f"  失败: {failed.row_name} - {failed.error}")
+```
+
+### 场景2: 上传ZIP压缩包并记录源路径
+
+```python
+# 适用于数据集已经打包成ZIP的情况
+result = uploader.upload_zip(
+    zip_path="./my_dataset.zip",
+    source_path="D:/raw/my_dataset.zip",           # 原始路径
+    cloud_source_path="tos://bucket/raw/my_dataset.zip"  # 云端路径
+)
+```
+
+### 场景3: 并发上传大型数据集
+
+```python
+# 增加并发数以提高上传速度
+uploader = VLAUploader(
+    api_key="your_api_key",
+    dataset_id="your_dataset_id",
+    workers=8  # 8个并发线程
+)
+
+result = uploader.upload_directory("./large_dataset")
+```
+
+### 场景4: 重试失败的上传
+
+```python
+import glob
+
+# 查找最新的失败记录
+failed_files = glob.glob("./.vla_upload_work/run-*/failed_rows.json")
+if failed_files:
+    latest_failed = max(failed_files)
+    print(f"重试: {latest_failed}")
+
+    result = uploader.retry_failed(latest_failed)
+    print(f"重试结果: {result.success_count}/{result.total_rows}")
+```
+
+### 场景5: 批量处理多个数据集
+
+```python
+datasets = [
+    "./dataset1",
+    "./dataset2.zip",
+    "./dataset3"
+]
+
+for dataset_path in datasets:
+    print(f"\n处理: {dataset_path}")
+
+    try:
+        if dataset_path.endswith(".zip"):
+            result = uploader.upload_zip(dataset_path)
+        else:
+            result = uploader.upload_directory(dataset_path)
+
+        if result.success_rate == 100.0:
+            print(f"✅ 成功: {result.success_count} 行")
+        else:
+            print(f"⚠️ 部分失败: {result.failure_count} 行")
+
+    except Exception as e:
+        print(f"❌ 失败: {e}")
+```
+
+## API参考
+
+### 主要方法
+
+#### `upload_directory(data_path, workers=None, ...)`
+
+上传目录中的VLA数据
+
+**参数:**
+
+- `data_path` - 数据目录路径
+- `workers` - 并发线程数（默认3）
+- `source_path` - 原始路径（可选）
+- `cloud_source_path` - 云端路径（可选）
+
+**返回:** `BatchUploadResult` 对象
+
+#### `upload_zip(zip_path, workers=None, ...)`
+
+上传ZIP压缩包
+
+**参数:** 同上
+
+#### `retry_failed(failed_json_path)`
+
+重试失败的上传
+
+**参数:**
+
+- `failed_json_path` - 失败记录文件路径
+
+### 返回结果
+
+#### `BatchUploadResult`
+
+```python
+result.total_rows       # 总行数
+result.success_count    # 成功数量
+result.failure_count    # 失败数量
+result.success_rate     # 成功率(%)
+result.succeeded_rows   # 成功的行列表
+result.failed_rows      # 失败的行列表
+```
+
+#### `UploadResult`
+
+```python
+row.success        # 是否成功
+row.row_name       # 行名称
+row.datarow_id     # 数据行ID
+row.error          # 错误信息（如果失败）
+```
+
+## 数据格式要求
+
+SDK要求VLA数据集遵循以下目录结构:
+
+```
+dataset/
+├── episode_0001/
+│   ├── data/
+│   │   ├── video/
+│   │   │   └── front.mp4
+│   │   ├── actions/
+│   │   │   └── actions.jsonl
+│   │   ├── depth/
+│   │   │   └── frame_000001.png
+│   │   └── camera_params.json
+│   └── manifest.json (可选)
+├── episode_0002/
+│   └── ...
+```
+
+**必需文件:**
+
+- 每个episode必须包含至少一个视频文件
+- 视频文件位于 `data/video/` 或 `video/` 目录
+
+**支持的格式:**
+
+- 视频: `.mp4`, `.mov`, `.avi`, `.mkv`, `.webm`
+- 图像: `.jpg`, `.jpeg`, `.png`, `.bmp`, `.webp`, `.tif`, `.tiff`
+- 文本: `.json`, `.jsonl`
+
+## 配置参数
+
+SDK支持以下配置:
+
+```python
+uploader = VLAUploader(
+    api_key="...",                    # 必需：API密钥
+    dataset_id="...",                  # 必需：数据集ID
+
+    # 可选配置
+    api_base="https://...",           # API地址
+    workers=4,                         # 并发线程数(默认3)
+
+    # 重试配置
+    max_upload_attempts=5,            # 上传重试次数(默认5)
+    max_sync_attempts=3,              # 同步重试次数(默认3)
+
+    # 其他配置
+    media_type="video",                # 媒体类型(默认video)
+    dedupe_by_filename=True            # 按文件名去重(默认True)
+)
+```
+
+## 故障排查
+
+### 常见问题
+
+**Q: 上传失败怎么办？**
+
+```python
+# SDK自动保存失败记录到 .vla_upload_work/run-*/failed_rows.json
+# 可以重试:
+result = uploader.retry_failed("./.vla_upload_work/run-xxx/failed_rows.json")
+```
+
+**Q: 如何查看详细日志？**
+
+```python
+import logging
+logging.basicConfig(level=logging.DEBUG)
+```
+
+**Q: 如何调整并发数？**
+
+```python
+# 根据网络带宽和服务器性能调整
+uploader = VLAUploader(..., workers=8)  # 增加
+uploader = VLAUploader(..., workers=1)  # 减少
+```
+
+**Q: 如何获取API密钥？**
+联系系统管理员获取API密钥和数据集ID。
+
+### 错误处理
+
+```python
+from vla_uploader import VLAUploader, UploadError, CredentialError
+
+try:
+    result = uploader.upload_directory("./dataset")
+except CredentialError as e:
+    print(f"凭证错误: {e}")
+    print("请检查API密钥是否正确")
+except UploadError as e:
+    print(f"上传错误: {e}")
+    print("请检查网络连接和数据格式")
+```
+
+## 更多信息
+
+- **项目地址**: https://github.com/yourorg/cdt-data-uploader
+- **问题反馈**: https://github.com/yourorg/cdt-data-uploader/issues
+- **许可证**: MIT License
+
+## 版本历史
+
+- **v1.0.1** (2026-08-12): 修正包名，重新发版
+  - 统一包名为 cdt_data_uploader（下划线）
+  - 支持目录和ZIP文件上传
+  - 支持并发上传和自动重试
+  - 支持失败恢复
+  - Python 3.6+兼容
+- **v1.0.0** (2026-08-12): 首次发布（包名有误，已废弃）
+
+

cdt_data_uploader-1.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+vla_uploader/__init__.py,sha256=-q2NeWlaWobR0yY4CvPlJ9dWM3K_uLiPX_SDa_LRmUk,846
+vla_uploader/client.py,sha256=XvIuCimE4qb-48s8cooJUl-ExN_rZu_PbFhhBTcHP4Q,12610
+vla_uploader/constants.py,sha256=19A83u7aTfJB0xx4OojKlNwveL9Cr8vMWwCaGUbXtmE,1605
+vla_uploader/credentials.py,sha256=GD8Xb40r1lF7cbzS_u-hPwPkjSbcf9Sa6dAIrfDsmyY,4074
+vla_uploader/exceptions.py,sha256=B_2-ztnZrX6Yc4bWI--37xBOXa75EYIge2aFHRNCgFM,695
+vla_uploader/models.py,sha256=cGyL7ZH8kvZV4Wm6oZRNmhKfzV8mRrgu8wA8YsxTJac,2720
+vla_uploader/uploader.py,sha256=Y9rKutHUWflaHVxQg-bhNGJS_2GhkCzOJ4FhlY1ZMEI,12658
+vla_uploader/utils.py,sha256=_lf4eG43hqjsDrts0MR7m-0nJIH9mnIfcjD92DpNzQ0,6384
+cdt_data_uploader-1.0.1.dist-info/LICENSE,sha256=3COYSrCouiADl1N3hgE9ibjb33ldyBHBr9Rkq4vwxrE,1064
+cdt_data_uploader-1.0.1.dist-info/METADATA,sha256=p56ya7r3V4VjuemmstKKvghXooWM5IIVcQ4XWbONB6U,10176
+cdt_data_uploader-1.0.1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+cdt_data_uploader-1.0.1.dist-info/top_level.txt,sha256=t-9T9Mu6qTHs2jRYKb95Wd6WVuiFRUxX8tPIjC4a9r8,13
+cdt_data_uploader-1.0.1.dist-info/RECORD,,

cdt_data_uploader-1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.37.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cdt_data_uploader-1.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+vla_uploader

vla_uploader/__init__.py

@@ -0,0 +1,47 @@
+"""
+VLA Uploader SDK
+
+一个用于上传VLA(Visual Language Action)数据集的Python SDK
+"""
+
+__version__ = "1.0.1"
+__author__ = "VLA Team"
+
+from .client import VLAUploader
+from .models import (
+    UploaderConfig,
+    FileInfo,
+    UploadResult,
+    BatchUploadResult,
+    RetryInfo,
+)
+from .exceptions import (
+    VLAUploaderError,
+    ConfigurationError,
+    CredentialError,
+    UploadError,
+    SyncError,
+    ValidationError,
+    FileNotFoundError,
+    InvalidDataError,
+)
+
+__all__ = [
+    # 主客户端
+    "VLAUploader",
+    # 数据模型
+    "UploaderConfig",
+    "FileInfo",
+    "UploadResult",
+    "BatchUploadResult",
+    "RetryInfo",
+    # 异常
+    "VLAUploaderError",
+    "ConfigurationError",
+    "CredentialError",
+    "UploadError",
+    "SyncError",
+    "ValidationError",
+    "FileNotFoundError",
+    "InvalidDataError",
+]