trossen-cloud-cli 0.1.2__tar.gz → 1.2.0__tar.gz

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/.github/workflows/publish.yml

@@ -17,7 +17,24 @@ on:
         default: testpypi
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.release_tag }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+
+      - name: Install dependencies
+        run: uv sync --frozen --extra dev
+
+      - name: Run checks
+        run: make check
+
   build:
+    needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trossen_cloud_cli
-Version: 0.1.2
+Version: 1.2.0
 Summary: CLI for interacting with Trossen Cloud APIs
 Project-URL: Homepage, https://github.com/TrossenRobotics/trossen_cloud_cli
 Project-URL: Repository, https://github.com/TrossenRobotics/trossen_cloud_cli

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "trossen_cloud_cli"
-version = "0.1.2"
+version = "1.2.0"
 description = "CLI for interacting with Trossen Cloud APIs"
 readme = "README.md"
 requires-python = ">=3.11"

trossen_cloud_cli-1.2.0/scripts/release.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# Cut a release: bump pyproject.toml, commit, tag, push, create a GitHub Release.
+# Prints the command for triggering publish.yml manually after the release is up.
+#
+# Usage: scripts/release.sh v<MAJOR.MINOR.PATCH> [--dry-run]
+
+set -euo pipefail
+
+INPUT="${1:?usage: scripts/release.sh v<MAJOR.MINOR.PATCH> [--dry-run]}"
+DRY_RUN=false
+[[ "${2:-}" == "--dry-run" ]] && DRY_RUN=true
+
+# Strict vX.Y.Z only — no pre-release suffixes.
+[[ "$INPUT" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] \
+  || { echo "error: version must be vX.Y.Z (got '$INPUT')"; exit 1; }
+TAG="$INPUT"
+VERSION="${INPUT#v}"  # strip 'v' for pyproject.toml (PEP 440 disallows it).
+
+# Pre-flight: clean tree, on main, in sync, tag doesn't exist, version changed.
+[[ -z "$(git status --porcelain)" ]] || { echo "error: working tree dirty"; exit 1; }
+[[ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]] || { echo "error: not on main"; exit 1; }
+git fetch origin main --tags --quiet
+[[ "$(git rev-parse HEAD)" == "$(git rev-parse origin/main)" ]] \
+  || { echo "error: local main not in sync with origin/main"; exit 1; }
+if git rev-parse "$TAG" >/dev/null 2>&1; then
+  echo "error: tag $TAG already exists"; exit 1
+fi
+
+CURRENT="$(grep -E '^version = ' pyproject.toml | head -1 | sed -E 's/version = "(.*)"/\1/')"
+[[ "$CURRENT" != "$VERSION" ]] || { echo "error: pyproject.toml already at $VERSION"; exit 1; }
+
+echo "Running make check..."
+make check
+
+# Preview release notes before committing anything irreversible.
+PREV_TAG="$(git tag --list 'v*' --sort=version:refname | tail -1)"
+NOTES_ARGS=(-f tag_name="$TAG" -f target_commitish="$(git rev-parse HEAD)")
+[[ -n "$PREV_TAG" ]] && NOTES_ARGS+=(-f previous_tag_name="$PREV_TAG")
+
+echo
+echo "=== Auto-generated release notes preview ==="
+gh api repos/{owner}/{repo}/releases/generate-notes "${NOTES_ARGS[@]}" --jq .body
+echo "============================================"
+echo
+
+if $DRY_RUN; then
+  echo "Dry run: stopping before any changes."
+  exit 0
+fi
+
+read -rp "Proceed with release $TAG? [y/N] " ans
+[[ "$ans" == "y" || "$ans" == "Y" ]] || { echo "aborted"; exit 1; }
+
+# Bump, commit, tag, push.
+sed -i.bak -E "s/^version = \".*\"/version = \"$VERSION\"/" pyproject.toml
+rm pyproject.toml.bak
+git add pyproject.toml
+git commit -m "Release $TAG"
+git tag -a "$TAG" -m "Release $TAG"
+git push origin main "$TAG"
+
+# Create the GitHub Release with auto-generated notes (the same ones previewed above).
+gh release create "$TAG" --title "$TAG" --generate-notes
+
+echo
+echo "Release $TAG created. Smoke-test on testpypi first, then publish to pypi:"
+echo "  gh workflow run publish.yml -f release_tag=$TAG -f index=testpypi"
+echo "  gh workflow run publish.yml -f release_tag=$TAG -f index=pypi"
+echo "Or run the workflow from the GitHub Actions UI."

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/__init__.py

@@ -1,6 +1,8 @@
 """Trossen CLI - A Python CLI for Trossen Cloud."""
 
-__version__ = "0.1.0"
+from importlib.metadata import version
+
+__version__ = version("trossen_cloud_cli")
 
 from .cli import app
 

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/commands/datasets.py

@@ -14,9 +14,10 @@ from rich.table import Table
 from ..api_client import ApiClient, ApiError
 from ..auth import require_auth
 from ..download import download_dataset
-from ..output import console, print_error, print_info, print_success
+from ..output import console, print_error, print_info, print_success, print_warning
 from ..types import DatasetType, PrivacyLevel
 from ..upload import UploadError, create_and_upload_dataset
+from ..validators import validate_dataset
 
 app = typer.Typer(help="Manage datasets")
 
@@ -57,7 +58,7 @@ def upload(
     dataset_type: Annotated[
         DatasetType,
         typer.Option("--type", "-t", help="Dataset type"),
-    ] = DatasetType.RAW,
+    ],
     privacy: Annotated[
         PrivacyLevel,
         typer.Option("--privacy", "-p", help="Privacy level"),
@@ -66,6 +67,10 @@ def upload(
         str | None,
         typer.Option("--metadata", "-m", help="JSON metadata string"),
     ] = None,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Skip validation confirmation prompt"),
+    ] = False,
 ) -> None:
     """
     Upload a dataset to Trossen Cloud.
@@ -81,6 +86,18 @@ def upload(
             print_error("Invalid JSON metadata")
             raise typer.Exit(1)
 
+    # Validate dataset before upload
+    validation_warnings = validate_dataset(path, dataset_type)
+    if validation_warnings:
+        console.print(
+            f"\n[warning]Found {len(validation_warnings)} validation warning(s):[/warning]"
+        )
+        for w in validation_warnings:
+            print_warning(w)
+        console.print()
+        if not force and not typer.confirm("Continue with upload?"):
+            raise typer.Exit(0)
+
     try:
         dataset = asyncio.run(
             create_and_upload_dataset(
@@ -152,6 +169,10 @@ def import_hf(
         bool,
         typer.Option("--keep-local", help="Keep the downloaded files after upload"),
     ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Skip validation confirmation prompt"),
+    ] = False,
 ) -> None:
     """
     Import a public HuggingFace dataset into Trossen Cloud.
@@ -201,6 +222,18 @@ def import_hf(
 
         print_success(f"Downloaded to {local_path}")
 
+        # Validate dataset before upload
+        validation_warnings = validate_dataset(local_path, dataset_type)
+        if validation_warnings:
+            console.print(
+                f"\n[warning]Found {len(validation_warnings)} validation warning(s):[/warning]"
+            )
+            for w in validation_warnings:
+                print_warning(w)
+            console.print()
+            if not force and not typer.confirm("Continue with upload?"):
+                raise typer.Exit(0)
+
         # Upload to Trossen Cloud
         dataset = asyncio.run(
             create_and_upload_dataset(

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/config.py

@@ -14,8 +14,8 @@ class UploadConfig(BaseModel):
     """
 
     chunk_size_mb: int = 50
-    parallel_parts: int = 6
-    parallel_files: int = 32
+    parallel_parts: int = 8
+    parallel_files: int = 64
 
 
 class DownloadConfig(BaseModel):

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/download.py

@@ -114,7 +114,10 @@ async def download_resource(
 
         async with httpx.AsyncClient(
             timeout=httpx.Timeout(300.0, connect=30.0),
-            limits=httpx.Limits(max_connections=config.download.parallel_files),
+            limits=httpx.Limits(
+                max_connections=config.download.parallel_files,
+                max_keepalive_connections=config.download.parallel_files,
+            ),
         ) as download_client:
             if show_progress:
                 total_size = sum(f["size_bytes"] for f in raw_files)

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/types.py

@@ -12,8 +12,6 @@ class DatasetType(StrEnum):
 
     MCAP = "mcap"
     LEROBOT = "lerobot"
-    TROSSEN = "trossen"
-    RAW = "raw"
 
 
 class PrivacyLevel(StrEnum):

{trossen_cloud_cli-0.1.2 → trossen_cloud_cli-1.2.0}/src/trossen_cloud_cli/upload.py

@@ -135,6 +135,7 @@ UPLOAD_MAX_RETRIES = 5
 STREAM_CHUNK_SIZE = 256 * 1024  # 256 KB chunks for streaming progress
 BATCH_CHUNK_SIZE = 500  # Max files per batch API call
 STATE_SAVE_INTERVAL = 10  # Save state every N part completions
+MAX_UPLOAD_CONNECTIONS = 128  # Safety cap on the upload connection pool
 
 
 async def upload_part(
@@ -167,39 +168,53 @@ async def upload_part(
     offset = (part_number - 1) * part_size
     chunk_size = min(part_size, file_size - offset)
 
-    def _read_data() -> bytes:
-        with open(file_path, "rb") as f:
-            f.seek(offset)
-            return f.read(chunk_size)
-
     last_error: Exception | None = None
     for attempt in range(UPLOAD_MAX_RETRIES):
-        data = _read_data()
+        # Re-check on every attempt: the file could be truncated/replaced between attempts,
+        # invalidating chunk_size and Content-Length.
+        current_size = file_path.stat().st_size
+        if current_size < offset + chunk_size:
+            raise UploadError(
+                f"file {file_path} truncated mid-upload "
+                f"(expected at least {offset + chunk_size} bytes, found {current_size})"
+            )
+
         bytes_sent_this_attempt = 0
 
         async def _streaming_body():
             """
-            Async generator that yields chunks and tracks bytes sent.
+            Stream the part body from disk in small chunks to bound memory.
             """
             nonlocal bytes_sent_this_attempt
-            sent = 0
-            while sent < len(data):
-                end = min(sent + STREAM_CHUNK_SIZE, len(data))
-                yield data[sent:end]
-                chunk_len = end - sent
-                if progress:
-                    progress.advance_file(filename, chunk_len)
-                bytes_sent_this_attempt += chunk_len
-                sent = end
+            with open(file_path, "rb") as f:
+                f.seek(offset)
+                remaining = chunk_size
+                while remaining > 0:
+                    buf = f.read(min(STREAM_CHUNK_SIZE, remaining))
+                    if not buf:
+                        # Truncated mid-stream after the pre-flight check passed.
+                        raise UploadError(
+                            f"file {file_path} truncated during upload ({remaining} bytes short)"
+                        )
+                    yield buf
+                    if progress:
+                        progress.advance_file(filename, len(buf))
+                    bytes_sent_this_attempt += len(buf)
+                    remaining -= len(buf)
 
         try:
             response = await upload_client.put(
                 upload_url,
                 content=_streaming_body(),
-                headers={"Content-Length": str(len(data))},
+                headers={"Content-Length": str(chunk_size)},
             )
             response.raise_for_status()
             return response.headers.get("ETag", "")
+        except UploadError:
+            # Truncation isn't recoverable; rewind progress and bail without retrying.
+            if progress and bytes_sent_this_attempt > 0:
+                progress.advance_file(filename, -bytes_sent_this_attempt)
+            raise
         except (httpx.ConnectError, httpx.ConnectTimeout, httpx.HTTPStatusError) as e:
             if isinstance(e, httpx.HTTPStatusError) and e.response.status_code < 500:
                 raise
@@ -217,6 +232,7 @@ async def _upload_file_parts(
     upload_client: httpx.AsyncClient,
     file_path: str,
     local_path: Path,
+    file_size: int,
     part_urls: dict[int, str],
     part_size: int,
     progress: TransferProgress | None = None,
@@ -228,6 +244,9 @@ async def _upload_file_parts(
     :param upload_client: Shared async HTTP client for storage requests.
     :param file_path: The relative path of the file within the resource.
     :param local_path: The local filesystem path to the file.
+    :param file_size: The expected size in bytes (from the FileInfo captured at collection
+        time, used to generate the presigned URLs). The pre-flight truncation check inside
+        upload_part compares against this declared size, not a fresh stat.
     :param part_urls: Mapping of part_number -> presigned URL.
     :param part_size: Size of each part in bytes.
     :param progress: Optional progress tracker for UI updates.
@@ -235,7 +254,6 @@ async def _upload_file_parts(
 
     """
     config = get_config()
-    file_size = local_path.stat().st_size
     semaphore = asyncio.Semaphore(config.upload.parallel_parts)
 
     # Filter out parts that have already been uploaded (resume support)
@@ -395,10 +413,15 @@ async def upload_resource(
     file_sem = asyncio.Semaphore(config.upload.parallel_files)
     failed_files: list[str] = []
 
+    # Cap concurrent connections to avoid overwhelming the system
+    max_conns = min(
+        config.upload.parallel_files * config.upload.parallel_parts, MAX_UPLOAD_CONNECTIONS
+    )
     async with httpx.AsyncClient(
         timeout=httpx.Timeout(300.0),
         limits=httpx.Limits(
-            max_connections=config.upload.parallel_files * config.upload.parallel_parts
+            max_connections=max_conns,
+            max_keepalive_connections=max_conns,
         ),
     ) as upload_client:
 
@@ -437,6 +460,7 @@ async def upload_resource(
                             upload_client,
                             fp,
                             actual_path,
+                            file_info.size_bytes,
                             part_urls_map,
                             part_size,
                             progress,

trossen_cloud_cli-1.2.0/src/trossen_cloud_cli/validators/__init__.py

@@ -0,0 +1,26 @@
+"""Dataset validators for pre-upload structural checks."""
+
+from pathlib import Path
+
+from ..types import DatasetType
+from .lerobot import validate_lerobot
+from .mcap import validate_mcap
+
+
+def validate_dataset(path: Path, dataset_type: DatasetType) -> list[str]:
+    """
+    Validate a dataset directory against its type-specific spec.
+
+    Returns a list of warning messages. An empty list means no issues found.
+    Only runs for dataset types that have a validator (mcap, lerobot).
+    """
+    validators = {
+        DatasetType.MCAP: validate_mcap,
+        DatasetType.LEROBOT: validate_lerobot,
+    }
+
+    validator = validators.get(dataset_type)
+    if validator is None:
+        return []
+
+    return validator(path)