dexverse-cli 0.1.1__tar.gz → 0.1.3.post1__tar.gz

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dexverse-cli
-Version: 0.1.1
+Version: 0.1.3.post1
 Summary: Download, pack, and unpack DexVerse datasets
 Requires-Python: >=3.10
 Requires-Dist: av<17,>=14
@@ -49,6 +49,15 @@ corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.
 
 Downloads use bounded parallelism, retries, and compatible partial-file resume.
+Within a manifest, the first occurrence of a target path keeps its name; later
+occurrences receive `-1`, `-2`, etc. before the final extension (for example,
+`episode1.hdf5`, `episode1-1.hdf5`, `episode1-2.hdf5`). Numbering skips names and
+paths already claimed by other entries, including partial files and metadata.
+Different relative directories remain independent. Names are assigned in manifest
+order, with Windows case-insensitive matching, before any downloads start.
+Rerunning the same manifest preserves the assigned names for skipping and resume;
+existing files on disk do not trigger new suffixes. This is not cross-batch name
+allocation: changing the manifest entries or their order can change the mapping.
 When `sha256` is present it is always checked before the final file is replaced.
 TLS certificate verification is disabled by default for internal endpoints;
 pass `--secure` to enable certificate verification.

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dexverse-cli"
-version = "0.1.1"
+version = "0.1.3.post1"
 description = "Download, pack, and unpack DexVerse datasets"
 readme = "src/dexverse_cli/README.md"
 requires-python = ">=3.10"

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/src/dexverse_cli/README.md

@@ -33,6 +33,15 @@ corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.
 
 Downloads use bounded parallelism, retries, and compatible partial-file resume.
+Within a manifest, the first occurrence of a target path keeps its name; later
+occurrences receive `-1`, `-2`, etc. before the final extension (for example,
+`episode1.hdf5`, `episode1-1.hdf5`, `episode1-2.hdf5`). Numbering skips names and
+paths already claimed by other entries, including partial files and metadata.
+Different relative directories remain independent. Names are assigned in manifest
+order, with Windows case-insensitive matching, before any downloads start.
+Rerunning the same manifest preserves the assigned names for skipping and resume;
+existing files on disk do not trigger new suffixes. This is not cross-batch name
+allocation: changing the manifest entries or their order can change the mapping.
 When `sha256` is present it is always checked before the final file is replaced.
 TLS certificate verification is disabled by default for internal endpoints;
 pass `--secure` to enable certificate verification.

dexverse_cli-0.1.3.post1/src/dexverse_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.3.post1"

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/src/dexverse_cli/cli.py

@@ -87,14 +87,21 @@ def download(
                 "warning: TLS certificate verification is disabled; pass --secure to enable it",
                 err=True,
             )
-        report = download_files(
-            files,
-            out,
-            parallel=parallel,
-            secure=secure,
-            read_timeout=http_read_timeout,
-            max_retries=http_retries,
-        )
+        with batch_progress(show_detail=True) as progress:
+            task_id = progress.add_task(
+                f"Downloading 0/{len(files)}",
+                total=len(files),
+                detail=f"{len(files)} files → {out}",
+            )
+            report = download_files(
+                files,
+                out,
+                parallel=parallel,
+                secure=secure,
+                read_timeout=http_read_timeout,
+                max_retries=http_retries,
+                on_progress=progress_callback(progress, task_id, "Downloading"),
+            )
     except (ManifestError, OSError) as exc:
         typer.echo(f"error: {exc}", err=True)
         raise typer.Exit(code=1) from exc

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/src/dexverse_cli/download.py

@@ -9,6 +9,7 @@ from dataclasses import dataclass
 from functools import partial
 from hashlib import sha256
 from pathlib import Path
+from threading import Lock
 
 from dexverse_cli.http import (
     HttpClientProvider,
@@ -26,6 +27,8 @@ from dexverse_cli.state import (
     write_file_metadata,
 )
 
+ProgressCallback = Callable[[int, int], None]
+
 
 @dataclass(frozen=True, slots=True)
 class DownloadReport:
@@ -74,7 +77,6 @@ class OutputRootPlan:
             ):
                 raise UnsafeOutputPathError("unsafe output path")
         check_claim(path, self.root)
-        reject_existing_aliases(self.claims)
 
 
 def safe_lstat(path: Path) -> os.stat_result | None:
@@ -140,24 +142,19 @@ def check_claim(path: Path, root: Path) -> None:
 
 def reject_existing_aliases(claims: tuple[Path, ...]) -> None:
     resolved: set[str] = set()
-    existing: list[Path] = []
+    identities: set[tuple[int, int]] = set()
     for claim in claims:
         key = os.path.normcase(os.path.normpath(str(claim.resolve(strict=False))))
         if key in resolved:
             raise UnsafeOutputPathError("unsafe output path")
         resolved.add(key)
-        if safe_lstat(claim) is not None:
-            for other in existing:
-                try:
-                    if os.path.samefile(claim, other):
-                        raise UnsafeOutputPathError("unsafe output path")
-                except FileNotFoundError:
-                    continue
-                except UnsafeOutputPathError:
-                    raise
-                except OSError as exc:
-                    raise UnsafeOutputPathError("unsafe output path") from exc
-            existing.append(claim)
+        value = safe_lstat(claim)
+        if value is None:
+            continue
+        identity = value.st_dev, value.st_ino
+        if identity in identities:
+            raise UnsafeOutputPathError("unsafe output path")
+        identities.add(identity)
 
 
 def build_output_plan(files: tuple[ManifestFile, ...], out: Path) -> OutputRootPlan:
@@ -295,17 +292,32 @@ def download_files(
     secure: bool,
     read_timeout: float,
     max_retries: int,
+    on_progress: ProgressCallback | None = None,
 ) -> DownloadReport:
     output = build_output_plan(files, out)
     if not output.files:
         return DownloadReport(downloaded=0, skipped=0, failures=())
+    if on_progress is not None:
+        on_progress(0, len(output.files))
     client_provider = HttpClientProvider(parallel=parallel, secure=secure, read_timeout=read_timeout)
-    worker = partial(
+    download_worker = partial(
         download_one,
         output=output,
         client_provider=client_provider,
         max_retries=max_retries,
     )
+    completed = 0
+    progress_lock = Lock()
+
+    def worker(file: FileOutputPlan) -> tuple[str, str | None]:
+        nonlocal completed
+        result = download_worker(file)
+        if on_progress is not None:
+            with progress_lock:
+                completed += 1
+                on_progress(completed, len(output.files))
+        return result
+
     try:
         with ThreadPoolExecutor(max_workers=parallel) as executor:
             results = tuple(executor.map(worker, output.files))

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/src/dexverse_cli/manifest.py

@@ -4,7 +4,7 @@ import json
 import os
 import re
 import sys
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlsplit, urlunsplit
@@ -199,18 +199,17 @@ def parse_manifest(text: str) -> tuple[ManifestFile, ...]:
             raise ManifestError(f"files[{index}].file_name must be a non-empty string")
         safe_name = safe_file_name(file_name, index=index)
         duplicate_key = os.path.normcase(safe_name)
-        if duplicate_key in destinations:
-            raise ManifestError(f"files[{index}].file_name resolves to a duplicate target")
-        destinations.add(duplicate_key)
         final_claim = claim_parts(safe_name)
         part_claim = (*final_claim[:-1], f"{final_claim[-1]}.part")
         instruction_claim = (*final_claim[:-1], f"{final_claim[-1]}.json")
-        if any(
-            paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
-        ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
-            raise ManifestError(f"files[{index}].file_name creates an ambiguous target collision")
-        output_claims.extend((final_claim, part_claim))
-        metadata_claims.append(instruction_claim)
+        if duplicate_key not in destinations:
+            if any(
+                paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
+            ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
+                raise ManifestError(f"files[{index}].file_name creates an ambiguous target collision")
+            destinations.add(duplicate_key)
+            output_claims.extend((final_claim, part_claim))
+            metadata_claims.append(instruction_claim)
         files.append(
             ManifestFile(
                 download_url=download_url,
@@ -225,6 +224,31 @@ def parse_manifest(text: str) -> tuple[ManifestFile, ...]:
                 meta_data=metadata(item, index=index),
             )
         )
+    # Reserve every original name before allocating suffixes, independent of disk
+    # contents and worker order so the same manifest can resume at the same paths.
+    next_suffix: dict[str, int] = {}
+    for index, file in enumerate(files):
+        key = os.path.normcase(file.file_name)
+        if key not in next_suffix:
+            next_suffix[key] = 1
+            continue
+        stem, extension = os.path.splitext(file.file_name)
+        suffix = next_suffix[key]
+        while True:
+            candidate = f"{stem}-{suffix}{extension}"
+            suffix += 1
+            final_claim = claim_parts(candidate)
+            part_claim = (*final_claim[:-1], f"{final_claim[-1]}.part")
+            instruction_claim = (*final_claim[:-1], f"{final_claim[-1]}.json")
+            if any(
+                paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
+            ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
+                continue
+            output_claims.extend((final_claim, part_claim))
+            metadata_claims.append(instruction_claim)
+            files[index] = replace(file, file_name=candidate)
+            next_suffix[key] = suffix
+            break
     return tuple(files)
 
 

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3.post1}/uv.lock

@@ -108,7 +108,7 @@ wheels = [
 
 [[package]]
 name = "dexverse-cli"
-version = "0.1.1"
+version = "0.1.3.post1"
 source = { editable = "." }
 dependencies = [
     { name = "av" },

dexverse_cli-0.1.1/src/dexverse_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.1"