dexverse-cli 0.1.1__tar.gz → 0.1.3__tar.gz

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dexverse-cli
-Version: 0.1.1
+Version: 0.1.3
 Summary: Download, pack, and unpack DexVerse datasets
 Requires-Python: >=3.10
 Requires-Dist: av<17,>=14
@@ -49,6 +49,15 @@ corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.
 
 Downloads use bounded parallelism, retries, and compatible partial-file resume.
+Within a manifest, the first occurrence of a target path keeps its name; later
+occurrences receive `-1`, `-2`, etc. before the final extension (for example,
+`episode1.hdf5`, `episode1-1.hdf5`, `episode1-2.hdf5`). Numbering skips names and
+paths already claimed by other entries, including partial files and metadata.
+Different relative directories remain independent. Names are assigned in manifest
+order, with Windows case-insensitive matching, before any downloads start.
+Rerunning the same manifest preserves the assigned names for skipping and resume;
+existing files on disk do not trigger new suffixes. This is not cross-batch name
+allocation: changing the manifest entries or their order can change the mapping.
 When `sha256` is present it is always checked before the final file is replaced.
 TLS certificate verification is disabled by default for internal endpoints;
 pass `--secure` to enable certificate verification.

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dexverse-cli"
-version = "0.1.1"
+version = "0.1.3"
 description = "Download, pack, and unpack DexVerse datasets"
 readme = "src/dexverse_cli/README.md"
 requires-python = ">=3.10"

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/src/dexverse_cli/README.md

@@ -33,6 +33,15 @@ corresponding HDF5 file is unpacked; only top-level `data_protocol` is removed
 from that algorithm-facing projection.
 
 Downloads use bounded parallelism, retries, and compatible partial-file resume.
+Within a manifest, the first occurrence of a target path keeps its name; later
+occurrences receive `-1`, `-2`, etc. before the final extension (for example,
+`episode1.hdf5`, `episode1-1.hdf5`, `episode1-2.hdf5`). Numbering skips names and
+paths already claimed by other entries, including partial files and metadata.
+Different relative directories remain independent. Names are assigned in manifest
+order, with Windows case-insensitive matching, before any downloads start.
+Rerunning the same manifest preserves the assigned names for skipping and resume;
+existing files on disk do not trigger new suffixes. This is not cross-batch name
+allocation: changing the manifest entries or their order can change the mapping.
 When `sha256` is present it is always checked before the final file is replaced.
 TLS certificate verification is disabled by default for internal endpoints;
 pass `--secure` to enable certificate verification.

dexverse_cli-0.1.3/src/dexverse_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.3"

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/src/dexverse_cli/download.py

@@ -74,7 +74,6 @@ class OutputRootPlan:
             ):
                 raise UnsafeOutputPathError("unsafe output path")
         check_claim(path, self.root)
-        reject_existing_aliases(self.claims)
 
 
 def safe_lstat(path: Path) -> os.stat_result | None:
@@ -140,24 +139,19 @@ def check_claim(path: Path, root: Path) -> None:
 
 def reject_existing_aliases(claims: tuple[Path, ...]) -> None:
     resolved: set[str] = set()
-    existing: list[Path] = []
+    identities: set[tuple[int, int]] = set()
     for claim in claims:
         key = os.path.normcase(os.path.normpath(str(claim.resolve(strict=False))))
         if key in resolved:
             raise UnsafeOutputPathError("unsafe output path")
         resolved.add(key)
-        if safe_lstat(claim) is not None:
-            for other in existing:
-                try:
-                    if os.path.samefile(claim, other):
-                        raise UnsafeOutputPathError("unsafe output path")
-                except FileNotFoundError:
-                    continue
-                except UnsafeOutputPathError:
-                    raise
-                except OSError as exc:
-                    raise UnsafeOutputPathError("unsafe output path") from exc
-            existing.append(claim)
+        value = safe_lstat(claim)
+        if value is None:
+            continue
+        identity = value.st_dev, value.st_ino
+        if identity in identities:
+            raise UnsafeOutputPathError("unsafe output path")
+        identities.add(identity)
 
 
 def build_output_plan(files: tuple[ManifestFile, ...], out: Path) -> OutputRootPlan:

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/src/dexverse_cli/manifest.py

@@ -4,7 +4,7 @@ import json
 import os
 import re
 import sys
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlsplit, urlunsplit
@@ -199,18 +199,17 @@ def parse_manifest(text: str) -> tuple[ManifestFile, ...]:
             raise ManifestError(f"files[{index}].file_name must be a non-empty string")
         safe_name = safe_file_name(file_name, index=index)
         duplicate_key = os.path.normcase(safe_name)
-        if duplicate_key in destinations:
-            raise ManifestError(f"files[{index}].file_name resolves to a duplicate target")
-        destinations.add(duplicate_key)
         final_claim = claim_parts(safe_name)
         part_claim = (*final_claim[:-1], f"{final_claim[-1]}.part")
         instruction_claim = (*final_claim[:-1], f"{final_claim[-1]}.json")
-        if any(
-            paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
-        ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
-            raise ManifestError(f"files[{index}].file_name creates an ambiguous target collision")
-        output_claims.extend((final_claim, part_claim))
-        metadata_claims.append(instruction_claim)
+        if duplicate_key not in destinations:
+            if any(
+                paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
+            ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
+                raise ManifestError(f"files[{index}].file_name creates an ambiguous target collision")
+            destinations.add(duplicate_key)
+            output_claims.extend((final_claim, part_claim))
+            metadata_claims.append(instruction_claim)
         files.append(
             ManifestFile(
                 download_url=download_url,
@@ -225,6 +224,31 @@ def parse_manifest(text: str) -> tuple[ManifestFile, ...]:
                 meta_data=metadata(item, index=index),
             )
         )
+    # Reserve every original name before allocating suffixes, independent of disk
+    # contents and worker order so the same manifest can resume at the same paths.
+    next_suffix: dict[str, int] = {}
+    for index, file in enumerate(files):
+        key = os.path.normcase(file.file_name)
+        if key not in next_suffix:
+            next_suffix[key] = 1
+            continue
+        stem, extension = os.path.splitext(file.file_name)
+        suffix = next_suffix[key]
+        while True:
+            candidate = f"{stem}-{suffix}{extension}"
+            suffix += 1
+            final_claim = claim_parts(candidate)
+            part_claim = (*final_claim[:-1], f"{final_claim[-1]}.part")
+            instruction_claim = (*final_claim[:-1], f"{final_claim[-1]}.json")
+            if any(
+                paths_conflict(claim, existing) for claim in (final_claim, part_claim) for existing in output_claims
+            ) or any(paths_conflict(instruction_claim, existing) for existing in metadata_claims):
+                continue
+            output_claims.extend((final_claim, part_claim))
+            metadata_claims.append(instruction_claim)
+            files[index] = replace(file, file_name=candidate)
+            next_suffix[key] = suffix
+            break
     return tuple(files)
 
 

{dexverse_cli-0.1.1 → dexverse_cli-0.1.3}/uv.lock

@@ -108,7 +108,7 @@ wheels = [
 
 [[package]]
 name = "dexverse-cli"
-version = "0.1.1"
+version = "0.1.3"
 source = { editable = "." }
 dependencies = [
     { name = "av" },

dexverse_cli-0.1.1/src/dexverse_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.1"