hafnia 0.4.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

hafnia/dataset/operations/dataset_s3_storage.py

@@ -0,0 +1,211 @@
+import tempfile
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+import polars as pl
+
+from hafnia.dataset.dataset_helpers import hash_file_xxhash
+from hafnia.dataset.dataset_names import (
+    DatasetVariant,
+    ResourceCredentials,
+    SampleField,
+)
+from hafnia.dataset.hafnia_dataset import HafniaDataset
+from hafnia.log import user_logger
+from hafnia.platform import s5cmd_utils
+from hafnia.platform.datasets import get_upload_credentials
+from hafnia.utils import progress_bar
+from hafnia_cli.config import Config
+
+
+def delete_hafnia_dataset_files_on_platform(
+    dataset_name: str,
+    interactive: bool = True,
+    cfg: Optional[Config] = None,
+) -> bool:
+    cfg = cfg or Config()
+    resource_credentials = get_upload_credentials(dataset_name, cfg=cfg)
+
+    if resource_credentials is None:
+        raise RuntimeError("Failed to get upload credentials from the platform.")
+
+    return delete_hafnia_dataset_files_from_resource_credentials(
+        interactive=interactive,
+        resource_credentials=resource_credentials,
+    )
+
+
+def delete_hafnia_dataset_files_from_resource_credentials(
+    resource_credentials: ResourceCredentials,
+    interactive: bool = True,
+) -> bool:
+    envs = resource_credentials.aws_credentials()
+    bucket_name = resource_credentials.bucket_name()
+    if interactive:
+        confirmation = (
+            input(
+                f"WARNING THIS WILL delete all files stored in 's3://{bucket_name}'.\n"
+                "Meaning that all previous versions of the dataset will be deleted. \n"
+                "Normally this is not needed, but if you have changed the dataset structure or want to start from fresh, "
+                "you can delete all files in the S3 bucket. "
+                "\nDo you really want to delete all files? (yes/NO): "
+            )
+            .strip()
+            .lower()
+        )
+        if confirmation != "yes":
+            user_logger.info("Delete operation cancelled by the user.")
+            return False
+    user_logger.info(f"Deleting all files in S3 bucket '{bucket_name}'...")
+    s5cmd_utils.delete_bucket_content(bucket_prefix=f"s3://{bucket_name}", remove_bucket=True, append_envs=envs)
+    return True
+
+
+def sync_hafnia_dataset_to_s3(
+    dataset: HafniaDataset,
+    bucket_prefix: str,
+    allow_version_overwrite: bool = False,
+    interactive: bool = True,
+    envs: Optional[Dict[str, str]] = None,
+) -> None:
+    t0 = time.time()
+    # bucket_prefix e.g. 's3://bucket-name/sample'
+    remote_paths = []
+    for file_str in progress_bar(dataset.samples[SampleField.FILE_PATH], description="Hashing data files"):
+        path_file = Path(file_str)
+        file_hash = hash_file_xxhash(path_file)
+
+        # Relative path in S3 bucket e.g. 'data/e2/b0/e2b000ac47b19a999bee5456a6addb88.png'
+        relative_path = s3_prefix_from_hash(hash=file_hash, suffix=path_file.suffix)
+
+        # Remote path in S3 bucket e.g. 's3://bucket-name/sample/data/e2/b0/e2b000ac47b19a999bee5456a6addb88.png'
+        remote_path = f"{bucket_prefix}/{relative_path}"
+        remote_paths.append(remote_path)
+
+    dataset.samples = dataset.samples.with_columns(pl.Series(remote_paths).alias(SampleField.REMOTE_PATH))
+
+    user_logger.info(f"Syncing dataset to S3 bucket '{bucket_prefix}'")
+    files_in_s3 = set(s5cmd_utils.list_bucket(bucket_prefix=bucket_prefix, append_envs=envs))
+
+    # Discover data files (images, videos, etc.) missing in s3
+    data_files_missing = dataset.samples.filter(~pl.col(SampleField.REMOTE_PATH).is_in(files_in_s3))
+    files_already_in_s3 = dataset.samples.filter(pl.col(SampleField.REMOTE_PATH).is_in(files_in_s3))
+
+    with tempfile.TemporaryDirectory() as temp_dir:  # Temp folder to store metadata files
+        path_temp = Path(temp_dir)
+        # File paths are dropped when uploading to S3
+        dataset = dataset.update_samples(dataset.samples.drop(SampleField.FILE_PATH))
+        dataset.write_annotations(path_temp)
+
+        # Discover versioned metadata files (e.g. "annotations.jsonl", "dataset_info.json") missing in s3
+        metadata_files_local = []
+        metadata_files_s3 = []
+        for filename in path_temp.iterdir():
+            metadata_files_s3.append(f"{bucket_prefix}/versions/{dataset.info.version}/{filename.name}")
+            metadata_files_local.append(filename.as_posix())
+
+        overwrite_metadata_files = files_in_s3.intersection(set(metadata_files_s3))
+        will_overwrite_metadata_files = len(overwrite_metadata_files) > 0
+
+        n_files_already_in_s3 = len(files_already_in_s3)
+        user_logger.info(f"Sync dataset to {bucket_prefix}")
+        user_logger.info(
+            f"- Found that {n_files_already_in_s3} / {len(dataset.samples)} data files already exist. "
+            f"Meaning {len(data_files_missing)} data files will be uploaded. \n"
+            f"- Will upload {len(metadata_files_local)} metadata files. \n"
+            f"- Total files to upload: {len(data_files_missing) + len(metadata_files_local)}"
+        )
+        if will_overwrite_metadata_files:
+            msg = f"Metadata files for dataset version '{dataset.info.version}' already exist"
+            if allow_version_overwrite:
+                user_logger.warning(
+                    f"- WARNING: {msg}. Version will be overwritten as 'allow_version_overwrite=True' is set."
+                )
+            else:
+                raise ValueError(
+                    f"Upload cancelled. {msg}. \nTo overwrite existing metadata files, "
+                    "you will need to set 'allow_version_overwrite=True' explicitly."
+                )
+
+        has_missing_files = len(data_files_missing) > 0
+        if interactive and (has_missing_files or will_overwrite_metadata_files):
+            print("Please type 'yes' to upload files.")
+            confirmation = input("Do you want to continue? (yes/NO): ").strip().lower()
+
+            if confirmation != "yes":
+                raise RuntimeError("Upload cancelled by user.")
+
+        local_paths = metadata_files_local + data_files_missing[SampleField.FILE_PATH].to_list()
+        s3_paths = metadata_files_s3 + data_files_missing[SampleField.REMOTE_PATH].to_list()
+        s5cmd_utils.fast_copy_files(local_paths, s3_paths, append_envs=envs, description="Uploading files")
+    user_logger.info(f"- Synced dataset in {time.time() - t0:.2f} seconds.")
+
+
+def sync_dataset_files_to_platform(
+    dataset: HafniaDataset,
+    sample_dataset: Optional[HafniaDataset] = None,
+    interactive: bool = True,
+    allow_version_overwrite: bool = False,
+    cfg: Optional[Config] = None,
+) -> None:
+    cfg = cfg or Config()
+    resource_credentials = get_upload_credentials(dataset.info.dataset_name, cfg=cfg)
+
+    if resource_credentials is None:
+        raise RuntimeError("Failed to get upload credentials from the platform.")
+
+    sync_dataset_files_to_platform_from_resource_credentials(
+        dataset=dataset,
+        sample_dataset=sample_dataset,
+        interactive=interactive,
+        allow_version_overwrite=allow_version_overwrite,
+        resource_credentials=resource_credentials,
+    )
+
+
+def sync_dataset_files_to_platform_from_resource_credentials(
+    dataset: HafniaDataset,
+    sample_dataset: Optional[HafniaDataset],
+    interactive: bool,
+    allow_version_overwrite: bool,
+    resource_credentials: ResourceCredentials,
+):
+    envs = resource_credentials.aws_credentials()
+    bucket_name = resource_credentials.bucket_name()
+
+    for dataset_variant_type in [DatasetVariant.SAMPLE, DatasetVariant.HIDDEN]:
+        if dataset_variant_type == DatasetVariant.SAMPLE:
+            if sample_dataset is None:
+                dataset_variant = dataset.create_sample_dataset()
+            else:
+                dataset_variant = sample_dataset
+        else:
+            dataset_variant = dataset
+
+        sync_hafnia_dataset_to_s3(
+            dataset=dataset_variant,
+            bucket_prefix=f"s3://{bucket_name}/{dataset_variant_type.value}",
+            interactive=interactive,
+            allow_version_overwrite=allow_version_overwrite,
+            envs=envs,
+        )
+
+
+def s3_prefix_from_hash(hash: str, suffix: str) -> str:
+    """
+    Generate a relative S3 path from a hash value for objects stored in S3.
+
+    This function deliberately uses a hierarchical directory layout based on the
+    hash prefix to avoid putting too many objects in a single S3 prefix, which
+    can run into AWS S3 rate limits and performance issues. For example, for
+    hash "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4" and suffix ".png", the returned
+    path will be:
+
+        "data/df/e8/dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4.png"
+
+    Note: This intentionally differs from when images are stored to disk locally, where
+    a flat path of the form ``data/<hash><suffix>`` is used.
+    """
+    s3_prefix = f"data/{hash[:2]}/{hash[2:4]}/{hash}{suffix}"
+    return s3_prefix

hafnia/dataset/operations/table_transformations.py

@@ -45,7 +45,8 @@ def create_primitive_table(
         remove_no_object_frames = remove_no_object_frames.drop(drop_columns_names)
         # Rename columns "height", "width" and "meta" for sample to avoid conflicts with object fields names
         remove_no_object_frames = remove_no_object_frames.rename(
-            {"height": "image.height", "width": "image.width", "meta": "image.meta"}
+            {"height": "image.height", "width": "image.width", "meta": "image.meta"},
+            strict=False,
         )
         objects_df = remove_no_object_frames.explode(column_name).unnest(column_name)
     else:

hafnia/http.py

@@ -24,7 +24,8 @@ def fetch(endpoint: str, headers: Dict, params: Optional[Dict] = None) -> Union[
     try:
         response = http.request("GET", endpoint, fields=params, headers=headers)
         if response.status != 200:
-            raise urllib3.exceptions.HTTPError(f"Request failed with status {response.status}")
+            error_details = response.data.decode("utf-8")
+            raise urllib3.exceptions.HTTPError(f"Request failed with status {response.status}: {error_details}")
 
         return json.loads(response.data.decode("utf-8"))
     finally:

hafnia/platform/datasets.py

@@ -1,29 +1,85 @@
-import os
+import collections
 import shutil
-import subprocess
-import sys
-import tempfile
-import uuid
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 import rich
+from packaging.version import Version
 from rich import print as rprint
 
 from hafnia import http, utils
-from hafnia.dataset.dataset_names import DATASET_FILENAMES_REQUIRED
+from hafnia.dataset.dataset_names import DATASET_FILENAMES_REQUIRED, ResourceCredentials
 from hafnia.dataset.dataset_recipe.dataset_recipe import (
     DatasetRecipe,
     get_dataset_path_from_recipe,
 )
 from hafnia.dataset.hafnia_dataset import HafniaDataset
-from hafnia.http import fetch
-from hafnia.log import sys_logger, user_logger
+from hafnia.http import fetch, post
+from hafnia.log import user_logger
+from hafnia.platform import s5cmd_utils
 from hafnia.platform.download import get_resource_credentials
-from hafnia.utils import progress_bar, timed
+from hafnia.utils import timed
 from hafnia_cli.config import Config
 
 
+@timed("Fetching dataset by name.")
+def get_dataset_by_name(dataset_name: str, cfg: Optional[Config] = None) -> Optional[Dict[str, Any]]:
+    """Get dataset details by name from the Hafnia platform."""
+    cfg = cfg or Config()
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    header = {"Authorization": cfg.api_key}
+    full_url = f"{endpoint_dataset}?name__iexact={dataset_name}"
+    datasets: List[Dict[str, Any]] = http.fetch(full_url, headers=header)  # type: ignore[assignment]
+    if len(datasets) == 0:
+        return None
+
+    if len(datasets) > 1:
+        raise ValueError(f"Multiple datasets found with the name '{dataset_name}'.")
+
+    return datasets[0]
+
+
+@timed("Fetching dataset by ID.")
+def get_dataset_by_id(dataset_id: str, cfg: Optional[Config] = None) -> Optional[Dict[str, Any]]:
+    """Get dataset details by ID from the Hafnia platform."""
+    cfg = cfg or Config()
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    header = {"Authorization": cfg.api_key}
+    full_url = f"{endpoint_dataset}/{dataset_id}"
+    dataset: Dict[str, Any] = http.fetch(full_url, headers=header)  # type: ignore[assignment]
+    if not dataset:
+        return None
+
+    return dataset
+
+
+def get_or_create_dataset(dataset_name: str = "", cfg: Optional[Config] = None) -> Dict[str, Any]:
+    """Create a new dataset on the Hafnia platform."""
+    cfg = cfg or Config()
+    dataset = get_dataset_by_name(dataset_name, cfg)
+
+    if dataset is not None:
+        user_logger.info(f"Dataset '{dataset_name}' already exists on the Hafnia platform.")
+        return dataset
+
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    header = {"Authorization": cfg.api_key}
+    dataset_title = dataset_name.replace("-", " ").title()  # convert dataset-name to title "Dataset Name"
+    payload = {
+        "title": dataset_title,
+        "name": dataset_name,
+        "overview": "No description provided.",
+    }
+
+    dataset = http.post(endpoint_dataset, headers=header, data=payload)  # type: ignore[assignment]
+
+    # TODO: Handle issue when dataset creation fails because name is taken by another user from a different organization
+    if not dataset:
+        raise ValueError("Failed to create dataset on the Hafnia platform. ")
+
+    return dataset
+
+
 @timed("Fetching dataset list.")
 def get_datasets(cfg: Optional[Config] = None) -> List[Dict[str, str]]:
     """List available datasets on the Hafnia platform."""
@@ -50,6 +106,80 @@ def get_dataset_id(dataset_name: str, endpoint: str, api_key: str) -> str:
         raise ValueError("Dataset information is missing or invalid") from e
 
 
+@timed("Get upload access credentials")
+def get_upload_credentials(dataset_name: str, cfg: Optional[Config] = None) -> Optional[ResourceCredentials]:
+    """Get dataset details by name from the Hafnia platform."""
+    cfg = cfg or Config()
+    dataset_response = get_dataset_by_name(dataset_name=dataset_name, cfg=cfg)
+    if dataset_response is None:
+        return None
+
+    return get_upload_credentials_by_id(dataset_response["id"], cfg=cfg)
+
+
+@timed("Get upload access credentials by ID")
+def get_upload_credentials_by_id(dataset_id: str, cfg: Optional[Config] = None) -> Optional[ResourceCredentials]:
+    """Get dataset details by ID from the Hafnia platform."""
+    cfg = cfg or Config()
+
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    header = {"Authorization": cfg.api_key}
+    full_url = f"{endpoint_dataset}/{dataset_id}/temporary-credentials-upload"
+    credentials_response: Dict = http.fetch(full_url, headers=header)  # type: ignore[assignment]
+
+    return ResourceCredentials.fix_naming(credentials_response)
+
+
+@timed("Delete dataset by id")
+def delete_dataset_by_id(dataset_id: str, cfg: Optional[Config] = None) -> Dict:
+    cfg = cfg or Config()
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    header = {"Authorization": cfg.api_key}
+    full_url = f"{endpoint_dataset}/{dataset_id}"
+    return http.delete(full_url, headers=header)  # type: ignore
+
+
+@timed("Delete dataset by name")
+def delete_dataset_by_name(dataset_name: str, cfg: Optional[Config] = None) -> Dict:
+    cfg = cfg or Config()
+    dataset_response = get_dataset_by_name(dataset_name=dataset_name, cfg=cfg)
+    if dataset_response is None:
+        raise ValueError(f"Dataset '{dataset_name}' not found on the Hafnia platform.")
+
+    dataset_id = dataset_response["id"]  # type: ignore[union-attr]
+    response = delete_dataset_by_id(dataset_id=dataset_id, cfg=cfg)
+    user_logger.info(f"Dataset '{dataset_name}' has been deleted from the Hafnia platform.")
+    return response
+
+
+def delete_dataset_completely_by_name(dataset_name: str, interactive: bool = True) -> None:
+    from hafnia.dataset.operations.dataset_s3_storage import delete_hafnia_dataset_files_on_platform
+
+    cfg = Config()
+
+    is_deleted = delete_hafnia_dataset_files_on_platform(
+        dataset_name=dataset_name,
+        interactive=interactive,
+        cfg=cfg,
+    )
+    if not is_deleted:
+        return
+    delete_dataset_by_name(dataset_name, cfg=cfg)
+
+
+@timed("Import dataset details to platform")
+def upload_dataset_details(cfg: Config, data: dict, dataset_name: str) -> dict:
+    dataset_endpoint = cfg.get_platform_endpoint("datasets")
+    dataset_id = get_dataset_id(dataset_name, dataset_endpoint, cfg.api_key)
+
+    import_endpoint = f"{dataset_endpoint}/{dataset_id}/import"
+    headers = {"Authorization": cfg.api_key}
+
+    user_logger.info("Exporting dataset details to platform. This may take up to 30 seconds...")
+    response = post(endpoint=import_endpoint, headers=headers, data=data)  # type: ignore[assignment]
+    return response  # type: ignore[return-value]
+
+
 def download_or_get_dataset_path(
     dataset_name: str,
     cfg: Optional[Config] = None,
@@ -72,9 +202,11 @@ def download_or_get_dataset_path(
     shutil.rmtree(path_dataset, ignore_errors=True)
 
     endpoint_dataset = cfg.get_platform_endpoint("datasets")
-    dataset_id = get_dataset_id(dataset_name=dataset_name, endpoint=endpoint_dataset, api_key=api_key)
-    if dataset_id is None:
-        sys_logger.error(f"Dataset '{dataset_name}' not found on the Hafnia platform.")
+    dataset_res = get_dataset_by_name(dataset_name, cfg)  # Check if dataset exists
+    if dataset_res is None:
+        raise ValueError(f"Dataset '{dataset_name}' not found on the Hafnia platform.")
+
+    dataset_id = dataset_res.get("id")  # type: ignore[union-attr]
 
     if utils.is_hafnia_cloud_job():
         credentials_endpoint_suffix = "temporary-credentials-hidden"  # Access to hidden datasets
@@ -95,22 +227,17 @@ def download_dataset_from_access_endpoint(
     endpoint: str,
     api_key: str,
     path_dataset: Path,
+    version: Optional[str] = None,
     download_files: bool = True,
 ) -> None:
-    resource_credentials = get_resource_credentials(endpoint, api_key)
-
-    local_dataset_paths = [(path_dataset / filename).as_posix() for filename in DATASET_FILENAMES_REQUIRED]
-    s3_uri = resource_credentials.s3_uri()
-    s3_dataset_files = [f"{s3_uri}/{filename}" for filename in DATASET_FILENAMES_REQUIRED]
-
-    envs = resource_credentials.aws_credentials()
     try:
-        fast_copy_files_s3(
-            src_paths=s3_dataset_files,
-            dst_paths=local_dataset_paths,
-            append_envs=envs,
-            description="Downloading annotations",
+        resource_credentials = get_resource_credentials(endpoint, api_key)
+        download_annotation_dataset_from_version(
+            version=version,
+            credentials=resource_credentials,
+            path_dataset=path_dataset,
         )
+
     except ValueError as e:
         user_logger.error(f"Failed to download annotations: {e}")
         return
@@ -126,87 +253,6 @@ def download_dataset_from_access_endpoint(
     dataset.write_annotations(path_folder=path_dataset)  # Overwrite annotations as files have been re-downloaded
 
 
-def fast_copy_files_s3(
-    src_paths: List[str],
-    dst_paths: List[str],
-    append_envs: Optional[Dict[str, str]] = None,
-    description: str = "Copying files",
-) -> List[str]:
-    if len(src_paths) != len(dst_paths):
-        raise ValueError("Source and destination paths must have the same length.")
-    cmds = [f"cp {src} {dst}" for src, dst in zip(src_paths, dst_paths)]
-    lines = execute_s5cmd_commands(cmds, append_envs=append_envs, description=description)
-    return lines
-
-
-def find_s5cmd() -> Optional[str]:
-    """Locate the s5cmd executable across different installation methods.
-
-    Searches for s5cmd in:
-    1. System PATH (via shutil.which)
-    2. Python bin directory (Unix-like systems)
-    3. Python executable directory (direct installs)
-
-    Returns:
-        str: Absolute path to s5cmd executable if found, None otherwise.
-    """
-    result = shutil.which("s5cmd")
-    if result:
-        return result
-    python_dir = Path(sys.executable).parent
-    locations = (python_dir / "Scripts" / "s5cmd.exe", python_dir / "bin" / "s5cmd", python_dir / "s5cmd")
-    for loc in locations:
-        if loc.exists():
-            return str(loc)
-    return None
-
-
-def execute_s5cmd_commands(
-    commands: List[str],
-    append_envs: Optional[Dict[str, str]] = None,
-    description: str = "Executing s5cmd commands",
-) -> List[str]:
-    append_envs = append_envs or {}
-    # In Windows default "Temp" directory can not be deleted that is why we need to create a
-    # temporary directory.
-    with tempfile.TemporaryDirectory() as temp_dir:
-        tmp_file_path = Path(temp_dir, f"{uuid.uuid4().hex}.txt")
-        tmp_file_path.write_text("\n".join(commands))
-
-        s5cmd_bin = find_s5cmd()
-        if s5cmd_bin is None:
-            raise ValueError("Can not find s5cmd executable.")
-        run_cmds = [s5cmd_bin, "run", str(tmp_file_path)]
-        sys_logger.debug(run_cmds)
-        envs = os.environ.copy()
-        envs.update(append_envs)
-
-        process = subprocess.Popen(
-            run_cmds,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            universal_newlines=True,
-            env=envs,
-        )
-
-        error_lines = []
-        lines = []
-        for line in progress_bar(process.stdout, total=len(commands), description=description):  # type: ignore[arg-type]
-            if "ERROR" in line or "error" in line:
-                error_lines.append(line.strip())
-            lines.append(line.strip())
-
-        if len(error_lines) > 0:
-            show_n_lines = min(5, len(error_lines))
-            str_error_lines = "\n".join(error_lines[:show_n_lines])
-            user_logger.error(
-                f"Detected {len(error_lines)} errors occurred while executing a total of {len(commands)} "
-                f" commands with s5cmd. The first {show_n_lines} is printed below:\n{str_error_lines}"
-            )
-            raise RuntimeError("Errors occurred during s5cmd execution.")
-    return lines
-
-
 TABLE_FIELDS = {
     "ID": "id",
     "Hidden\nSamples": "hidden.samples",
@@ -241,3 +287,48 @@ def extend_dataset_details(datasets: List[Dict[str, Any]]) -> List[Dict[str, Any
             dataset[f"{variant_type}.samples"] = variant["number_of_data_items"]
             dataset[f"{variant_type}.size"] = utils.size_human_readable(variant["size_bytes"])
     return datasets
+
+
+def download_annotation_dataset_from_version(
+    version: Optional[str],
+    credentials: ResourceCredentials,
+    path_dataset: Path,
+) -> list[str]:
+    path_dataset.mkdir(parents=True, exist_ok=True)
+
+    envs = credentials.aws_credentials()
+    bucket_prefix_sample_versions = f"{credentials.s3_uri()}/versions"
+    all_s3_annotation_files = s5cmd_utils.list_bucket(bucket_prefix=bucket_prefix_sample_versions, append_envs=envs)
+    s3_files = _annotation_files_from_version(version=version, all_annotation_files=all_s3_annotation_files)
+
+    local_paths = [(path_dataset / filename.split("/")[-1]).as_posix() for filename in s3_files]
+    s5cmd_utils.fast_copy_files(
+        src_paths=s3_files,
+        dst_paths=local_paths,
+        append_envs=envs,
+        description="Downloading annotation files",
+    )
+    return local_paths
+
+
+def _annotation_files_from_version(version: Optional[str], all_annotation_files: list[str]) -> list[str]:
+    version_files = collections.defaultdict(list)
+    for metadata_file in all_annotation_files:
+        version_str, filename = metadata_file.split("/")[-2:]
+        if filename not in DATASET_FILENAMES_REQUIRED:
+            continue
+        version_files[version_str].append(metadata_file)
+    available_versions = {v for v, files in version_files.items() if len(files) == len(DATASET_FILENAMES_REQUIRED)}
+
+    if len(available_versions) == 0:
+        raise ValueError("No versions were found in the dataset.")
+
+    if version is None:
+        latest_version = max(Version(ver) for ver in available_versions)
+        version = str(latest_version)
+        user_logger.info(f"No version selected. Using latest version: {version}")
+
+    if version not in available_versions:
+        raise ValueError(f"Selected version '{version}' not found in available versions: {available_versions}")
+
+    return version_files[version]