hafnia 0.4.3__py3-none-any.whl → 0.5.1__py3-none-any.whl

hafnia/dataset/dataset_details_uploader.py

@@ -4,7 +4,7 @@ import base64
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
 import boto3
 import polars as pl
@@ -13,7 +13,6 @@ from pydantic import BaseModel, ConfigDict, field_validator
 
 from hafnia.dataset.dataset_names import (
     DatasetVariant,
-    DeploymentStage,
     PrimitiveField,
     SampleField,
     SplitName,
@@ -29,26 +28,21 @@ from hafnia.dataset.primitives import (
     Segmentation,
 )
 from hafnia.dataset.primitives.primitive import Primitive
-from hafnia.http import post
-from hafnia.log import user_logger
-from hafnia.platform.datasets import get_dataset_id
+from hafnia.platform.datasets import upload_dataset_details
+from hafnia.utils import get_path_dataset_gallery_images
 from hafnia_cli.config import Config
 
 
-def generate_bucket_name(dataset_name: str, deployment_stage: DeploymentStage) -> str:
-    # TODO: When moving to versioning we do NOT need 'staging' and 'production' specific buckets
-    # and the new name convention should be: f"hafnia-dataset-{dataset_name}"
-    return f"mdi-{deployment_stage.value}-{dataset_name}"
-
-
 class DatasetDetails(BaseModel, validate_assignment=True):  # type: ignore[call-arg]
     model_config = ConfigDict(use_enum_values=True)  # To parse Enum values as strings
     name: str
+    title: Optional[str] = None
+    overview: Optional[str] = None
     data_captured_start: Optional[datetime] = None
     data_captured_end: Optional[datetime] = None
     data_received_start: Optional[datetime] = None
     data_received_end: Optional[datetime] = None
-    latest_update: Optional[datetime] = None
+    dataset_updated_at: Optional[datetime] = None
     license_citation: Optional[str] = None
     version: Optional[str] = None
     s3_bucket_name: Optional[str] = None
@@ -281,26 +275,32 @@ def get_folder_size(path: Path) -> int:
     return sum([path.stat().st_size for path in path.rglob("*")])
 
 
-def upload_to_hafnia_dataset_detail_page(dataset_update: DatasetDetails, upload_gallery_images: bool) -> dict:
-    if not upload_gallery_images:
-        dataset_update.imgs = None
-
-    cfg = Config()
-    dataset_details = dataset_update.model_dump_json()
-    data = upload_dataset_details(cfg=cfg, data=dataset_details, dataset_name=dataset_update.name)
-    return data
-
-
-def upload_dataset_details(cfg: Config, data: str, dataset_name: str) -> dict:
-    dataset_endpoint = cfg.get_platform_endpoint("datasets")
-    dataset_id = get_dataset_id(dataset_name, dataset_endpoint, cfg.api_key)
+def upload_dataset_details_to_platform(
+    dataset: HafniaDataset,
+    path_gallery_images: Optional[Path] = None,
+    gallery_image_names: Optional[List[str]] = None,
+    distribution_task_names: Optional[List[str]] = None,
+    update_platform: bool = True,
+    cfg: Optional[Config] = None,
+) -> dict:
+    cfg = cfg or Config()
+    dataset_details = dataset_details_from_hafnia_dataset(
+        dataset=dataset,
+        path_gallery_images=path_gallery_images,
+        gallery_image_names=gallery_image_names,
+        distribution_task_names=distribution_task_names,
+    )
 
-    import_endpoint = f"{dataset_endpoint}/{dataset_id}/import"
-    headers = {"Authorization": cfg.api_key}
+    if update_platform:
+        dataset_details_exclude_none = dataset_details.model_dump(exclude_none=True, mode="json")
+        upload_dataset_details(
+            cfg=cfg,
+            data=dataset_details_exclude_none,
+            dataset_name=dataset_details.name,
+        )
 
-    user_logger.info("Exporting dataset details to platform. This may take up to 30 seconds...")
-    response = post(endpoint=import_endpoint, headers=headers, data=data)  # type: ignore[assignment]
-    return response  # type: ignore[return-value]
+    dataset_details_dict = dataset_details.model_dump(exclude_none=False, mode="json")
+    return dataset_details_dict
 
 
 def get_resolutions(dataset: HafniaDataset, max_resolutions_selected: int = 8) -> List[DbResolution]:
@@ -360,9 +360,6 @@ def s3_based_fields(bucket_name: str, variant_type: DatasetVariant, session: bot
 
 def dataset_details_from_hafnia_dataset(
     dataset: HafniaDataset,
-    deployment_stage: DeploymentStage,
-    path_sample: Optional[Path],
-    path_hidden: Optional[Path],
     path_gallery_images: Optional[Path] = None,
     gallery_image_names: Optional[List[str]] = None,
     distribution_task_names: Optional[List[str]] = None,
@@ -371,33 +368,24 @@ def dataset_details_from_hafnia_dataset(
     dataset_reports = []
     dataset_meta_info = dataset.info.meta or {}
 
-    path_and_variant: List[Tuple[Path, DatasetVariant]] = []
-    if path_sample is not None:
-        path_and_variant.append((path_sample, DatasetVariant.SAMPLE))
-
-    if path_hidden is not None:
-        path_and_variant.append((path_hidden, DatasetVariant.HIDDEN))
-
-    if len(path_and_variant) == 0:
-        raise ValueError("At least one path must be provided for sample or hidden dataset.")
-
+    path_and_variant = [DatasetVariant.SAMPLE, DatasetVariant.HIDDEN]
     gallery_images = create_gallery_images(
         dataset=dataset,
         path_gallery_images=path_gallery_images,
         gallery_image_names=gallery_image_names,
     )
 
-    for path_dataset, variant_type in path_and_variant:
+    for variant_type in path_and_variant:
         if variant_type == DatasetVariant.SAMPLE:
             dataset_variant = dataset.create_sample_dataset()
         else:
             dataset_variant = dataset
 
-        size_bytes = get_folder_size(path_dataset)
+        files_paths = dataset_variant.samples[SampleField.FILE_PATH].to_list()
+        size_bytes = sum([Path(file_path).stat().st_size for file_path in files_paths])
         dataset_variants.append(
             DbDatasetVariant(
                 variant_type=VARIANT_TYPE_MAPPING[variant_type],  # type: ignore[index]
-                # upload_date: Optional[datetime] = None
                 size_bytes=size_bytes,
                 data_type=DataTypeChoices.images,
                 number_of_data_items=len(dataset_variant),
@@ -405,7 +393,6 @@ def dataset_details_from_hafnia_dataset(
                 duration=dataset_meta_info.get("duration", None),
                 duration_average=dataset_meta_info.get("duration_average", None),
                 frame_rate=dataset_meta_info.get("frame_rate", None),
-                # bit_rate: Optional[float] = None
                 n_cameras=dataset_meta_info.get("n_cameras", None),
             )
         )
@@ -435,19 +422,19 @@ def dataset_details_from_hafnia_dataset(
             object_reports = sorted(object_reports, key=lambda x: x.obj.name)  # Sort object reports by name
             report.annotated_object_reports = object_reports
 
-        if report.distribution_values is None:
-            report.distribution_values = []
+            if report.distribution_values is None:
+                report.distribution_values = []
 
-        dataset_reports.append(report)
+            dataset_reports.append(report)
     dataset_name = dataset.info.dataset_name
-    bucket_sample = generate_bucket_name(dataset_name, deployment_stage=deployment_stage)
     dataset_info = DatasetDetails(
         name=dataset_name,
+        title=dataset.info.dataset_title,
+        overview=dataset.info.description,
         version=dataset.info.version,
-        s3_bucket_name=bucket_sample,
         dataset_variants=dataset_variants,
         split_annotations_reports=dataset_reports,
-        latest_update=dataset.info.updated_at,
+        dataset_updated_at=dataset.info.updated_at,
         dataset_format_version=dataset.info.format_version,
         license_citation=dataset.info.reference_bibtex,
         data_captured_start=dataset_meta_info.get("data_captured_start", None),
@@ -565,7 +552,7 @@ def create_gallery_images(
     gallery_images = None
     if (gallery_image_names is not None) and (len(gallery_image_names) > 0):
         if path_gallery_images is None:
-            raise ValueError("Path to gallery images must be provided.")
+            path_gallery_images = get_path_dataset_gallery_images(dataset.info.dataset_name)
         path_gallery_images.mkdir(parents=True, exist_ok=True)
         COL_IMAGE_NAME = "image_name"
         samples = dataset.samples.with_columns(

hafnia/dataset/dataset_helpers.py

@@ -3,12 +3,70 @@ import math
 import random
 import shutil
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional, Tuple
 
 import numpy as np
 import xxhash
+from packaging.version import InvalidVersion, Version
 from PIL import Image
 
+from hafnia.log import user_logger
+
+
+def is_valid_version_string(version: Optional[str], allow_none: bool = False, allow_latest: bool = False) -> bool:
+    if allow_none and version is None:
+        return True
+    if allow_latest and version == "latest":
+        return True
+    return version_from_string(version, raise_error=False) is not None
+
+
+def version_from_string(version: Optional[str], raise_error: bool = True) -> Optional[Version]:
+    if version is None:
+        if raise_error:
+            raise ValueError("Version is 'None'. A valid version string is required e.g '1.0.0'")
+        return None
+
+    try:
+        version_casted = Version(version)
+    except (InvalidVersion, TypeError) as e:
+        if raise_error:
+            raise ValueError(f"Invalid version string/type: {version}") from e
+        return None
+
+    # Check if version is semantic versioning (MAJOR.MINOR.PATCH)
+    if len(version_casted.release) < 3:
+        if raise_error:
+            raise ValueError(f"Version string '{version}' is not semantic versioning (MAJOR.MINOR.PATCH)")
+        return None
+    return version_casted
+
+
+def dataset_name_and_version_from_string(
+    string: str,
+    resolve_missing_version: bool = True,
+) -> Tuple[str, Optional[str]]:
+    if not isinstance(string, str):
+        raise TypeError(f"'{type(string)}' for '{string}' is an unsupported type. Expected 'str' e.g 'mnist:1.0.0'")
+
+    parts = string.split(":")
+    if len(parts) == 1:
+        dataset_name = parts[0]
+        if resolve_missing_version:
+            version = "latest"  # Default to 'latest' if version is missing. This will be resolved to a specific version later.
+            user_logger.info(f"Version is missing in dataset name: {string}. Defaulting to version='latest'.")
+        else:
+            raise ValueError(f"Version is missing in dataset name: {string}. Use 'name:version'")
+    elif len(parts) == 2:
+        dataset_name, version = parts
+    else:
+        raise ValueError(f"Invalid dataset name format: {string}. Use 'name' or 'name:version' ")
+
+    if not is_valid_version_string(version, allow_none=True, allow_latest=True):
+        raise ValueError(f"Invalid version string: {version}. Use semantic versioning e.g. '1.0.0' or 'latest'")
+
+    return dataset_name, version
+
 
 def create_split_name_list_from_ratios(split_ratios: Dict[str, float], n_items: int, seed: int = 42) -> List[str]:
     samples_per_split = split_sizes_from_ratios(split_ratios=split_ratios, n_items=n_items)
@@ -57,20 +115,6 @@ def save_pil_image_with_hash_name(image: Image.Image, path_folder: Path, allow_s
 def copy_and_rename_file_to_hash_value(path_source: Path, path_dataset_root: Path) -> Path:
     """
     Copies a file to a dataset root directory with a hash-based name and sub-directory structure.
-
-    E.g. for an "image.png" with hash "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4", the image will be copied to
-    'path_dataset_root / "data" / "dfe" / "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4.png"'
-    Notice that the hash is used for both the filename and the subfolder name.
-
-    Placing image/video files into multiple sub-folders (instead of one large folder) is seemingly
-    unnecessary, but it is actually a requirement when the dataset is later downloaded from S3.
-
-    The reason is that AWS has a rate limit of 3500 ops/sec per prefix (sub-folder) in S3 - meaning we can "only"
-    download 3500 files per second from a single folder (prefix) in S3.
-
-    For even a single user, we found that this limit was being reached when files are stored in single folder (prefix)
-    in S3. To support multiple users and concurrent experiments, we are required to separate files into
-    multiple sub-folders (prefixes) in S3 to not hit the rate limit.
     """
 
     if not path_source.exists():
@@ -86,7 +130,7 @@ def copy_and_rename_file_to_hash_value(path_source: Path, path_dataset_root: Pat
 
 
 def relative_path_from_hash(hash: str, suffix: str) -> Path:
-    path_file = Path("data") / hash[:3] / f"{hash}{suffix}"
+    path_file = Path("data") / f"{hash}{suffix}"
     return path_file
 
 

hafnia/dataset/dataset_names.py

@@ -1,8 +1,5 @@
 from enum import Enum
-from typing import Dict, List, Optional
-
-import boto3
-from pydantic import BaseModel, field_validator
+from typing import List
 
 FILENAME_RECIPE_JSON = "recipe.json"
 FILENAME_DATASET_INFO = "dataset_info.json"
@@ -124,93 +121,3 @@ class DatasetVariant(Enum):
     DUMP = "dump"
     SAMPLE = "sample"
     HIDDEN = "hidden"
-
-
-class AwsCredentials(BaseModel):
-    access_key: str
-    secret_key: str
-    session_token: str
-    region: Optional[str]
-
-    def aws_credentials(self) -> Dict[str, str]:
-        """
-        Returns the AWS credentials as a dictionary.
-        """
-        environment_vars = {
-            "AWS_ACCESS_KEY_ID": self.access_key,
-            "AWS_SECRET_ACCESS_KEY": self.secret_key,
-            "AWS_SESSION_TOKEN": self.session_token,
-        }
-        if self.region:
-            environment_vars["AWS_REGION"] = self.region
-
-        return environment_vars
-
-    @staticmethod
-    def from_session(session: boto3.Session) -> "AwsCredentials":
-        """
-        Creates AwsCredentials from a Boto3 session.
-        """
-        frozen_credentials = session.get_credentials().get_frozen_credentials()
-        return AwsCredentials(
-            access_key=frozen_credentials.access_key,
-            secret_key=frozen_credentials.secret_key,
-            session_token=frozen_credentials.token,
-            region=session.region_name,
-        )
-
-
-ARN_PREFIX = "arn:aws:s3:::"
-
-
-class ResourceCredentials(AwsCredentials):
-    s3_arn: str
-
-    @staticmethod
-    def fix_naming(payload: Dict[str, str]) -> "ResourceCredentials":
-        """
-        The endpoint returns a payload with a key called 's3_path', but it
-        is actually an ARN path (starts with arn:aws:s3::). This method renames it to 's3_arn' for consistency.
-        """
-        if "s3_path" in payload and payload["s3_path"].startswith(ARN_PREFIX):
-            payload["s3_arn"] = payload.pop("s3_path")
-
-        if "region" not in payload:
-            payload["region"] = "eu-west-1"
-        return ResourceCredentials(**payload)
-
-    @field_validator("s3_arn")
-    @classmethod
-    def validate_s3_arn(cls, value: str) -> str:
-        """Validate s3_arn to ensure it starts with 'arn:aws:s3:::'"""
-        if not value.startswith("arn:aws:s3:::"):
-            raise ValueError(f"Invalid S3 ARN: {value}. It should start with 'arn:aws:s3:::'")
-        return value
-
-    def s3_path(self) -> str:
-        """
-        Extracts the S3 path from the ARN.
-        Example: arn:aws:s3:::my-bucket/my-prefix -> my-bucket/my-prefix
-        """
-        return self.s3_arn[len(ARN_PREFIX) :]
-
-    def s3_uri(self) -> str:
-        """
-        Converts the S3 ARN to a URI format.
-        Example: arn:aws:s3:::my-bucket/my-prefix -> s3://my-bucket/my-prefix
-        """
-        return f"s3://{self.s3_path()}"
-
-    def bucket_name(self) -> str:
-        """
-        Extracts the bucket name from the S3 ARN.
-        Example: arn:aws:s3:::my-bucket/my-prefix -> my-bucket
-        """
-        return self.s3_path().split("/")[0]
-
-    def object_key(self) -> str:
-        """
-        Extracts the object key from the S3 ARN.
-        Example: arn:aws:s3:::my-bucket/my-prefix -> my-prefix
-        """
-        return "/".join(self.s3_path().split("/")[1:])

hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -11,14 +11,19 @@ from pydantic import (
 )
 
 from hafnia import utils
+from hafnia.dataset.dataset_helpers import dataset_name_and_version_from_string
 from hafnia.dataset.dataset_recipe import recipe_transforms
 from hafnia.dataset.dataset_recipe.recipe_types import (
     RecipeCreation,
     RecipeTransform,
     Serializable,
 )
-from hafnia.dataset.hafnia_dataset import HafniaDataset
+from hafnia.dataset.hafnia_dataset import (
+    HafniaDataset,
+    available_dataset_versions_from_name,
+)
 from hafnia.dataset.primitives.primitive import Primitive
+from hafnia.log import user_logger
 
 
 class DatasetRecipe(Serializable):
@@ -41,8 +46,31 @@ class DatasetRecipe(Serializable):
 
     ### Creation Methods (using the 'from_X' )###
     @staticmethod
-    def from_name(name: str, force_redownload: bool = False, download_files: bool = True) -> DatasetRecipe:
-        creation = FromName(name=name, force_redownload=force_redownload, download_files=download_files)
+    def from_name(
+        name: str,
+        version: Optional[str] = None,
+        force_redownload: bool = False,
+        download_files: bool = True,
+    ) -> DatasetRecipe:
+        if version == "latest":
+            user_logger.info(
+                f"The dataset '{name}' in a dataset recipe uses 'latest' as version. For dataset recipes the "
+                "version is pinned to a specific version. Consider specifying a specific version to ensure "
+                "reproducibility of your experiments. "
+            )
+            available_versions = available_dataset_versions_from_name(name)
+            version = str(max(available_versions))
+        if version is None:
+            available_versions = available_dataset_versions_from_name(name)
+            str_versions = ", ".join([str(v) for v in available_versions])
+            raise ValueError(
+                f"Version must be specified when creating a DatasetRecipe from name. "
+                f"Available versions are: {str_versions}"
+            )
+
+        creation = FromName(
+            name=name, version=version, force_redownload=force_redownload, download_files=download_files
+        )
         return DatasetRecipe(creation=creation)
 
     @staticmethod
@@ -125,6 +153,21 @@ class DatasetRecipe(Serializable):
         recipe_id = recipe["id"]
         return DatasetRecipe.from_recipe_id(recipe_id)
 
+    @staticmethod
+    def from_name_and_version_string(string: str, resolve_missing_version: bool = False) -> "DatasetRecipe":
+        """
+        Validates and converts a dataset name and version string (name:version) to a DatasetRecipe.from_name recipe.
+        If version is missing and 'resolve_missing_version' is True, it will default to 'latest'.
+        If resolve_missing_version is False, it will raise an error if version is missing.
+        """
+
+        dataset_name, version = dataset_name_and_version_from_string(
+            string=string,
+            resolve_missing_version=resolve_missing_version,
+        )
+
+        return DatasetRecipe.from_name(name=dataset_name, version=version)
+
     @staticmethod
     def from_implicit_form(recipe: Any) -> DatasetRecipe:
         """
@@ -180,7 +223,7 @@ class DatasetRecipe(Serializable):
             return recipe
 
         if isinstance(recipe, str):  # str-type is convert to DatasetFromName
-            return DatasetRecipe.from_name(name=recipe)
+            return DatasetRecipe.from_name_and_version_string(string=recipe, resolve_missing_version=True)
 
         if isinstance(recipe, Path):  # Path-type is convert to DatasetFromPath
             return DatasetRecipe.from_path(path_folder=recipe)
@@ -409,6 +452,7 @@ class FromPath(RecipeCreation):
 
 class FromName(RecipeCreation):
     name: str
+    version: Optional[str] = None
     force_redownload: bool = False
     download_files: bool = True
 

hafnia/dataset/format_conversions/torchvision_datasets.py

@@ -40,7 +40,7 @@ def mnist_as_hafnia_dataset(force_redownload=False, n_samples: Optional[int] = N
 
     dataset_info = DatasetInfo(
         dataset_name="mnist",
-        version="1.1.0",
+        version="1.0.0",
         tasks=tasks,
         reference_bibtex=textwrap.dedent("""\
             @article{lecun2010mnist,
@@ -78,7 +78,7 @@ def caltech_101_as_hafnia_dataset(
         n_samples=n_samples,
         dataset_name=dataset_name,
     )
-    hafnia_dataset.info.version = "1.1.0"
+    hafnia_dataset.info.version = "1.0.0"
     hafnia_dataset.info.reference_bibtex = textwrap.dedent("""\
         @article{FeiFei2004LearningGV,
             title={Learning Generative Visual Models from Few Training Examples: An Incremental Bayesian
@@ -108,7 +108,7 @@ def caltech_256_as_hafnia_dataset(
         n_samples=n_samples,
         dataset_name=dataset_name,
     )
-    hafnia_dataset.info.version = "1.1.0"
+    hafnia_dataset.info.version = "1.0.0"
     hafnia_dataset.info.reference_bibtex = textwrap.dedent("""\
         @misc{griffin_2023_5sv1j-ytw97,
             author       = {Griffin, Gregory and
@@ -150,7 +150,7 @@ def cifar_as_hafnia_dataset(
 
     dataset_info = DatasetInfo(
         dataset_name=dataset_name,
-        version="1.1.0",
+        version="1.0.0",
         tasks=tasks,
         reference_bibtex=textwrap.dedent("""\
         @@TECHREPORT{Krizhevsky09learningmultiple,
@@ -268,7 +268,10 @@ def _download_and_extract_caltech_dataset(dataset_name: str, force_redownload: b
             path_output_extracted = path_tmp_output / "caltech-101"
             for gzip_file in os.listdir(path_output_extracted):
                 if gzip_file.endswith(".gz"):
-                    extract_archive(os.path.join(path_output_extracted, gzip_file), path_output_extracted)
+                    extract_archive(
+                        from_path=os.path.join(path_output_extracted, gzip_file),
+                        to_path=path_output_extracted,
+                    )
             path_org = path_output_extracted / "101_ObjectCategories"
 
         elif dataset_name == "caltech-256":