hafnia 0.2.2__tar.gz → 0.2.4__tar.gz

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/build.yaml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       package-version: ${{ steps.extract-version.outputs.package_version }}
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v5.0.0
       - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/check_release.yaml

@@ -20,7 +20,7 @@ jobs:
       make_release: ${{ steps.check_release.outputs.make_release }}
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v4.3.0
+        uses: actions/download-artifact@v5.0.0
         with:
             name: python-package
             path: dist/

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/ci_cd.yaml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: lint
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v5.0.0
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@0.32.0
         with:

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/lint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v5.0.0
       - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/publish_docker.yaml

@@ -24,14 +24,14 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v5.0.0
       - uses: actions/setup-python@v5.6.0
         id: python
         with:
           python-version-file: ${{ inputs.python-version-file }}
 
       - name: Download package artifact
-        uses: actions/download-artifact@v4.3.0
+        uses: actions/download-artifact@v5.0.0
         with:
           name: python-package
           path: dist/
@@ -47,7 +47,7 @@ jobs:
             echo "aws_region=${{ secrets.STAGE_AWS_REGION }}" >> $GITHUB_OUTPUT
           fi
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4.2.1
+        uses: aws-actions/configure-aws-credentials@v4.3.1
         with:
           role-to-assume: arn:aws:iam::${{ steps.env-vars.outputs.aws_account_id }}:role/${{ secrets.AWS_ROLE_NAME }}
           aws-region: ${{ steps.env-vars.outputs.aws_region }} 

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/publish_pypi.yaml

@@ -17,7 +17,7 @@ jobs:
       contents: read
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v4.3.0
+        uses: actions/download-artifact@v5.0.0
         with:
           name: python-package
           path: dist/

{hafnia-0.2.2 → hafnia-0.2.4}/.github/workflows/tests.yaml

@@ -11,7 +11,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v5.0.0
       - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}

{hafnia-0.2.2 → hafnia-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.2.2
+Version: 0.2.4
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE

{hafnia-0.2.2 → hafnia-0.2.4}/examples/example_dataset_recipe.py

@@ -29,20 +29,24 @@ dataset_recipe = DatasetRecipe.from_name(name="mnist").shuffle().select_samples(
 dataset = dataset_recipe.build()
 # Note that the interface is similar, but to actually create the dataset you need to call `build()` on the recipe.
 
-# An important feature of a 'DatasetRecipe' is that the recipe itself - and not the dataset - can be saved as a file
-# and loaded from file. Meaning you can easily save, share, load and build the dataset later or in a different
-# environment.
-# In programming language, the recipe can be serialized to JSON and deserialized back to the original python object
-# recipe.
+# Unlike the HafniaDataset, a DatasetRecipe does not execute operations. It only registers
+# the operations applied to the recipe and can be used to build the dataset later.
+# You can print the dataset recipe to the operations that were applied to it.
+rprint(dataset_recipe)
+
+# Or as a JSON string:
+json_str: str = dataset_recipe.as_json_str()
+rprint(json_str)
+
+# This is an important feature of a 'DatasetRecipe' it only registers operations and that the recipe itself
+# - and not the dataset - can be saved as a file and loaded from file.
+# Meaning you can easily save, share, load and build the dataset later or in a different environment.
 # For TaaS, this is the only way to include multiple datasets during training.
 
-# This is how it looks like in practice:
-# 1) Save the dataset recipe to a file
-path_json = Path(".data/tmp/dataset_recipe.json")
-dataset_recipe.as_json_file(path_json)
 
-# 2) The recipe can be loaded from the file
-dataset_recipe_again = DatasetRecipe.from_json_file(path_json)
+# 2) The recipe can be loaded from json string
+dataset_recipe_again: DatasetRecipe = DatasetRecipe.from_json_str(json_str)
+# dataset_recipe_again.build()
 
 # We can verify that the loaded recipe is the same as the original recipe.
 assert dataset_recipe_again == dataset_recipe

{hafnia-0.2.2 → hafnia-0.2.4}/examples/example_hafnia_dataset.py

@@ -26,11 +26,10 @@ dataset = HafniaDataset.from_path(path_dataset)
 # Alternatively, you can use the 'load_dataset' function
 dataset = load_dataset("midwest-vehicle-detection")
 
-
 # Dataset information is stored in 'dataset.info'
 rprint(dataset.info)
 
-# Annotations are stored in 'dataset.table' as a Polars DataFrame
+# Annotations are stored in 'dataset.samples' as a Polars DataFrame
 dataset.samples.head(2)
 
 # Print dataset information
@@ -49,14 +48,29 @@ shuffled_dataset = dataset.shuffle(seed=42)  # Shuffle the dataset
 split_ratios = {SplitName.TRAIN: 0.8, SplitName.VAL: 0.1, SplitName.TEST: 0.1}
 new_dataset_splits = dataset.splits_by_ratios(split_ratios)
 
+# Support Chaining Operations (load, shuffle, select samples)
+dataset = load_dataset("midwest-vehicle-detection").shuffle(seed=42).select_samples(n_samples=10)
+
+
 # Write dataset to disk
 path_tmp = Path(".data/tmp")
 path_dataset = path_tmp / "hafnia_dataset"
-dataset.write(path_dataset)  # --> Check that data is human readable
+dataset.write(path_dataset)
 
 # Load dataset from disk
 dataset_again = HafniaDataset.from_path(path_dataset)
 
+
+# Want custom dataset transformations or statistics? Use the polars table (dataset.samples) directly
+n_objects = dataset.samples["objects"].list.len().sum()
+n_objects = dataset.samples[Bbox.column_name()].list.len().sum()  # Use Bbox.column_name() to avoid magic variables
+n_classifications = dataset.samples[Classification.column_name()].list.len().sum()
+
+class_counts = dataset.samples[Classification.column_name()].explode().struct.field("class_name").value_counts()
+class_counts = dataset.samples[Bbox.column_name()].explode().struct.field("class_name").value_counts()
+rprint(dict(class_counts.iter_rows()))
+
+
 # Access the first sample in the training split - data is stored in a dictionary
 sample_dict = dataset_train[0]
 
@@ -78,25 +92,15 @@ image: np.ndarray = sample.read_image()
 # Visualize sample and annotations
 image_with_annotations = sample.draw_annotations()
 
-
+# Save the image with annotations to a temporary directory
 path_tmp.mkdir(parents=True, exist_ok=True)
 Image.fromarray(image_with_annotations).save(path_tmp / "sample_with_annotations.png")
 
 
-# Do dataset transformations and statistics on the Polars DataFrame
-n_objects = dataset.samples["objects"].list.len().sum()
-n_objects = dataset.samples[Bbox.column_name()].list.len().sum()  # Use Bbox.column_name() to avoid magic variables
-n_classifications = dataset.samples[Classification.column_name()].list.len().sum()
-
-class_counts = dataset.samples[Classification.column_name()].explode().struct.field("class_name").value_counts()
-class_counts = dataset.samples[Bbox.column_name()].explode().struct.field("class_name").value_counts()
-rprint(dict(class_counts.iter_rows()))
-
-
 ## Bring-your-own-data: Create a new dataset from samples
 fake_samples = []
 for i_fake_sample in range(5):
-    bboxes = [Bbox(top_left_x=10, top_left_y=20, width=100, height=200, class_name="car")]
+    bboxes = [Bbox(top_left_x=0.1, top_left_y=0.20, width=0.1, height=0.2, class_name="car")]
     classifications = [Classification(class_name="vehicle", class_idx=0)]
     sample = Sample(
         file_name=f"path/to/image_{i_fake_sample:05}.jpg",
@@ -120,8 +124,14 @@ fake_dataset_info = DatasetInfo(
 )
 fake_dataset = HafniaDataset.from_samples_list(samples_list=fake_samples, info=fake_dataset_info)
 
+# Coming soon! Upload your dataset to the Hafnia Platform
+# fake_dataset.upload_to_hafnia()
+
+# Coming soon! Create your own dataset details page in Hafnia
+# fake_dataset.upload_dataset_details()
 
-## A hafnia dataset can also be used for storing predictions per sample set 'ground_truth=False' and add 'confidence'.
+## Storing predictions: A hafnia dataset can also be used for storing predictions per sample
+# set 'ground_truth=False' and add 'confidence'.
 bboxes_predictions = [
     Bbox(top_left_x=10, top_left_y=20, width=100, height=200, class_name="car", ground_truth=False, confidence=0.9)
 ]

{hafnia-0.2.2 → hafnia-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hafnia"
-version = "0.2.2"
+version = "0.2.4"
 description = "Python SDK for communication with Hafnia platform."
 readme = "README.md"
 authors = [

{hafnia-0.2.2 → hafnia-0.2.4}/src/cli/__main__.py

@@ -20,19 +20,15 @@ def configure(cfg: Config) -> None:
 
     profile_name = click.prompt("Profile Name", type=str, default=consts.DEFAULT_PROFILE_NAME)
     profile_name = profile_name.strip()
-    try:
-        cfg.add_profile(profile_name, ConfigSchema(), set_active=True)
-    except ValueError:
-        raise click.ClickException(consts.ERROR_CREATE_PROFILE)
+
+    cfg.check_profile_name(profile_name)
 
     api_key = click.prompt("Hafnia API Key", type=str, hide_input=True)
-    try:
-        cfg.api_key = api_key.strip()
-    except ValueError as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        return
+
     platform_url = click.prompt("Hafnia Platform URL", type=str, default=consts.DEFAULT_API_URL)
-    cfg.platform_url = platform_url.strip()
+
+    cfg_profile = ConfigSchema(api_key=api_key, platform_url=platform_url)
+    cfg.add_profile(profile_name, cfg_profile, set_active=True)
     cfg.save_config()
     profile_cmds.profile_show(cfg)
 

{hafnia-0.2.2 → hafnia-0.2.4}/src/cli/config.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional
 from pydantic import BaseModel, field_validator
 
 import cli.consts as consts
-from hafnia.log import user_logger
+from hafnia.log import sys_logger, user_logger
 
 PLATFORM_API_MAPPING = {
     "recipes": "/api/v1/recipes",
@@ -23,9 +23,17 @@ class ConfigSchema(BaseModel):
     api_key: Optional[str] = None
 
     @field_validator("api_key")
-    def validate_api_key(cls, value: str) -> str:
-        if value is not None and len(value) < 10:
+    def validate_api_key(cls, value: Optional[str]) -> Optional[str]:
+        if value is None:
+            return value
+
+        if len(value) < 10:
             raise ValueError("API key is too short.")
+
+        if not value.startswith("ApiKey "):
+            sys_logger.warning("API key is missing the 'ApiKey ' prefix. Prefix is being added automatically.")
+            value = f"ApiKey {value}"
+
         return value
 
 
@@ -51,6 +59,7 @@ class Config:
         if profile_name not in self.config_data.profiles:
             raise ValueError(f"Profile '{profile_name}' does not exist.")
         self.config_data.active_profile = profile_name
+        self.save_config()
 
     @property
     def config(self) -> ConfigSchema:
@@ -92,13 +101,18 @@ class Config:
 
         return Path.home() / ".hafnia" / "config.json"
 
-    def add_profile(self, profile_name: str, profile: ConfigSchema, set_active: bool = False) -> None:
-        profile_name = profile_name.strip()
+    def check_profile_name(self, profile_name: str) -> None:
+        if not profile_name or not isinstance(profile_name, str):
+            raise ValueError("Profile name must be a non-empty string.")
+
         if profile_name in self.config_data.profiles:
             user_logger.warning(
                 f"Profile with name '{profile_name}' already exists, it will be overwritten by the new one."
             )
 
+    def add_profile(self, profile_name: str, profile: ConfigSchema, set_active: bool = False) -> None:
+        profile_name = profile_name.strip()
+        self.check_profile_name(profile_name)
         self.config_data.profiles[profile_name] = profile
         if set_active:
             self.config_data.active_profile = profile_name

{hafnia-0.2.2 → hafnia-0.2.4}/src/cli/profile_cmds.py

@@ -56,6 +56,7 @@ def profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate:
     cfg_profile = ConfigSchema(platform_url=api_url, api_key=api_key)
 
     cfg.add_profile(profile_name=name, profile=cfg_profile, set_active=activate)
+    profile_show(cfg)
 
 
 @profile.command("rm")
@@ -87,7 +88,7 @@ def profile_active(cfg: Config) -> None:
 
 
 def profile_show(cfg: Config) -> None:
-    masked_key = f"{cfg.api_key[:4]}...{cfg.api_key[-4:]}" if len(cfg.api_key) > 8 else "****"
+    masked_key = f"{cfg.api_key[:11]}...{cfg.api_key[-4:]}" if len(cfg.api_key) > 20 else "****"
     console = Console()
 
     table = Table(title=f"{consts.PROFILE_TABLE_HEADER} {cfg.active_profile}", show_header=False)

{hafnia-0.2.2 → hafnia-0.2.4}/src/hafnia/dataset/dataset_helpers.py

@@ -1,6 +1,7 @@
 import io
 import math
 import random
+import shutil
 from pathlib import Path
 from typing import Dict, List
 
@@ -21,7 +22,7 @@ def create_split_name_list_from_ratios(split_ratios: Dict[str, float], n_items:
 
 
 def hash_file_xxhash(path: Path, chunk_size: int = 262144) -> str:
-    hasher = xxhash.xxh3_64()
+    hasher = xxhash.xxh3_128()
 
     with open(path, "rb") as f:
         for chunk in iter(lambda: f.read(chunk_size), b""):  # 8192, 16384, 32768, 65536
@@ -30,7 +31,7 @@ def hash_file_xxhash(path: Path, chunk_size: int = 262144) -> str:
 
 
 def hash_from_bytes(data: bytes) -> str:
-    hasher = xxhash.xxh3_64()
+    hasher = xxhash.xxh3_128()
     hasher.update(data)
     return hasher.hexdigest()
 
@@ -40,14 +41,46 @@ def save_image_with_hash_name(image: np.ndarray, path_folder: Path) -> Path:
     buffer = io.BytesIO()
     pil_image.save(buffer, format="PNG")
     hash_value = hash_from_bytes(buffer.getvalue())
-    path_image = Path(path_folder) / f"{hash_value}.png"
+    path_image = Path(path_folder) / relative_path_from_hash(hash=hash_value, suffix=".png")
+    path_image.parent.mkdir(parents=True, exist_ok=True)
     pil_image.save(path_image)
     return path_image
 
 
-def filename_as_hash_from_path(path_image: Path) -> str:
-    hash = hash_file_xxhash(path_image)
-    return f"{hash}{path_image.suffix}"
+def copy_and_rename_file_to_hash_value(path_source: Path, path_dataset_root: Path) -> Path:
+    """
+    Copies a file to a dataset root directory with a hash-based name and sub-directory structure.
+
+    E.g. for an "image.png" with hash "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4", the image will be copied to
+    'path_dataset_root / "data" / "dfe" / "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4.png"'
+    Notice that the hash is used for both the filename and the subfolder name.
+
+    Placing image/video files into multiple sub-folders (instead of one large folder) is seemingly
+    unnecessary, but it is actually a requirement when the dataset is later downloaded from S3.
+
+    The reason is that AWS has a rate limit of 3500 ops/sec per prefix (sub-folder) in S3 - meaning we can "only"
+    download 3500 files per second from a single folder (prefix) in S3.
+
+    For even a single user, we found that this limit was being reached when files are stored in single folder (prefix)
+    in S3. To support multiple users and concurrent experiments, we are required to separate files into
+    multiple sub-folders (prefixes) in S3 to not hit the rate limit.
+    """
+
+    if not path_source.exists():
+        raise FileNotFoundError(f"Source file {path_source} does not exist.")
+
+    hash_value = hash_file_xxhash(path_source)
+    path_file = path_dataset_root / relative_path_from_hash(hash=hash_value, suffix=path_source.suffix)
+    path_file.parent.mkdir(parents=True, exist_ok=True)
+    if not path_file.exists():
+        shutil.copy2(path_source, path_file)
+
+    return path_file
+
+
+def relative_path_from_hash(hash: str, suffix: str) -> Path:
+    path_file = Path("data") / hash[:3] / f"{hash}{suffix}"
+    return path_file
 
 
 def split_sizes_from_ratios(n_items: int, split_ratios: Dict[str, float]) -> Dict[str, int]:

{hafnia-0.2.2 → hafnia-0.2.4}/src/hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -216,6 +216,16 @@ class DatasetRecipe(Serializable):
         json_str = self.as_json_str(indent=indent)
         path_json.write_text(json_str, encoding="utf-8")
 
+    ### Helper methods ###
+    def get_dataset_names(self) -> List[str]:
+        """
+        Get all dataset names added with 'from_name'.
+        Function recursively gathers dataset names.
+        """
+        if self.creation is None:
+            return []
+        return self.creation.get_dataset_names()
+
     ### Validation and Serialization ###
     @field_validator("creation", mode="plain")
     @classmethod
@@ -282,7 +292,10 @@ class FromPath(RecipeCreation):
         return HafniaDataset.from_path
 
     def as_short_name(self) -> str:
-        return f"'{self.path_folder}'".replace(os.sep, "|")
+        return f"'{self.path_folder}'".replace(os.sep, "-")
+
+    def get_dataset_names(self) -> List[str]:
+        return []  # Only counts 'from_name' datasets
 
 
 class FromName(RecipeCreation):
@@ -297,6 +310,9 @@ class FromName(RecipeCreation):
     def as_short_name(self) -> str:
         return self.name
 
+    def get_dataset_names(self) -> List[str]:
+        return [self.name]
+
 
 class FromMerge(RecipeCreation):
     recipe0: DatasetRecipe
@@ -310,6 +326,11 @@ class FromMerge(RecipeCreation):
         merger = FromMerger(recipes=[self.recipe0, self.recipe1])
         return merger.as_short_name()
 
+    def get_dataset_names(self) -> List[str]:
+        """Get the dataset names from the merged recipes."""
+        names = [*self.recipe0.creation.get_dataset_names(), *self.recipe1.creation.get_dataset_names()]
+        return names
+
 
 class FromMerger(RecipeCreation):
     recipes: List[DatasetRecipe]
@@ -325,3 +346,40 @@ class FromMerger(RecipeCreation):
 
     def as_short_name(self) -> str:
         return f"Merger({','.join(recipe.as_short_name() for recipe in self.recipes)})"
+
+    def get_dataset_names(self) -> List[str]:
+        """Get the dataset names from the merged recipes."""
+        names = []
+        for recipe in self.recipes:
+            names.extend(recipe.creation.get_dataset_names())
+        return names
+
+
+def extract_dataset_names_from_json_dict(data: dict) -> list[str]:
+    """
+    Extract dataset names recursively from a JSON dictionary added with 'from_name'.
+
+    Even if the same functionality is achieved with `DatasetRecipe.get_dataset_names()`,
+    we want to keep this function in 'dipdatalib' to extract dataset names from json dictionaries
+    directly.
+    """
+    creation_field = data.get("creation")
+    if creation_field is None:
+        return []
+    if creation_field.get("__type__") == "FromName":
+        return [creation_field["name"]]
+    elif creation_field.get("__type__") == "FromMerge":
+        recipe_names = ["recipe0", "recipe1"]
+        dataset_name = []
+        for recipe_name in recipe_names:
+            recipe = creation_field.get(recipe_name)
+            if recipe is None:
+                continue
+            dataset_name.extend(extract_dataset_names_from_json_dict(recipe))
+        return dataset_name
+    elif creation_field.get("__type__") == "FromMerger":
+        dataset_name = []
+        for recipe in creation_field.get("recipes", []):
+            dataset_name.extend(extract_dataset_names_from_json_dict(recipe))
+        return dataset_name
+    return []

{hafnia-0.2.2 → hafnia-0.2.4}/src/hafnia/dataset/dataset_recipe/recipe_types.py

@@ -108,6 +108,10 @@ class RecipeCreation(Serializable):
     def get_function() -> Callable[..., "HafniaDataset"]:
         pass
 
+    @abstractmethod
+    def get_dataset_names(self) -> List[str]:
+        pass
+
     def build(self) -> "HafniaDataset":
         from hafnia.dataset.dataset_recipe.dataset_recipe import DatasetRecipe
 

{hafnia-0.2.2 → hafnia-0.2.4}/src/hafnia/dataset/hafnia_dataset.py

@@ -411,30 +411,18 @@ class HafniaDataset:
 
         return True
 
-    def write(self, path_folder: Path, name_by_hash: bool = True, add_version: bool = False) -> None:
+    def write(self, path_folder: Path, add_version: bool = False) -> None:
         user_logger.info(f"Writing dataset to {path_folder}...")
         if not path_folder.exists():
             path_folder.mkdir(parents=True)
-        path_folder_images = path_folder / "data"
-        path_folder_images.mkdir(parents=True, exist_ok=True)
 
         new_relative_paths = []
         for org_path in tqdm(self.samples["file_name"].to_list(), desc="- Copy images"):
-            org_path = Path(org_path)
-            if not org_path.exists():
-                raise FileNotFoundError(f"File {org_path} does not exist in the dataset.")
-            if name_by_hash:
-                filename = dataset_helpers.filename_as_hash_from_path(org_path)
-            else:
-                filename = Path(org_path).name
-            new_path = path_folder_images / filename
-            if not new_path.exists():
-                shutil.copy2(org_path, new_path)
-
-            if not new_path.exists():
-                raise FileNotFoundError(f"File {new_path} does not exist in the dataset.")
+            new_path = dataset_helpers.copy_and_rename_file_to_hash_value(
+                path_source=Path(org_path),
+                path_dataset_root=path_folder,
+            )
             new_relative_paths.append(str(new_path.relative_to(path_folder)))
-
         table = self.samples.with_columns(pl.Series(new_relative_paths).alias("file_name"))
         table.write_ndjson(path_folder / FILENAME_ANNOTATIONS_JSONL)  # Json for readability
         table.write_parquet(path_folder / FILENAME_ANNOTATIONS_PARQUET)  # Parquet for speed

{hafnia-0.2.2 → hafnia-0.2.4}/src/hafnia/platform/datasets.py

@@ -1,6 +1,7 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 import uuid
 from pathlib import Path
@@ -62,7 +63,12 @@ def download_or_get_dataset_path(
     dataset_id = get_dataset_id(dataset_name=dataset_name, endpoint=endpoint_dataset, api_key=api_key)
     if dataset_id is None:
         sys_logger.error(f"Dataset '{dataset_name}' not found on the Hafnia platform.")
-    access_dataset_endpoint = f"{endpoint_dataset}/{dataset_id}/temporary-credentials"
+
+    if utils.is_hafnia_cloud_job():
+        credentials_endpoint_suffix = "temporary-credentials-hidden"  # Access to hidden datasets
+    else:
+        credentials_endpoint_suffix = "temporary-credentials"  # Access to sample dataset
+    access_dataset_endpoint = f"{endpoint_dataset}/{dataset_id}/{credentials_endpoint_suffix}"
 
     download_dataset_from_access_endpoint(
         endpoint=access_dataset_endpoint,
@@ -86,22 +92,30 @@ def download_dataset_from_access_endpoint(
     s3_dataset_files = [f"{s3_uri}/{filename}" for filename in DATASET_FILENAMES_REQUIRED]
 
     envs = resource_credentials.aws_credentials()
-    fast_copy_files_s3(
-        src_paths=s3_dataset_files,
-        dst_paths=local_dataset_paths,
-        append_envs=envs,
-        description="Downloading annotations",
-    )
+    try:
+        fast_copy_files_s3(
+            src_paths=s3_dataset_files,
+            dst_paths=local_dataset_paths,
+            append_envs=envs,
+            description="Downloading annotations",
+        )
+    except ValueError as e:
+        user_logger.error(f"Failed to download annotations: {e}")
+        return
 
     if not download_files:
         return
     dataset = HafniaDataset.from_path(path_dataset, check_for_images=False)
-    fast_copy_files_s3(
-        src_paths=dataset.samples[ColumnName.REMOTE_PATH].to_list(),
-        dst_paths=dataset.samples[ColumnName.FILE_NAME].to_list(),
-        append_envs=envs,
-        description="Downloading images",
-    )
+    try:
+        fast_copy_files_s3(
+            src_paths=dataset.samples[ColumnName.REMOTE_PATH].to_list(),
+            dst_paths=dataset.samples[ColumnName.FILE_NAME].to_list(),
+            append_envs=envs,
+            description="Downloading images",
+        )
+    except ValueError as e:
+        user_logger.error(f"Failed to download images: {e}")
+        return
 
 
 def fast_copy_files_s3(
@@ -112,7 +126,6 @@ def fast_copy_files_s3(
 ) -> List[str]:
     if len(src_paths) != len(dst_paths):
         raise ValueError("Source and destination paths must have the same length.")
-
     cmds = [f"cp {src} {dst}" for src, dst in zip(src_paths, dst_paths)]
     lines = execute_s5cmd_commands(cmds, append_envs=append_envs, description=description)
     return lines
@@ -129,11 +142,9 @@ def execute_s5cmd_commands(
     with tempfile.TemporaryDirectory() as temp_dir:
         tmp_file_path = Path(temp_dir, f"{uuid.uuid4().hex}.txt")
         tmp_file_path.write_text("\n".join(commands))
-        run_cmds = [
-            "s5cmd",
-            "run",
-            str(tmp_file_path),
-        ]
+        s5cmd_bin = (Path(sys.executable).parent / "s5cmd").absolute().as_posix()
+        run_cmds = [s5cmd_bin, "run", str(tmp_file_path)]
+        sys_logger.debug(run_cmds)
         envs = os.environ.copy()
         envs.update(append_envs)