hafnia 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

cli/__main__.py

@@ -1,11 +1,21 @@
 #!/usr/bin/env python
 import click
 
-from cli import consts, dataset_cmds, experiment_cmds, profile_cmds, recipe_cmds, runc_cmds
+import hafnia
+from cli import (
+    consts,
+    dataset_cmds,
+    dataset_recipe_cmds,
+    experiment_cmds,
+    profile_cmds,
+    runc_cmds,
+    trainer_package_cmds,
+)
 from cli.config import Config, ConfigSchema
 
 
 @click.group()
+@click.version_option(version=hafnia.__version__)
 @click.pass_context
 def main(ctx: click.Context) -> None:
     """Hafnia CLI."""
@@ -45,7 +55,8 @@ main.add_command(profile_cmds.profile)
 main.add_command(dataset_cmds.dataset)
 main.add_command(runc_cmds.runc)
 main.add_command(experiment_cmds.experiment)
-main.add_command(recipe_cmds.recipe)
+main.add_command(trainer_package_cmds.trainer_package)
+main.add_command(dataset_recipe_cmds.dataset_recipe)
 
 if __name__ == "__main__":
     main(max_content_width=120)

cli/config.py

@@ -9,7 +9,8 @@ import cli.consts as consts
 from hafnia.log import sys_logger, user_logger
 
 PLATFORM_API_MAPPING = {
-    "recipes": "/api/v1/recipes",
+    "trainers": "/api/v1/trainers",
+    "dataset_recipes": "/api/v1/dataset-recipes",
     "experiments": "/api/v1/experiments",
     "experiment_environments": "/api/v1/experiment-environments",
     "experiment_runs": "/api/v1/experiment-runs",

cli/consts.py

@@ -10,7 +10,7 @@ ERROR_CREATE_PROFILE: str = "Failed to create profile. Profile name must be uniq
 ERROR_GET_RESOURCE: str = "Failed to get the data from platform. Verify url or api key."
 
 ERROR_EXPERIMENT_DIR: str = "Source directory does not exist"
-ERROR_RECIPE_FILE_FORMAT: str = "Recipe filename must be a '.zip' file"
+ERROR_TRAINER_PACKAGE_FILE_FORMAT: str = "Trainer package must be a '.zip' file"
 
 PROFILE_SWITCHED_SUCCESS: str = "Switched to profile:"
 PROFILE_REMOVED_SUCCESS: str = "Removed profile:"

cli/dataset_cmds.py

@@ -2,12 +2,10 @@ from pathlib import Path
 from typing import Optional
 
 import click
-from rich import print as rprint
 
-import cli.consts as consts
+from cli import consts
 from cli.config import Config
 from hafnia import utils
-from hafnia.platform.datasets import create_rich_table_from_dataset
 
 
 @click.group()
@@ -18,18 +16,12 @@ def dataset():
 
 @dataset.command("ls")
 @click.pass_obj
-def dataset_list(cfg: Config) -> None:
+def cmd_list_datasets(cfg: Config) -> None:
     """List available datasets on Hafnia platform"""
+    from hafnia.platform.datasets import get_datasets, pretty_print_datasets
 
-    from hafnia.platform.datasets import dataset_list
-
-    try:
-        datasets = dataset_list(cfg=cfg)
-    except Exception:
-        raise click.ClickException(consts.ERROR_GET_RESOURCE)
-
-    table = create_rich_table_from_dataset(datasets)
-    rprint(table)
+    datasets = get_datasets(cfg=cfg)
+    pretty_print_datasets(datasets)
 
 
 @dataset.command("download")
@@ -43,7 +35,7 @@ def dataset_list(cfg: Config) -> None:
 )
 @click.option("--force", "-f", is_flag=True, default=False, help="Flag to enable force redownload")
 @click.pass_obj
-def data_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> Path:
+def cmd_dataset_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> Path:
     """Download dataset from Hafnia platform"""
 
     from hafnia.platform import datasets

cli/dataset_recipe_cmds.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+from typing import Dict, Optional
+
+import click
+from rich import print as rprint
+
+from cli.config import Config
+
+
+@click.group(name="dataset-recipe")
+def dataset_recipe() -> None:
+    """Dataset recipe commands"""
+    pass
+
+
+@dataset_recipe.command(name="create")
+@click.argument("path_json_recipe", required=True)
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    default=None,
+    show_default=True,
+    help="Name of the dataset recipe.",
+)
+@click.pass_obj
+def cmd_get_or_create_dataset_recipe(cfg: Config, path_json_recipe: Path, name: Optional[str]) -> None:
+    """Create Hafnia dataset recipe from dataset recipe JSON file"""
+    from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe_from_path
+
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    recipe = get_or_create_dataset_recipe_from_path(path_json_recipe, endpoint=endpoint, api_key=cfg.api_key, name=name)
+
+    if recipe is None:
+        raise click.ClickException("Failed to create dataset recipe.")
+
+    rprint(recipe)
+
+
+@dataset_recipe.command(name="ls")
+@click.pass_obj
+@click.option("-l", "--limit", type=int, default=None, help="Limit number of listed dataset recipes.")
+def cmd_list_dataset_recipes(cfg: Config, limit: Optional[int]) -> None:
+    """List available dataset recipes"""
+    from hafnia.platform.dataset_recipe import get_dataset_recipes, pretty_print_dataset_recipes
+
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    recipes = get_dataset_recipes(endpoint=endpoint, api_key=cfg.api_key)
+    # Sort recipes to have the most recent first
+    recipes = sorted(recipes, key=lambda x: x["created_at"], reverse=True)
+    if limit is not None:
+        recipes = recipes[:limit]
+    pretty_print_dataset_recipes(recipes)
+
+
+@dataset_recipe.command(name="rm")
+@click.option("-i", "--id", type=str, help="Dataset recipe ID to delete.")
+@click.option("-n", "--name", type=str, help="Dataset recipe name to delete.")
+@click.pass_obj
+def cmd_delete_dataset_recipe(cfg: Config, id: Optional[str], name: Optional[str]) -> Dict:
+    """Delete a dataset recipe by ID or name"""
+    from hafnia.platform.dataset_recipe import delete_dataset_recipe_by_id, delete_dataset_recipe_by_name
+
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+
+    if id is not None:
+        return delete_dataset_recipe_by_id(id=id, endpoint=endpoint, api_key=cfg.api_key)
+    if name is not None:
+        dataset_recipe = delete_dataset_recipe_by_name(name=name, endpoint=endpoint, api_key=cfg.api_key)
+        if dataset_recipe is None:
+            raise click.ClickException(f"Dataset recipe with name '{name}' was not found.")
+
+        return dataset_recipe
+
+    raise click.MissingParameter(
+        "No dataset recipe identifier have been given. Provide either --id or --name. "
+        "Get available recipes with 'hafnia dataset-recipe ls'."
+    )

cli/experiment_cmds.py

@@ -1,10 +1,16 @@
 from pathlib import Path
+from typing import Dict, Optional
 
 import click
-from rich import print as rprint
 
-import cli.consts as consts
 from cli.config import Config
+from hafnia import utils
+from hafnia.platform.dataset_recipe import (
+    get_dataset_recipe_by_id,
+    get_dataset_recipe_by_name,
+    get_or_create_dataset_recipe_by_dataset_name,
+)
+from hafnia.platform.trainer_package import create_trainer_package
 
 
 @click.group(name="experiment")
@@ -13,48 +19,225 @@ def experiment() -> None:
     pass
 
 
+@experiment.command(name="environments")
+@click.pass_obj
+def cmd_view_environments(cfg: Config):
+    """
+    View available experiment training environments.
+    """
+    from hafnia.platform import get_environments, pretty_print_training_environments
+
+    envs = get_environments(cfg.get_platform_endpoint("experiment_environments"), cfg.api_key)
+
+    pretty_print_training_environments(envs)
+
+
+def default_experiment_run_name():
+    return f"run-{utils.now_as_str()}"
+
+
 @experiment.command(name="create")
-@click.argument("name")
-@click.argument("source_dir", type=Path)
-@click.argument("exec_cmd", type=str)
-@click.argument("dataset_name")
-@click.argument("env_name")
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    default=default_experiment_run_name(),
+    required=False,
+    help=f"Name of the experiment. [default: run-[DATETIME] e.g. {default_experiment_run_name()}] ",
+)
+@click.option(
+    "-c",
+    "--cmd",
+    type=str,
+    default="python scripts/train.py",
+    show_default=True,
+    help="Command to run the experiment.",
+)
+@click.option(
+    "-p",
+    "--trainer-path",
+    type=Path,
+    default=None,
+    help="Path to the trainer package directory. ",
+)
+@click.option(
+    "-i",
+    "--trainer-id",
+    type=str,
+    default=None,
+    help="ID of the trainer package. View available trainers with 'hafnia trainer ls'",
+)
+@click.option(
+    "-d",
+    "--dataset",
+    type=str,
+    default=None,
+    required=False,
+    help="DatasetIdentifier: Name of the dataset. View Available datasets with 'hafnia dataset ls'",
+)
+@click.option(
+    "-r",
+    "--dataset-recipe",
+    type=str,
+    default=None,
+    required=False,
+    help="DatasetIdentifier: Name of the dataset recipe. View available dataset recipes with 'hafnia dataset-recipe ls'",
+)
+@click.option(
+    "--dataset-recipe-id",
+    type=str,
+    default=None,
+    required=False,
+    help="DatasetIdentifier: ID of the dataset recipe. View dataset recipes with 'hafnia dataset-recipe ls'",
+)
+@click.option(
+    "-e",
+    "--environment",
+    type=str,
+    default="Free Tier",
+    show_default=True,
+    help="Experiment environment name. View available environments with 'hafnia experiment environments'",
+)
 @click.pass_obj
-def create(cfg: Config, name: str, source_dir: Path, exec_cmd: str, dataset_name: str, env_name: str) -> None:
-    """Create a new experiment run"""
-    from hafnia.platform import create_experiment, create_recipe, get_dataset_id, get_exp_environment_id
-
-    if not source_dir.exists():
-        raise click.ClickException(consts.ERROR_EXPERIMENT_DIR)
-
-    try:
-        dataset_id = get_dataset_id(dataset_name, cfg.get_platform_endpoint("datasets"), cfg.api_key)
-    except Exception:
-        raise click.ClickException(f"Error retrieving dataset '{dataset_name}'.")
-
-    try:
-        recipe_id = create_recipe(source_dir, cfg.get_platform_endpoint("recipes"), cfg.api_key)
-    except Exception:
-        raise click.ClickException(f"Failed to create recipe from '{source_dir}'")
-
-    try:
-        env_id = get_exp_environment_id(env_name, cfg.get_platform_endpoint("experiment_environments"), cfg.api_key)
-    except Exception:
-        raise click.ClickException(f"Environment '{env_name}' not found")
-
-    try:
-        experiment_id = create_experiment(
-            name, dataset_id, recipe_id, exec_cmd, env_id, cfg.get_platform_endpoint("experiments"), cfg.api_key
+def cmd_create_experiment(
+    cfg: Config,
+    name: str,
+    cmd: str,
+    trainer_path: Path,
+    trainer_id: Optional[str],
+    dataset: Optional[str],
+    dataset_recipe: Optional[str],
+    dataset_recipe_id: Optional[str],
+    environment: str,
+) -> None:
+    """
+    Create and launch a new experiment run
+
+    Requires one dataset recipe and one trainer package:.
+        - One dataset identifier is required either '--dataset', '--dataset-recipe' or '--dataset-recipe-id'.
+        - One trainer identifier is required either '--trainer-path' or '--trainer-id'.
+
+    \b
+    Examples:
+    # Launch an experiment with a dataset and a trainer package from local path
+    hafnia experiment create --dataset mnist --trainer-path ../trainer-classification
+
+    \b
+    # Launch experiment with dataset recipe by name and trainer package by id
+    hafnia experiment create --dataset-recipe mnist-recipe --trainer-id 5e454c0d-fdf1-4d1f-9732-771d7fecd28e
+
+    \b
+    # Show available options:
+    hafnia experiment create --name "My Experiment" -d mnist --cmd "python scripts/train.py" -e "Free Tier" -p ../trainer-classification
+    """
+    from hafnia.platform import create_experiment, get_exp_environment_id
+
+    dataset_recipe_response = get_dataset_recipe_by_dataset_identifies(
+        cfg=cfg,
+        dataset_name=dataset,
+        dataset_recipe_name=dataset_recipe,
+        dataset_recipe_id=dataset_recipe_id,
+    )
+    dataset_recipe_id = dataset_recipe_response["id"]
+
+    trainer_id = get_trainer_package_by_identifies(
+        cfg=cfg,
+        trainer_path=trainer_path,
+        trainer_id=trainer_id,
+    )
+
+    env_id = get_exp_environment_id(environment, cfg.get_platform_endpoint("experiment_environments"), cfg.api_key)
+
+    experiment = create_experiment(
+        experiment_name=name,
+        dataset_recipe_id=dataset_recipe_id,
+        trainer_id=trainer_id,
+        exec_cmd=cmd,
+        environment_id=env_id,
+        endpoint=cfg.get_platform_endpoint("experiments"),
+        api_key=cfg.api_key,
+    )
+
+    experiment_properties = {
+        "ID": experiment.get("id", "N/A"),
+        "Name": experiment.get("name", "N/A"),
+        "State": experiment.get("state", "N/A"),
+        "Trainer Package ID": experiment.get("trainer", "N/A"),
+        "Dataset Recipe ID": experiment.get("dataset_recipe", "N/A"),
+        "Dataset ID": experiment.get("dataset", "N/A"),
+        "Created At": experiment.get("created_at", "N/A"),
+    }
+    print("Successfully created experiment: ")
+    for key, value in experiment_properties.items():
+        print(f"  {key}: {value}")
+
+
+def get_dataset_recipe_by_dataset_identifies(
+    cfg: Config,
+    dataset_name: Optional[str],
+    dataset_recipe_name: Optional[str],
+    dataset_recipe_id: Optional[str],
+) -> Dict:
+    dataset_identifiers = [dataset_name, dataset_recipe_name, dataset_recipe_id]
+    n_dataset_identifies_defined = sum([bool(identifier) for identifier in dataset_identifiers])
+
+    if n_dataset_identifies_defined > 1:
+        raise click.ClickException(
+            "Multiple dataset identifiers have been provided. Define only one dataset identifier."
+        )
+
+    dataset_recipe_endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    if dataset_name:
+        return get_or_create_dataset_recipe_by_dataset_name(dataset_name, dataset_recipe_endpoint, cfg.api_key)
+
+    if dataset_recipe_name:
+        recipe = get_dataset_recipe_by_name(dataset_recipe_name, dataset_recipe_endpoint, cfg.api_key)
+        if recipe is None:
+            raise click.ClickException(f"Dataset recipe '{dataset_recipe_name}' was not found in the dataset library.")
+        return recipe
+
+    if dataset_recipe_id:
+        return get_dataset_recipe_by_id(dataset_recipe_id, dataset_recipe_endpoint, cfg.api_key)
+
+    raise click.MissingParameter(
+        "At least one dataset identifier must be provided. Set one of the following:\n"
+        "  --dataset <name>  -- E.g. '--dataset mnist'\n"
+        "  --dataset-recipe <name>  -- E.g. '--dataset-recipe my-recipe'\n"
+        "  --dataset-recipe-id <id>  -- E.g. '--dataset-recipe-id 5e454c0d-fdf1-4d1f-9732-771d7fecd28e'\n"
+    )
+
+
+def get_trainer_package_by_identifies(
+    cfg: Config,
+    trainer_path: Optional[Path],
+    trainer_id: Optional[str],
+) -> str:
+    from hafnia.platform import get_trainer_package_by_id
+
+    if trainer_path is not None and trainer_id is not None:
+        raise click.ClickException(
+            "Multiple trainer identifiers (--trainer-path, --trainer-id) have been provided. Define only one."
         )
-    except Exception:
-        raise click.ClickException(f"Failed to create experiment '{name}'")
-
-    rprint(
-        {
-            "dataset_id": dataset_id,
-            "recipe_id": recipe_id,
-            "environment_id": env_id,
-            "experiment_id": experiment_id,
-            "status": "CREATED",
-        }
+
+    if trainer_path is not None:
+        trainer_path = Path(trainer_path)
+        if not trainer_path.exists():
+            raise click.ClickException(f"Trainer package path '{trainer_path}' does not exist.")
+        trainer_id = create_trainer_package(
+            trainer_path,
+            cfg.get_platform_endpoint("trainers"),
+            cfg.api_key,
+        )
+        return trainer_id
+
+    if trainer_id:
+        trainer_response = get_trainer_package_by_id(
+            id=trainer_id, endpoint=cfg.get_platform_endpoint("trainers"), api_key=cfg.api_key
+        )
+        return trainer_response["id"]
+
+    raise click.MissingParameter(
+        "At least one trainer identifier must be provided. Set one of the following:\n"
+        "  --trainer-path <path>  -- E.g. '--trainer-path .'\n"
+        "  --trainer-id <id>  -- E.g. '--trainer-id 5e454c0d-fdf1-4d1f-9732-771d7fecd28e'\n"
     )

cli/profile_cmds.py

@@ -14,7 +14,7 @@ def profile():
 
 @profile.command("ls")
 @click.pass_obj
-def profile_ls(cfg: Config) -> None:
+def cmd_profile_ls(cfg: Config) -> None:
     """List all available profiles."""
     profiles = cfg.available_profiles
     if not profiles:
@@ -31,7 +31,7 @@ def profile_ls(cfg: Config) -> None:
 @profile.command("use")
 @click.argument("profile_name", required=True)
 @click.pass_obj
-def profile_use(cfg: Config, profile_name: str) -> None:
+def cmd_profile_use(cfg: Config, profile_name: str) -> None:
     """Switch to a different profile."""
     if len(cfg.available_profiles) == 0:
         raise click.ClickException(consts.ERROR_CONFIGURE)
@@ -51,7 +51,7 @@ def profile_use(cfg: Config, profile_name: str) -> None:
     "--activate/--no-activate", help="Activate the created profile after creation", default=True, show_default=True
 )
 @click.pass_obj
-def profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate: bool) -> None:
+def cmd_profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate: bool) -> None:
     """Create a new profile."""
     cfg_profile = ConfigSchema(platform_url=api_url, api_key=api_key)
 
@@ -62,7 +62,7 @@ def profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate:
 @profile.command("rm")
 @click.argument("profile_name", required=True)
 @click.pass_obj
-def profile_rm(cfg: Config, profile_name: str) -> None:
+def cmd_profile_rm(cfg: Config, profile_name: str) -> None:
     """Remove a profile."""
     if len(cfg.available_profiles) == 0:
         raise click.ClickException(consts.ERROR_CONFIGURE)
@@ -80,7 +80,8 @@ def profile_rm(cfg: Config, profile_name: str) -> None:
 
 @profile.command("active")
 @click.pass_obj
-def profile_active(cfg: Config) -> None:
+def cmd_profile_active(cfg: Config) -> None:
+    """Show the currently active profile."""
     try:
         profile_show(cfg)
     except Exception as e:

cli/runc_cmds.py

@@ -13,7 +13,7 @@ from hafnia.log import sys_logger, user_logger
 
 @click.group(name="runc")
 def runc():
-    """Experiment management commands"""
+    """Creating and running trainer packages locally"""
     pass
 
 
@@ -90,10 +90,10 @@ def launch_local(cfg: Config, exec_cmd: str, dataset: str, image_name: str) -> N
 @click.pass_obj
 def build(cfg: Config, recipe_url: str, state_file: str, repo: str) -> None:
     """Build docker image with a given recipe."""
-    from hafnia.platform.builder import build_image, prepare_recipe
+    from hafnia.platform.builder import build_image, prepare_trainer_package
 
     with TemporaryDirectory() as temp_dir:
-        metadata = prepare_recipe(recipe_url, Path(temp_dir), cfg.api_key)
+        metadata = prepare_trainer_package(recipe_url, Path(temp_dir), cfg.api_key)
         build_image(metadata, repo, state_file=state_file)
 
 
@@ -109,7 +109,7 @@ def build_local(recipe: Path, state_file: str, repo: str) -> None:
     import seedir
 
     from hafnia.platform.builder import build_image
-    from hafnia.utils import filter_recipe_files
+    from hafnia.utils import filter_trainer_package_files
 
     recipe = Path(recipe)
 
@@ -123,7 +123,7 @@ def build_local(recipe: Path, state_file: str, repo: str) -> None:
             with zipfile.ZipFile(recipe.as_posix(), "r") as zip_ref:
                 zip_ref.extractall(recipe_dir)
         elif recipe.is_dir():
-            for rf in filter_recipe_files(recipe):
+            for rf in filter_trainer_package_files(recipe):
                 src_path = (recipe / rf).absolute()
                 target_path = recipe_dir / rf
                 target_path.parent.mkdir(parents=True, exist_ok=True)

cli/trainer_package_cmds.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+from typing import Optional
+
+import click
+
+import cli.consts as consts
+from cli.config import Config
+
+
+@click.group(name="trainer")
+def trainer_package() -> None:
+    """Trainer package commands"""
+    pass
+
+
+@trainer_package.command(name="ls")
+@click.pass_obj
+@click.option("-l", "--limit", type=int, default=None, help="Limit number of listed trainer packages.")
+def cmd_list_trainer_packages(cfg: Config, limit: Optional[int]) -> None:
+    """List available trainer packages on the platform"""
+
+    from hafnia.platform.trainer_package import get_trainer_packages, pretty_print_trainer_packages
+
+    endpoint = cfg.get_platform_endpoint("trainers")
+    trainers = get_trainer_packages(endpoint, cfg.api_key)
+
+    pretty_print_trainer_packages(trainers, limit=limit)
+
+
+@trainer_package.command(name="create-zip")
+@click.argument("source")
+@click.option(
+    "--output",
+    type=click.Path(writable=True),
+    default="./trainer.zip",
+    show_default=True,
+    help="Output trainer package path.",
+)
+def cmd_create_trainer_package_zip(source: str, output: str) -> None:
+    """Create Hafnia trainer package as zip-file from local path"""
+
+    from hafnia.utils import archive_dir
+
+    path_output_zip = Path(output)
+    if path_output_zip.suffix != ".zip":
+        raise click.ClickException(consts.ERROR_TRAINER_PACKAGE_FILE_FORMAT)
+
+    path_source = Path(source)
+    path_output_zip = archive_dir(path_source, path_output_zip)
+
+
+@trainer_package.command(name="view-zip")
+@click.option("--path", type=str, default="./trainer.zip", show_default=True, help="Path of trainer.zip.")
+@click.option("--depth-limit", type=int, default=3, help="Limit the depth of the tree view.", show_default=True)
+def cmd_view_trainer_package_zip(path: str, depth_limit: int) -> None:
+    """View the content of a trainer package zip file."""
+    from hafnia.utils import show_trainer_package_content
+
+    path_trainer_package = Path(path)
+    if not path_trainer_package.exists():
+        raise click.ClickException(
+            f"Trainer package file '{path_trainer_package}' does not exist. Please provide a valid path. "
+            f"To create a trainer package, use the 'hafnia trainer create-zip' command."
+        )
+    show_trainer_package_content(path_trainer_package, depth_limit=depth_limit)

hafnia/__init__.py

@@ -2,3 +2,5 @@ from importlib.metadata import version
 
 __package_name__ = "hafnia"
 __version__ = version(__package_name__)
+
+__dataset_format_version__ = "0.0.2"  # Hafnia dataset format version

hafnia/data/factory.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 from typing import Any
 
@@ -16,7 +15,7 @@ def load_dataset(recipe: Any, force_redownload: bool = False) -> HafniaDataset:
 
 def get_dataset_path(recipe: Any, force_redownload: bool = False) -> Path:
     if utils.is_hafnia_cloud_job():
-        return Path(os.getenv("MDI_DATASET_DIR", "/opt/ml/input/data/training"))
+        return utils.get_dataset_path_in_hafnia_cloud()
 
     path_dataset = get_or_create_dataset_path_from_recipe(recipe, force_redownload=force_redownload)
 

hafnia/dataset/dataset_helpers.py

@@ -110,15 +110,3 @@ def split_sizes_from_ratios(n_items: int, split_ratios: Dict[str, float]) -> Dic
         raise ValueError("Something is wrong. The split sizes do not match the number of items.")
 
     return split_sizes
-
-
-def select_evenly_across_list(lst: list, num_samples: int):
-    if num_samples >= len(lst):
-        return lst  # No need to sample
-    step = (len(lst) - 1) / (num_samples - 1)
-    indices = [int(round(step * i)) for i in range(num_samples)]  # noqa: RUF046
-    return [lst[index] for index in indices]
-
-
-def prefix_dict(d: dict, prefix: str) -> dict:
-    return {f"{prefix}.{k}": v for k, v in d.items()}