hafnia 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

hafnia/dataset/dataset_details_uploader.py

@@ -294,9 +294,9 @@ def upload_dataset_details_to_platform(
     if update_platform:
         dataset_details_exclude_none = dataset_details.model_dump(exclude_none=True, mode="json")
         upload_dataset_details(
-            cfg=cfg,
             data=dataset_details_exclude_none,
             dataset_name=dataset_details.name,
+            cfg=cfg,
         )
 
     dataset_details_dict = dataset_details.model_dump(exclude_none=False, mode="json")

hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import shutil
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 
@@ -12,6 +13,7 @@ from pydantic import (
 
 from hafnia import utils
 from hafnia.dataset.dataset_helpers import dataset_name_and_version_from_string
+from hafnia.dataset.dataset_names import FILENAME_RECIPE_JSON
 from hafnia.dataset.dataset_recipe import recipe_transforms
 from hafnia.dataset.dataset_recipe.recipe_types import (
     RecipeCreation,
@@ -22,6 +24,7 @@ from hafnia.dataset.hafnia_dataset import (
     HafniaDataset,
     available_dataset_versions_from_name,
 )
+from hafnia.dataset.hafnia_dataset_types import DatasetMetadataFilePaths
 from hafnia.dataset.primitives.primitive import Primitive
 from hafnia.log import user_logger
 
@@ -117,6 +120,23 @@ class DatasetRecipe(Serializable):
         json_str = path_json.read_text(encoding="utf-8")
         return DatasetRecipe.from_json_str(json_str)
 
+    @staticmethod
+    def from_recipe_field(recipe_field: Union[str, Dict[str, Any]]) -> "DatasetRecipe":
+        """
+
+        Deserialize from a recipe field which can be either a string or a dictionary.
+
+        string: A dataset name and version string in the format 'name:version'.
+        dict: A dictionary representation of the DatasetRecipe.
+
+        """
+        if isinstance(recipe_field, str):
+            return DatasetRecipe.from_name_and_version_string(recipe_field)
+        elif isinstance(recipe_field, dict):
+            return DatasetRecipe.from_dict(recipe_field)
+
+        raise TypeError(f"Expected str or dict for recipe_field, got {type(recipe_field).__name__}.")
+
     @staticmethod
     def from_dict(data: Dict[str, Any]) -> "DatasetRecipe":
         """Deserialize from a dictionary."""
@@ -130,14 +150,9 @@ class DatasetRecipe(Serializable):
         from hafnia_cli.config import Config
 
         cfg = Config()
-        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
-        recipe_dict = get_dataset_recipe_by_id(recipe_id, endpoint=endpoint_dataset, api_key=cfg.api_key)
+        recipe_dict = get_dataset_recipe_by_id(recipe_id, cfg=cfg)
         recipe_dict = recipe_dict["template"]["body"]
-        if isinstance(recipe_dict, str):
-            return DatasetRecipe.from_implicit_form(recipe_dict)
-
-        recipe = DatasetRecipe.from_dict(recipe_dict)
-        return recipe
+        return DatasetRecipe.from_recipe_field(recipe_dict)
 
     @staticmethod
     def from_recipe_name(name: str) -> "DatasetRecipe":
@@ -146,8 +161,7 @@ class DatasetRecipe(Serializable):
         from hafnia_cli.config import Config
 
         cfg = Config()
-        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
-        recipe = get_dataset_recipe_by_name(name=name, endpoint=endpoint_dataset, api_key=cfg.api_key)
+        recipe = get_dataset_recipe_by_name(name=name, cfg=cfg)
         if not recipe:
             raise ValueError(f"Dataset recipe '{name}' not found.")
         recipe_id = recipe["id"]
@@ -168,82 +182,6 @@ class DatasetRecipe(Serializable):
 
         return DatasetRecipe.from_name(name=dataset_name, version=version)
 
-    @staticmethod
-    def from_implicit_form(recipe: Any) -> DatasetRecipe:
-        """
-        Recursively convert from implicit recipe to explicit form.
-        Handles mixed implicit/explicit recipes.
-
-        Conversion rules:
-        - str: Will get a dataset by name -> DatasetRecipeFromName
-        - Path: Will get a dataset from path -> DatasetRecipeFromPath
-        - tuple: Will merge datasets specified in the tuple -> RecipeMerger
-        - list: Will define a list of transformations -> RecipeTransforms
-
-        Example: DataRecipe from dataset name:
-        ```python
-        recipe_implicit = "mnist"
-        recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
-        >>> recipe_explicit
-        DatasetRecipeFromName(dataset_name='mnist', force_redownload=False)
-        ```
-
-        Example: DataRecipe from tuple (merging multiple recipes):
-        ```python
-        recipe_implicit = ("dataset1", "dataset2")
-        recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
-        >>> recipe_explicit
-        RecipeMerger(
-            recipes=[
-                DatasetRecipeFromName(dataset_name='dataset1', force_redownload=False),
-                DatasetRecipeFromName(dataset_name='dataset2', force_redownload=False)
-            ]
-        )
-
-        Example: DataRecipe from list (recipe and transformations):
-        ```python
-        recipe_implicit = ["mnist", SelectSamples(n_samples=20), Shuffle(seed=123)]
-        recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
-        >>> recipe_explicit
-        Transforms(
-            recipe=DatasetRecipeFromName(dataset_name='mnist', force_redownload=False),
-            transforms=[SelectSamples(n_samples=20), Shuffle(seed=123)]
-        )
-        ```
-
-        """
-        if isinstance(recipe, DatasetRecipe):  # type: ignore
-            # It is possible to do an early return if recipe is a 'DataRecipe'-type even for nested and
-            # potentially mixed recipes. If you (really) think about it, this might surprise you,
-            # as this will bypass the conversion logic for nested recipes.
-            # However, this is not a problem as 'DataRecipe' classes are also pydantic models,
-            # so if a user introduces a 'DataRecipe'-class in the recipe (in potentially
-            # some nested and mixed implicit/explicit form) it will (due to pydantic validation) force
-            # the user to specify all nested recipes to be converted to explicit form.
-            return recipe
-
-        if isinstance(recipe, str):  # str-type is convert to DatasetFromName
-            return DatasetRecipe.from_name_and_version_string(string=recipe, resolve_missing_version=True)
-
-        if isinstance(recipe, Path):  # Path-type is convert to DatasetFromPath
-            return DatasetRecipe.from_path(path_folder=recipe)
-
-        if isinstance(recipe, tuple):  # tuple-type is convert to DatasetMerger
-            recipes = [DatasetRecipe.from_implicit_form(item) for item in recipe]
-            return DatasetRecipe.from_merger(recipes=recipes)
-
-        if isinstance(recipe, list):  # list-type is convert to Transforms
-            if len(recipe) == 0:
-                raise ValueError("List of recipes cannot be empty")
-
-            dataset_recipe = recipe[0]  # First element is the dataset recipe
-            loader = DatasetRecipe.from_implicit_form(dataset_recipe)
-
-            transforms = recipe[1:]  # Remaining items are transformations
-            return DatasetRecipe(creation=loader.creation, operations=transforms)
-
-        raise ValueError(f"Unsupported recipe type: {type(recipe)}")
-
     ### Upload, store and recipe conversions ###
     def as_python_code(self, keep_default_fields: bool = False, as_kwargs: bool = True) -> str:
         str_operations = [self.creation.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs)]
@@ -285,17 +223,10 @@ class DatasetRecipe(Serializable):
         from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe
         from hafnia_cli.config import Config
 
-        recipe = self.as_dict()
         cfg = Config()
-        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
-        recipe_dict = get_or_create_dataset_recipe(
-            recipe=recipe,
-            endpoint=endpoint_dataset,
-            api_key=cfg.api_key,
-            name=recipe_name,
-            overwrite=overwrite,
-        )
 
+        recipe = self.as_dict()
+        recipe_dict = get_or_create_dataset_recipe(recipe=recipe, name=recipe_name, overwrite=overwrite, cfg=cfg)
         return recipe_dict
 
     ### Dataset Recipe Transformations ###
@@ -428,13 +359,6 @@ def unique_name_from_recipe(recipe: DatasetRecipe) -> str:
     return unique_name
 
 
-def get_dataset_path_from_recipe(recipe: DatasetRecipe, path_datasets: Optional[Union[Path, str]] = None) -> Path:
-    path_datasets = path_datasets or utils.PATH_DATASETS
-    path_datasets = Path(path_datasets)
-    unique_dataset_name = unique_name_from_recipe(recipe)
-    return path_datasets / unique_dataset_name
-
-
 class FromPath(RecipeCreation):
     path_folder: Path
     check_for_images: bool = True
@@ -525,3 +449,34 @@ class FromMerger(RecipeCreation):
         for recipe in self.recipes:
             names.extend(recipe.creation.get_dataset_names())
         return names
+
+
+def get_dataset_path_from_recipe(recipe: DatasetRecipe, path_datasets: Optional[Union[Path, str]] = None) -> Path:
+    path_datasets = path_datasets or utils.PATH_DATASETS
+    path_datasets = Path(path_datasets)
+    unique_dataset_name = unique_name_from_recipe(recipe)
+    return path_datasets / unique_dataset_name
+
+
+def get_or_create_dataset_path_from_recipe(
+    dataset_recipe: DatasetRecipe,
+    force_redownload: bool = False,
+    path_datasets: Optional[Union[Path, str]] = None,
+) -> Path:
+    path_dataset = get_dataset_path_from_recipe(dataset_recipe, path_datasets=path_datasets)
+
+    if force_redownload:
+        shutil.rmtree(path_dataset, ignore_errors=True)
+
+    dataset_metadata_files = DatasetMetadataFilePaths.from_path(path_dataset)
+    if dataset_metadata_files.exists(raise_error=False):
+        return path_dataset
+
+    path_dataset.mkdir(parents=True, exist_ok=True)
+    path_recipe_json = path_dataset / FILENAME_RECIPE_JSON
+    path_recipe_json.write_text(dataset_recipe.model_dump_json(indent=4))
+
+    dataset: HafniaDataset = dataset_recipe.build()
+    dataset.write(path_dataset)
+
+    return path_dataset

hafnia/dataset/hafnia_dataset.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import copy
-import shutil
 from dataclasses import dataclass
 from pathlib import Path
 from random import Random
@@ -14,7 +13,6 @@ from hafnia import utils
 from hafnia.dataset import dataset_helpers
 from hafnia.dataset.dataset_helpers import is_valid_version_string, version_from_string
 from hafnia.dataset.dataset_names import (
-    FILENAME_RECIPE_JSON,
     TAG_IS_SAMPLE,
     PrimitiveField,
     SampleField,
@@ -146,17 +144,6 @@ class HafniaDataset:
         table = table_transformations.add_dataset_name_if_missing(table, dataset_name=info.dataset_name)
         return HafniaDataset(info=info, samples=table)
 
-    @staticmethod
-    def from_recipe(dataset_recipe: Any) -> "HafniaDataset":
-        """
-        Load a dataset from a recipe. The recipe can be a string (name of the dataset), a dictionary, or a DataRecipe object.
-        """
-        from hafnia.dataset.dataset_recipe.dataset_recipe import DatasetRecipe
-
-        recipe_explicit = DatasetRecipe.from_implicit_form(dataset_recipe)
-
-        return recipe_explicit.build()  # Build dataset from the recipe
-
     @staticmethod
     def from_merge(dataset0: "HafniaDataset", dataset1: "HafniaDataset") -> "HafniaDataset":
         return HafniaDataset.merge(dataset0, dataset1)
@@ -172,6 +159,8 @@ class HafniaDataset:
         If the dataset is already cached, it will be loaded from the cache.
         """
 
+        from hafnia.dataset.dataset_recipe.dataset_recipe import get_or_create_dataset_path_from_recipe
+
         path_dataset = get_or_create_dataset_path_from_recipe(
             dataset_recipe,
             path_datasets=path_datasets,
@@ -245,7 +234,7 @@ class HafniaDataset:
 
         Example: Defining split ratios and applying the transformation
 
-        >>> dataset = HafniaDataset.read_from_path(Path("path/to/dataset"))
+        >>> dataset = HafniaDataset.from_path(Path("path/to/dataset"))
         >>> split_ratios = {SplitName.TRAIN: 0.8, SplitName.VAL: 0.1, SplitName.TEST: 0.1}
         >>> dataset_with_splits = splits_by_ratios(dataset, split_ratios, seed=42)
         Or use the function as a
@@ -270,7 +259,7 @@ class HafniaDataset:
         splits based on the provided ratios.
 
         Example: Defining split ratios and applying the transformation
-        >>> dataset = HafniaDataset.read_from_path(Path("path/to/dataset"))
+        >>> dataset = HafniaDataset.from_path(Path("path/to/dataset"))
         >>> split_name = SplitName.TEST
         >>> split_ratios = {SplitName.TEST: 0.8, SplitName.VAL: 0.2}
         >>> dataset_with_splits = split_into_multiple_splits(dataset, split_name, split_ratios)
@@ -543,7 +532,7 @@ class HafniaDataset:
         primitive: Type[Primitive],
         task_name: Optional[str] = None,
         keep_sample_data: bool = False,
-    ) -> pl.DataFrame:
+    ) -> Optional[pl.DataFrame]:
         return table_transformations.create_primitive_table(
             samples_table=self.samples,
             PrimitiveType=primitive,
@@ -741,36 +730,6 @@ def check_hafnia_dataset_from_path(path_dataset: Path) -> None:
     dataset.check_dataset()
 
 
-def get_or_create_dataset_path_from_recipe(
-    dataset_recipe: Any,
-    force_redownload: bool = False,
-    path_datasets: Optional[Union[Path, str]] = None,
-) -> Path:
-    from hafnia.dataset.dataset_recipe.dataset_recipe import (
-        DatasetRecipe,
-        get_dataset_path_from_recipe,
-    )
-
-    recipe: DatasetRecipe = DatasetRecipe.from_implicit_form(dataset_recipe)
-    path_dataset = get_dataset_path_from_recipe(recipe, path_datasets=path_datasets)
-
-    if force_redownload:
-        shutil.rmtree(path_dataset, ignore_errors=True)
-
-    dataset_metadata_files = DatasetMetadataFilePaths.from_path(path_dataset)
-    if dataset_metadata_files.exists(raise_error=False):
-        return path_dataset
-
-    path_dataset.mkdir(parents=True, exist_ok=True)
-    path_recipe_json = path_dataset / FILENAME_RECIPE_JSON
-    path_recipe_json.write_text(recipe.model_dump_json(indent=4))
-
-    dataset: HafniaDataset = recipe.build()
-    dataset.write(path_dataset)
-
-    return path_dataset
-
-
 def available_dataset_versions_from_name(dataset_name: str) -> Dict[Version, "DatasetMetadataFilePaths"]:
     credentials: ResourceCredentials = get_read_credentials_by_name(dataset_name=dataset_name)
     return available_dataset_versions(credentials=credentials)
@@ -795,12 +754,13 @@ def select_version_from_available_versions(
 
     if version is None:
         str_versions = [str(v) for v in available_versions]
-        raise ValueError(f"Version must be specified. Available versions: {str_versions}")
-    elif version == "latest":
+        raise ValueError(f"Version must be specified. Available versions: {str_versions}. ")
+
+    if version == "latest":
         version_casted = max(available_versions)
         user_logger.info(f"'latest' version '{version_casted}' has been selected")
     else:
-        version_casted = version_from_string(version)
+        version_casted = version_from_string(version, raise_error=True)
 
     if version_casted not in available_versions:
         raise ValueError(f"Selected version '{version}' not found in available versions: {available_versions}")

hafnia/dataset/hafnia_dataset_types.py

@@ -470,7 +470,7 @@ class Sample(BaseModel):
         return image
 
     def draw_annotations(self, image: Optional[np.ndarray] = None) -> np.ndarray:
-        from hafnia.visualizations import image_visualizations
+        from hafnia.dataset import image_visualizations
 
         if image is None:
             image = self.read_image()

hafnia/dataset/operations/dataset_stats.py

@@ -104,7 +104,8 @@ def calculate_primitive_counts(dataset: HafniaDataset) -> Dict[str, int]:
         name = task.primitive.__name__
         if task.name != task.primitive.default_task_name():
             name = f"{name}.{task.name}"
-        annotation_counts[name] = len(objects)
+        n_objects = 0 if objects is None else len(objects)
+        annotation_counts[name] = n_objects
     return annotation_counts
 
 

hafnia/dataset/primitives/classification.py

@@ -39,7 +39,7 @@ class Classification(Primitive):
     def draw(self, image: np.ndarray, inplace: bool = False, draw_label: bool = True) -> np.ndarray:
         if draw_label is False:
             return image
-        from hafnia.visualizations import image_visualizations
+        from hafnia.dataset import image_visualizations
 
         class_name = self.get_class_name()
         if self.task_name == self.default_task_name():

hafnia/dataset/primitives/segmentation.py

@@ -4,9 +4,9 @@ import cv2
 import numpy as np
 from pydantic import Field
 
+from hafnia.dataset.colors import get_n_colors
 from hafnia.dataset.primitives.primitive import Primitive
 from hafnia.dataset.primitives.utils import get_class_name
-from hafnia.visualizations.colors import get_n_colors
 
 
 class Segmentation(Primitive):

hafnia/experiment/command_builder.py

@@ -103,7 +103,7 @@ def auto_save_command_builder_schema(
         bool_handling=bool_handling,
     )
 
-    path_schema = path_schema or path_of_function(cli_function).with_suffix(".json")
+    path_schema = path_schema or path_of_function(cli_function).with_suffix(".schema.json")
     launch_schema.to_json_file(path_schema)
     return path_schema
 

hafnia/platform/dataset_recipe.py

@@ -7,17 +7,20 @@ from flatten_dict import flatten
 from hafnia import http
 from hafnia.log import user_logger
 from hafnia.utils import pretty_print_list_as_table, timed
+from hafnia_cli.config import Config
 
 
 @timed("Get or create dataset recipe")
 def get_or_create_dataset_recipe(
     recipe: dict,
-    endpoint: str,
-    api_key: str,
     name: Optional[str] = None,
     overwrite: bool = False,
+    cfg: Optional[Config] = None,
 ) -> Optional[Dict]:
-    headers = {"Authorization": api_key}
+    cfg = cfg or Config()
+
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    headers = {"Authorization": cfg.api_key}
     data = {"template": {"body": recipe}, "overwrite": overwrite}
     if name is not None:
         data["name"] = name  # type: ignore[assignment]
@@ -26,18 +29,22 @@ def get_or_create_dataset_recipe(
     return response
 
 
-def get_or_create_dataset_recipe_by_dataset_name(dataset_name: str, endpoint: str, api_key: str) -> Dict:
-    return get_or_create_dataset_recipe(recipe=dataset_name, endpoint=endpoint, api_key=api_key)
+def get_or_create_dataset_recipe_by_dataset_name(dataset_name: str, cfg: Optional[Config] = None) -> Dict:
+    return get_or_create_dataset_recipe(recipe=dataset_name, cfg=cfg)
 
 
-def get_dataset_recipes(endpoint: str, api_key: str) -> List[Dict]:
-    headers = {"Authorization": api_key}
+def get_dataset_recipes(cfg: Optional[Config] = None) -> List[Dict]:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    headers = {"Authorization": cfg.api_key}
     dataset_recipes: List[Dict] = http.fetch(endpoint, headers=headers)  # type: ignore[assignment]
     return dataset_recipes
 
 
-def get_dataset_recipe_by_id(dataset_recipe_id: str, endpoint: str, api_key: str) -> Dict:
-    headers = {"Authorization": api_key}
+def get_dataset_recipe_by_id(dataset_recipe_id: str, cfg: Optional[Config] = None) -> Dict:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    headers = {"Authorization": cfg.api_key}
     full_url = f"{endpoint}/{dataset_recipe_id}"
     dataset_recipe_info: Dict = http.fetch(full_url, headers=headers)  # type: ignore[assignment]
     if not dataset_recipe_info:
@@ -46,25 +53,30 @@ def get_dataset_recipe_by_id(dataset_recipe_id: str, endpoint: str, api_key: str
 
 
 def get_or_create_dataset_recipe_from_path(
-    path_recipe_json: Path, endpoint: str, api_key: str, name: Optional[str] = None
+    path_recipe_json: Path, name: Optional[str] = None, cfg: Optional[Config] = None
 ) -> Dict:
     path_recipe_json = Path(path_recipe_json)
     if not path_recipe_json.exists():
         raise FileNotFoundError(f"Dataset recipe file '{path_recipe_json}' does not exist.")
     json_dict = json.loads(path_recipe_json.read_text())
-    return get_or_create_dataset_recipe(json_dict, endpoint=endpoint, api_key=api_key, name=name)
+    return get_or_create_dataset_recipe(json_dict, name=name, cfg=cfg)
 
 
-def delete_dataset_recipe_by_id(id: str, endpoint: str, api_key: str) -> Dict:
-    headers = {"Authorization": api_key}
+def delete_dataset_recipe_by_id(id: str, cfg: Optional[Config] = None) -> Dict:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    headers = {"Authorization": cfg.api_key}
     full_url = f"{endpoint}/{id}"
     response = http.delete(endpoint=full_url, headers=headers)
     return response
 
 
 @timed("Get dataset recipe")
-def get_dataset_recipe_by_name(name: str, endpoint: str, api_key: str) -> Optional[Dict]:
-    headers = {"Authorization": api_key}
+def get_dataset_recipe_by_name(name: str, cfg: Optional[Config] = None) -> Optional[Dict]:
+    cfg = cfg or Config()
+
+    endpoint = cfg.get_platform_endpoint("dataset_recipes")
+    headers = {"Authorization": cfg.api_key}
     full_url = f"{endpoint}?name__iexact={name}"
     dataset_recipes: List[Dict] = http.fetch(full_url, headers=headers)  # type: ignore[assignment]
     if len(dataset_recipes) == 0:
@@ -77,11 +89,11 @@ def get_dataset_recipe_by_name(name: str, endpoint: str, api_key: str) -> Option
     return dataset_recipe
 
 
-def delete_dataset_recipe_by_name(name: str, endpoint: str, api_key: str) -> Optional[Dict]:
-    recipe_response = get_dataset_recipe_by_name(name, endpoint=endpoint, api_key=api_key)
+def delete_dataset_recipe_by_name(name: str, cfg: Optional[Config] = None) -> Optional[Dict]:
+    recipe_response = get_dataset_recipe_by_name(name, cfg=cfg)
 
     if recipe_response:
-        return delete_dataset_recipe_by_id(recipe_response["id"], endpoint=endpoint, api_key=api_key)
+        return delete_dataset_recipe_by_id(recipe_response["id"], cfg=cfg)
     return recipe_response
 
 

hafnia/platform/datasets.py

@@ -83,8 +83,11 @@ def get_datasets(cfg: Optional[Config] = None) -> List[Dict[str, str]]:
 
 
 @timed("Fetching dataset info.")
-def get_dataset_id(dataset_name: str, endpoint: str, api_key: str) -> str:
-    headers = {"Authorization": api_key}
+def get_dataset_id(dataset_name: str, cfg: Optional[Config] = None) -> str:
+    """Get dataset ID by name from the Hafnia platform."""
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("datasets")
+    headers = {"Authorization": cfg.api_key}
     full_url = f"{endpoint}?name__iexact={dataset_name}"
     dataset_responses: List[Dict] = http.fetch(full_url, headers=headers)  # type: ignore[assignment]
     if not dataset_responses:
@@ -186,9 +189,10 @@ def delete_dataset_completely_by_name(
 
 
 @timed("Import dataset details to platform")
-def upload_dataset_details(cfg: Config, data: dict, dataset_name: str) -> dict:
+def upload_dataset_details(data: dict, dataset_name: str, cfg: Optional[Config] = None) -> dict:
+    cfg = cfg or Config()
     dataset_endpoint = cfg.get_platform_endpoint("datasets")
-    dataset_id = get_dataset_id(dataset_name, dataset_endpoint, cfg.api_key)
+    dataset_id = get_dataset_id(dataset_name, cfg=cfg)
 
     import_endpoint = f"{dataset_endpoint}/{dataset_id}/import"
     headers = {"Authorization": cfg.api_key}

hafnia/platform/experiment.py

@@ -1,7 +1,8 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from hafnia import http
 from hafnia.utils import pretty_print_list_as_table, timed
+from hafnia_cli.config import Config
 
 
 @timed("Creating experiment.")
@@ -11,10 +12,11 @@ def create_experiment(
     trainer_id: str,
     exec_cmd: str,
     environment_id: str,
-    endpoint: str,
-    api_key: str,
+    cfg: Optional[Config] = None,
 ) -> Dict:
-    headers = {"Authorization": api_key}
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("experiments")
+    headers = {"Authorization": cfg.api_key}
     response = http.post(
         endpoint,
         headers=headers,
@@ -30,8 +32,10 @@ def create_experiment(
 
 
 @timed("Fetching environment info.")
-def get_environments(endpoint: str, api_key: str) -> List[Dict]:
-    headers = {"Authorization": api_key}
+def get_environments(cfg: Optional[Config] = None) -> List[Dict]:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("experiment_environments")
+    headers = {"Authorization": cfg.api_key}
     envs: List[Dict] = http.fetch(endpoint, headers=headers)  # type: ignore[assignment]
     return envs
 
@@ -54,8 +58,8 @@ def pretty_print_training_environments(envs: List[Dict]) -> None:
     )
 
 
-def get_exp_environment_id(name: str, endpoint: str, api_key: str) -> str:
-    envs = get_environments(endpoint=endpoint, api_key=api_key)
+def get_exp_environment_id(name: str, cfg: Optional[Config] = None) -> str:
+    envs = get_environments(cfg=cfg)
 
     for env in envs:
         if env["name"] == name:

hafnia/platform/trainer_package.py

@@ -1,39 +1,85 @@
+import json
 from pathlib import Path
 from typing import Dict, List, Optional
 
 from hafnia import http
 from hafnia.log import user_logger
-from hafnia.utils import archive_dir, get_trainer_package_path, pretty_print_list_as_table, timed
+from hafnia.utils import (
+    archive_dir,
+    get_trainer_package_path,
+    pretty_print_list_as_table,
+    timed,
+)
+from hafnia_cli.config import Config
 
 
 @timed("Uploading trainer package.")
-def create_trainer_package(source_dir: Path, endpoint: str, api_key: str) -> str:
-    source_dir = source_dir.resolve()  # Ensure the path is absolute to handle '.' paths are given an appropriate name.
+def create_trainer_package(
+    source_dir: Path,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    cmd: Optional[str] = None,
+    cfg: Optional[Config] = None,
+) -> Dict:
+    # Ensure the path is absolute to handle '.' paths are given an appropriate name.
+    source_dir = Path(source_dir).resolve()
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("trainers")
+
     path_trainer = get_trainer_package_path(trainer_name=source_dir.name)
-    zip_path = archive_dir(source_dir, output_path=path_trainer)
+    name = name or path_trainer.stem
+    zip_path, package_files = archive_dir(source_dir, output_path=path_trainer)
     user_logger.info(f"Trainer package created and stored in '{path_trainer}'")
 
-    headers = {"Authorization": api_key, "accept": "application/json"}
+    cmd_builder_schemas = auto_discover_cmd_builder_schemas(package_files)
+    cmd = cmd or "python scripts/train.py"
+    description = description or f"Trainer package for '{name}'. Created with Hafnia SDK Cli."
+    headers = {"Authorization": cfg.api_key, "accept": "application/json"}
     data = {
-        "name": path_trainer.name,
-        "description": "Trainer package created by Hafnia CLI",
+        "name": name,
+        "description": description,
+        "default_command": cmd,
         "file": (zip_path.name, Path(zip_path).read_bytes()),
     }
+    if len(cmd_builder_schemas) == 0:
+        data["command_builder_schemas"] = json.dumps(cmd_builder_schemas)
+    user_logger.info(f"Uploading trainer package '{name}' to platform...")
     response = http.post(endpoint, headers=headers, data=data, multipart=True)
-    return response["id"]
+    user_logger.info(f"Trainer package uploaded successfully with id '{response['id']}'")
+    return response
+
+
+def auto_discover_cmd_builder_schemas(package_files: List[Path]) -> List[Dict]:
+    """
+    Auto-discover command builder schema files in the trainer package files.
+    Looks for files ending with '.schema.json' and loads their content as JSON.
+    """
+    cmd_builder_schema_files = [file for file in package_files if file.name.endswith(".schema.json")]
+    cmd_builder_schemas = []
+    for cmd_builder_schema_file in cmd_builder_schema_files:
+        cmd_builder_schema = json.loads(cmd_builder_schema_file.read_text())
+        cmd_entrypoint = cmd_builder_schema.get("cmd", None)
+        user_logger.info(f"Found command builder schema file for entry point '{cmd_entrypoint}'")
+        cmd_builder_schemas.append(cmd_builder_schema)
+    return cmd_builder_schemas
 
 
 @timed("Get trainer package.")
-def get_trainer_package_by_id(id: str, endpoint: str, api_key: str) -> Dict:
+def get_trainer_package_by_id(id: str, cfg: Optional[Config] = None) -> Dict:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("trainers")
     full_url = f"{endpoint}/{id}"
-    headers = {"Authorization": api_key}
+    headers = {"Authorization": cfg.api_key}
     response: Dict = http.fetch(full_url, headers=headers)  # type: ignore[assignment]
+
     return response
 
 
 @timed("Get trainer packages")
-def get_trainer_packages(endpoint: str, api_key: str) -> List[Dict]:
-    headers = {"Authorization": api_key}
+def get_trainer_packages(cfg: Optional[Config] = None) -> List[Dict]:
+    cfg = cfg or Config()
+    endpoint = cfg.get_platform_endpoint("trainers")
+    headers = {"Authorization": cfg.api_key}
     trainers: List[Dict] = http.fetch(endpoint, headers=headers)  # type: ignore[assignment]
     return trainers
 

hafnia/utils.py

@@ -6,7 +6,7 @@ from collections.abc import Sized
 from datetime import datetime
 from functools import wraps
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple
 from zipfile import ZipFile
 
 import more_itertools
@@ -113,20 +113,22 @@ def archive_dir(
     recipe_path: Path,
     output_path: Optional[Path] = None,
     path_ignore_file: Optional[Path] = None,
-) -> Path:
+) -> Tuple[Path, List[Path]]:
     recipe_zip_path = output_path or recipe_path / "trainer.zip"
     assert recipe_zip_path.suffix == ".zip", "Output path must be a zip file"
     recipe_zip_path.parent.mkdir(parents=True, exist_ok=True)
 
     user_logger.info(f" Creating zip archive of '{recipe_path}'")
-    include_files = filter_trainer_package_files(recipe_path, path_ignore_file)
+    include_files_generator = filter_trainer_package_files(recipe_path, path_ignore_file)
+    included_files = []
     with ZipFile(recipe_zip_path, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as zip_ref:
-        for str_filepath in include_files:
+        for str_filepath in include_files_generator:
             full_path = recipe_path / str_filepath
             zip_ref.write(full_path, str_filepath)
+            included_files.append(full_path)
     show_trainer_package_content(recipe_zip_path)
 
-    return recipe_zip_path
+    return recipe_zip_path, included_files
 
 
 def size_human_readable(size_bytes: int, suffix="B") -> str:

{hafnia-0.5.2.dist-info → hafnia-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.5.2
+Version: 0.5.4
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -64,9 +64,16 @@ multiple GPUs and instances if needed.
 ## Getting started: Configuration
 To get started with Hafnia: 
 
-1. Install `hafnia` with your favorite python package manager. With pip do this:
+1. Install `hafnia` with your favorite python package manager:
+
+    ```bash
+    # With uv package manager 
+    uv add hafnia
+
+    # With pip
+    pip install hafnia
+    ```
 
-    `pip install hafnia`
 1. Sign in to the [Hafnia Platform](https://hafnia.milestonesys.com/). 
 1. Create an API KEY for Training aaS. For more instructions, follow this 
 [guide](https://hafnia.readme.io/docs/create-an-api-key). 
@@ -94,11 +101,9 @@ With Hafnia configured on your local machine, it is now possible to download
 and explore the dataset sample with a python script:
 
 ```python
-from hafnia.data import get_dataset_path
 from hafnia.dataset.hafnia_dataset import HafniaDataset
 
-# To download the sample dataset use:
-path_dataset = get_dataset_path("midwest-vehicle-detection")
+dataset = HafniaDataset.from_name("midwest-vehicle-detection")
 ```
 
 This will download the dataset sample `midwest-vehicle-detection` to the local `.data/datasets/` folder
@@ -124,11 +129,10 @@ midwest-vehicle-detection
 3 directories, 217 files
 ```
 
-You can interact with data as you want, but we also provide `HafniaDataset`
-for loading/saving, managing and interacting with the dataset.
+We provide the `HafniaDataset` format for loading/saving, managing and interacting with the dataset.
 
 We recommend the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py) 
-for a short introduction on the `HafniaDataset`.
+for a quick introduction on the `HafniaDataset`.
 
 Below is a short introduction to the `HafniaDataset` class.
 
@@ -136,7 +140,7 @@ Below is a short introduction to the `HafniaDataset` class.
 from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
 
 # Load dataset from path
-dataset = HafniaDataset.read_from_path(path_dataset)
+dataset = HafniaDataset.from_path(path_dataset)
 
 # Or get dataset directly by name
 dataset = HafniaDataset.from_name("midwest-vehicle-detection")

{hafnia-0.5.2.dist-info → hafnia-0.5.4.dist-info}/RECORD

@@ -1,17 +1,17 @@
 hafnia/__init__.py,sha256=0qpjWfVbcfKzLSnfUW6RdclSGkesMQRFS-n_aTJJoSE,179
 hafnia/http.py,sha256=PkEuanlUKeERABXttaGAJT6hOZ1_B2CwJodbUV4uZdg,3710
 hafnia/log.py,sha256=sWF8tz78yBtwZ9ddzm19L1MBSBJ3L4G704IGeT1_OEU,784
-hafnia/torch_helpers.py,sha256=Qj8pV5P8tGw6F3W2Rj9Kel7O8hWuUqiKfTdXd3h5UOo,14080
-hafnia/utils.py,sha256=l_awkrb3OttxqSMkPiYcpuP3c_kejkSmiqndSahc1s0,8703
-hafnia/data/__init__.py,sha256=o9QjiGbEcNa6r-qDmwwmxPXf-1UitNl5-WxFNcujqsg,111
-hafnia/data/factory.py,sha256=kHkvOtBUbwaShZBGf1kZzocDJBn_1dHHLrQxnUpJmfY,778
-hafnia/dataset/dataset_details_uploader.py,sha256=H_zz67bBwbgo4StUwBNmH89WlqydIc-tEQbrRnZDwgg,24161
+hafnia/utils.py,sha256=YqlrRGmaVuYilT0YaZNNlZDGsBLfUXnjGy1vxsIUFHc,8834
+hafnia/dataset/colors.py,sha256=003eAJVnBal4abaYIIpsrT7erIOIjTUHHYVJ1Tj1CDc,5226
+hafnia/dataset/dataset_details_uploader.py,sha256=TfwOVkuLqmWQXAYZ8FomvJC9IAnomG-4VZTPxf6sgWc,24161
 hafnia/dataset/dataset_helpers.py,sha256=N8W_ioDlxP2VvNJXzqXLDbcEqgPKz0WyPNOBakHoBUc,6443
 hafnia/dataset/dataset_names.py,sha256=42_UKrDwcKEW48oTbtBaeyi5qVFVaMAj8vRvDv-mcEI,3616
-hafnia/dataset/hafnia_dataset.py,sha256=OOenIMPm8K23AgxHvmc_y05KCzxIwaZa-gv3uNC50NU,38519
-hafnia/dataset/hafnia_dataset_types.py,sha256=eCLawdjIFoul67AAtQ4xaKjbVSNAFA-mvbJYofiu2Sg,26848
+hafnia/dataset/hafnia_dataset.py,sha256=7zxCgqWwpVpZEGdnwmEqEYYFNekx_vd05zEj7bkitLE,37154
+hafnia/dataset/hafnia_dataset_types.py,sha256=38sCW_ISlWuG0kdnp_MZdL4OVFSEt2ULVfoTpgDW3lk,26841
+hafnia/dataset/image_visualizations.py,sha256=rB7c-KK-qq0BsSdkaFxCAHOOCTXTUQx0VMEhib7ig0k,7509
 hafnia/dataset/license_types.py,sha256=b1Jt5e8N89sujIs4T9y39sJEkzpAwCoLDTHDTpkiEOI,2166
-hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=Ln49jcpOQ4qzumv-SkWSBCqNgSP1dGQloKSLs7psP90,20991
+hafnia/dataset/torch_helpers.py,sha256=Qj8pV5P8tGw6F3W2Rj9Kel7O8hWuUqiKfTdXd3h5UOo,14080
+hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=YHnSG4lDYLjRsnrybOrNNGASoMhOaLo3PaxiifIwHQ4,18484
 hafnia/dataset/dataset_recipe/recipe_transforms.py,sha256=j3Oiytt3LI2rCaJid7Y44oT9MXvlZVqvZanngMebIWg,3088
 hafnia/dataset/dataset_recipe/recipe_types.py,sha256=AcrG6gpRt3Igl-CCJ60uyh-WkfI1NCnQ55M8yClSI9Q,5328
 hafnia/dataset/format_conversions/format_coco.py,sha256=7GjeF016ZBaKxu-VYiqXxuPw8HuuODV1cxc2TbDDZBw,19628
@@ -20,44 +20,42 @@ hafnia/dataset/format_conversions/format_image_classification_folder.py,sha256=A
 hafnia/dataset/format_conversions/format_yolo.py,sha256=zvCHo2L_0mPJScMbDtwvZUts9UX2ERKhhYbY31Q6tQA,9912
 hafnia/dataset/format_conversions/torchvision_datasets.py,sha256=sC8DgAt10PEaCHFk_Lm-dIzr_0EF-2g24kG9EINYk7c,12096
 hafnia/dataset/operations/dataset_s3_storage.py,sha256=xPC77Og47xTpI0JBFAR1pgb5u7l18byAA6p7IlpnpGE,8971
-hafnia/dataset/operations/dataset_stats.py,sha256=Ltf-V4_o_IB4UXw9WG9bsVoqeX90yGsjivK0CDggriw,11930
+hafnia/dataset/operations/dataset_stats.py,sha256=uzQJWOoAM7YDLLeUhPVBpE1vFM38AOriDlPxKs2hj5M,11986
 hafnia/dataset/operations/dataset_transformations.py,sha256=qUNno0rAT1A452uzlR-k1WbatyY9VuMp1QJjkMg9GzE,19495
 hafnia/dataset/operations/table_transformations.py,sha256=mdjUE1lSQ7QyONjQapSHDg1MkYuKaflcoVUq1Y6Lkqc,13606
 hafnia/dataset/primitives/__init__.py,sha256=xFLJ3R7gpbuQnNJuFhuu836L3nicwoaY5aHkqk7Bbr8,927
 hafnia/dataset/primitives/bbox.py,sha256=QJJBebltOd9J3idisp3QdX0gCgz6P5xlIlGbth19fG0,6669
 hafnia/dataset/primitives/bitmask.py,sha256=Q7RiNYvMDlcFPkXAWXDJkCIERjnUTCrHu6VeEPX1jEA,7212
-hafnia/dataset/primitives/classification.py,sha256=rYcf9MS-pYE4O1YrkYutFjICnDhoOJu-t0xW62wh_TA,2669
+hafnia/dataset/primitives/classification.py,sha256=YAMwO_gSOfDiXLUrEq-ObzvChK478rwGTP-RBhWt1LE,2662
 hafnia/dataset/primitives/point.py,sha256=VzCNLTQOPA6wyJVVKddZHGhltkep6V_B7pg5pk7rd9Y,879
 hafnia/dataset/primitives/polygon.py,sha256=jZPNVwEs4A3IMJQzI_dlcDDfgju7hdoVc677tMAdEbQ,6271
 hafnia/dataset/primitives/primitive.py,sha256=Wvby0sCGgYj8ec39PLcHsmip5VKL96ZSCz2cGIBjPqM,1289
-hafnia/dataset/primitives/segmentation.py,sha256=hnXIUklkuDMxYkUaff1bgRTcXI_2b32RIbobxR3ejzk,2017
+hafnia/dataset/primitives/segmentation.py,sha256=ACexXYavoFsqviCRA76MDZUvEoBZLO_OTDCl2Px_rV4,2010
 hafnia/dataset/primitives/utils.py,sha256=3gT1as-xXEj8CamoIuBb9gQwUN9Ae9qnqtqF_uEe0zo,1993
 hafnia/experiment/__init__.py,sha256=OEFE6HqhO5zcTCLZcPcPVjIg7wMFFnvZ1uOtAVhRz7M,85
-hafnia/experiment/command_builder.py,sha256=9_Of8WsJyyaCSK4C0vJMLVaUU6qEhnVNaIUAEHc80rI,27580
+hafnia/experiment/command_builder.py,sha256=F1szeVKD68_W2oHbp-pjkjAo0bbjw9TymQwLjF7QVhE,27587
 hafnia/experiment/hafnia_logger.py,sha256=BHIOLAds_3JxT0cev_ikUH0XQVIxBJTkcBSx2Q_SIk0,10894
 hafnia/platform/__init__.py,sha256=L_Q7CNpsJ0HMNPy_rLlLK5RhmuCU7IF4BchxKv6amYc,782
 hafnia/platform/builder.py,sha256=kUEuj5-qtL1uk5v2tUvOCREn5yV-G4Fr6F31haIAb5E,5808
-hafnia/platform/dataset_recipe.py,sha256=ybfSSHVPG0eFUbzg_1McezPSOtMoDZEg7l6rFYndtb4,3857
-hafnia/platform/datasets.py,sha256=nXHg3I14p3tJeDX2woPH9NMiOxn_54zlIOPJXvXFI_w,9448
+hafnia/platform/dataset_recipe.py,sha256=UNvsDEbByT_WPuslILLGFsqXb87g65K5xz-Q2ZzvcKs,4242
+hafnia/platform/datasets.py,sha256=z7bQz1SIR-vVVjRJD1FwEPw2X5QPyE_J1Ea1M6XPXwc,9612
 hafnia/platform/download.py,sha256=e73Pm0afwRPTHxBvRy0gUZSFfDuePHPnfasyhaZ-KGQ,5019
-hafnia/platform/experiment.py,sha256=SrEH0nuwwBXf1Iu4diB1BEPqL-TxW3aQkZWBbM1-tY0,1846
+hafnia/platform/experiment.py,sha256=qNg9CKBLIYnOb-bMaEDecv-PptP4_ubQJunXGwdSiaQ,2049
 hafnia/platform/s5cmd_utils.py,sha256=hHsGPJ1S9_hFIVfCO-efvTF4qbLYreK1nl3VC5caU1w,9491
-hafnia/platform/trainer_package.py,sha256=w6JC7o-279ujcwtNTbUaQ9AnPcYRPPbD8EACa6XyUHA,2206
-hafnia/visualizations/colors.py,sha256=003eAJVnBal4abaYIIpsrT7erIOIjTUHHYVJ1Tj1CDc,5226
-hafnia/visualizations/image_visualizations.py,sha256=rB7c-KK-qq0BsSdkaFxCAHOOCTXTUQx0VMEhib7ig0k,7509
+hafnia/platform/trainer_package.py,sha256=MTmiPm02uy5TIUDRjgCSETL-Q_esxha0NWtHO1h53dw,3949
 hafnia_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hafnia_cli/__main__.py,sha256=CqD_7RfbfwB6MED3WZ8WxclrFxWcRMtZ5A1Snnst3GM,1761
 hafnia_cli/config.py,sha256=X0dJBYH-7mNAFkHgeZvDbawlQqoyCwoY4C-QhlyYCA0,7695
 hafnia_cli/consts.py,sha256=uCpYX44NCu_Zvte0QwChunxOo-qqhcaJRSYDAIsoJ8A,972
 hafnia_cli/dataset_cmds.py,sha256=JfSj7Cei1T2oYUXP1bpz63uQopgL3R_dMMYnPGGcuU8,2072
-hafnia_cli/dataset_recipe_cmds.py,sha256=OYSmpKL0Wxo1ZSxIGfH6w7pEWoI7CjUTmfIELJSZjGQ,2894
-hafnia_cli/experiment_cmds.py,sha256=_KxsMhbjlkIno1PIMXJ0Omw_PSJi8qi9hmtCUqwcj1M,7970
+hafnia_cli/dataset_recipe_cmds.py,sha256=cprz0RMxuPK8hLLeu3V5MojtKZlSSNb3THBo2pZzdiM,2589
+hafnia_cli/experiment_cmds.py,sha256=ZKSy2W-ke33MhXF5BKWdkQO0o31tqIb4Ld-l_cfc5Lw,7453
 hafnia_cli/keychain.py,sha256=bNyjjULVQu7kV338wUC65UvbCwmSGOmEjKWPLIQjT0k,2555
-hafnia_cli/profile_cmds.py,sha256=yTyOsPsUssLCzFIxURkxbKrFEhYIVDlUC0G2s5Uks-U,3476
+hafnia_cli/profile_cmds.py,sha256=QVTK_hLskuiod9Nmgqld61-McWQEyCMWAgvDlMtB7oE,3709
 hafnia_cli/runc_cmds.py,sha256=7P5TjF6KA9K4OKPG1qC_0gteXfLJbXlA858WWrosoGQ,5098
-hafnia_cli/trainer_package_cmds.py,sha256=hUBc6gCMV28fcAA0xQdXKL1z-a3aL9lMWcVqjvHO1Uo,2326
-hafnia-0.5.2.dist-info/METADATA,sha256=U8Qy_hlNHR-hfgFqsvAkM8dfzS28v9nRI9ne5oXA87o,19312
-hafnia-0.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hafnia-0.5.2.dist-info/entry_points.txt,sha256=j2jsj1pqajLAiSOnF7sq66A3d1SVeHPKVTVyIFzipSA,52
-hafnia-0.5.2.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
-hafnia-0.5.2.dist-info/RECORD,,
+hafnia_cli/trainer_package_cmds.py,sha256=RMSiinwzVlK-kDFPuwZ98EoSpw61aaXa6IyRj3UEVlw,3307
+hafnia-0.5.4.dist-info/METADATA,sha256=MIdQYVCwEQFQr6GLFtJHpNCwnaAhppQeeB1QrL3PlIE,19258
+hafnia-0.5.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hafnia-0.5.4.dist-info/entry_points.txt,sha256=j2jsj1pqajLAiSOnF7sq66A3d1SVeHPKVTVyIFzipSA,52
+hafnia-0.5.4.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
+hafnia-0.5.4.dist-info/RECORD,,

hafnia_cli/dataset_recipe_cmds.py

@@ -28,8 +28,7 @@ def cmd_get_or_create_dataset_recipe(cfg: Config, path_json_recipe: Path, name:
     """Create Hafnia dataset recipe from dataset recipe JSON file"""
     from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe_from_path
 
-    endpoint = cfg.get_platform_endpoint("dataset_recipes")
-    recipe = get_or_create_dataset_recipe_from_path(path_json_recipe, endpoint=endpoint, api_key=cfg.api_key, name=name)
+    recipe = get_or_create_dataset_recipe_from_path(path_json_recipe, name=name, cfg=cfg)
 
     if recipe is None:
         raise click.ClickException("Failed to create dataset recipe.")
@@ -44,8 +43,7 @@ def cmd_list_dataset_recipes(cfg: Config, limit: Optional[int]) -> None:
     """List available dataset recipes"""
     from hafnia.platform.dataset_recipe import get_dataset_recipes, pretty_print_dataset_recipes
 
-    endpoint = cfg.get_platform_endpoint("dataset_recipes")
-    recipes = get_dataset_recipes(endpoint=endpoint, api_key=cfg.api_key)
+    recipes = get_dataset_recipes(cfg=cfg)
     # Sort recipes to have the most recent first
     recipes = sorted(recipes, key=lambda x: x["created_at"], reverse=True)
     if limit is not None:
@@ -61,12 +59,10 @@ def cmd_delete_dataset_recipe(cfg: Config, id: Optional[str], name: Optional[str
     """Delete a dataset recipe by ID or name"""
     from hafnia.platform.dataset_recipe import delete_dataset_recipe_by_id, delete_dataset_recipe_by_name
 
-    endpoint = cfg.get_platform_endpoint("dataset_recipes")
-
     if id is not None:
-        return delete_dataset_recipe_by_id(id=id, endpoint=endpoint, api_key=cfg.api_key)
+        return delete_dataset_recipe_by_id(id=id, cfg=cfg)
     if name is not None:
-        dataset_recipe = delete_dataset_recipe_by_name(name=name, endpoint=endpoint, api_key=cfg.api_key)
+        dataset_recipe = delete_dataset_recipe_by_name(name=name, cfg=cfg)
         if dataset_recipe is None:
             raise click.ClickException(f"Dataset recipe with name '{name}' was not found.")
 

hafnia_cli/experiment_cmds.py

@@ -27,7 +27,7 @@ def cmd_view_environments(cfg: Config):
     """
     from hafnia.platform import get_environments, pretty_print_training_environments
 
-    envs = get_environments(cfg.get_platform_endpoint("experiment_environments"), cfg.api_key)
+    envs = get_environments(cfg=cfg)
 
     pretty_print_training_environments(envs)
 
@@ -132,7 +132,7 @@ def cmd_create_experiment(
     """
     from hafnia.platform import create_experiment, get_exp_environment_id
 
-    dataset_recipe_response = get_dataset_recipe_by_dataset_identifies(
+    dataset_recipe_response = get_dataset_recipe_by_identifiers(
         cfg=cfg,
         dataset_name=dataset,
         dataset_recipe_name=dataset_recipe,
@@ -140,13 +140,8 @@ def cmd_create_experiment(
     )
     dataset_recipe_id = dataset_recipe_response["id"]
 
-    trainer_id = get_trainer_package_by_identifies(
-        cfg=cfg,
-        trainer_path=trainer_path,
-        trainer_id=trainer_id,
-    )
-
-    env_id = get_exp_environment_id(environment, cfg.get_platform_endpoint("experiment_environments"), cfg.api_key)
+    trainer_id = get_trainer_package_by_identifiers(cfg=cfg, trainer_path=trainer_path, trainer_id=trainer_id)
+    env_id = get_exp_environment_id(environment, cfg=cfg)
 
     experiment = create_experiment(
         experiment_name=name,
@@ -154,8 +149,7 @@ def cmd_create_experiment(
         trainer_id=trainer_id,
         exec_cmd=cmd,
         environment_id=env_id,
-        endpoint=cfg.get_platform_endpoint("experiments"),
-        api_key=cfg.api_key,
+        cfg=cfg,
     )
 
     experiment_properties = {
@@ -172,7 +166,7 @@ def cmd_create_experiment(
         print(f"  {key}: {value}")
 
 
-def get_dataset_recipe_by_dataset_identifies(
+def get_dataset_recipe_by_identifiers(
     cfg: Config,
     dataset_name: Optional[str],
     dataset_recipe_name: Optional[str],
@@ -186,18 +180,17 @@ def get_dataset_recipe_by_dataset_identifies(
             "Multiple dataset identifiers have been provided. Define only one dataset identifier."
         )
 
-    dataset_recipe_endpoint = cfg.get_platform_endpoint("dataset_recipes")
     if dataset_name:
-        return get_or_create_dataset_recipe_by_dataset_name(dataset_name, dataset_recipe_endpoint, cfg.api_key)
+        return get_or_create_dataset_recipe_by_dataset_name(dataset_name, cfg=cfg)
 
     if dataset_recipe_name:
-        recipe = get_dataset_recipe_by_name(dataset_recipe_name, dataset_recipe_endpoint, cfg.api_key)
+        recipe = get_dataset_recipe_by_name(dataset_recipe_name, cfg=cfg)
         if recipe is None:
             raise click.ClickException(f"Dataset recipe '{dataset_recipe_name}' was not found in the dataset library.")
         return recipe
 
     if dataset_recipe_id:
-        return get_dataset_recipe_by_id(dataset_recipe_id, dataset_recipe_endpoint, cfg.api_key)
+        return get_dataset_recipe_by_id(dataset_recipe_id, cfg=cfg)
 
     raise click.MissingParameter(
         "At least one dataset identifier must be provided. Set one of the following:\n"
@@ -207,7 +200,7 @@ def get_dataset_recipe_by_dataset_identifies(
     )
 
 
-def get_trainer_package_by_identifies(
+def get_trainer_package_by_identifiers(
     cfg: Config,
     trainer_path: Optional[Path],
     trainer_id: Optional[str],
@@ -223,17 +216,14 @@ def get_trainer_package_by_identifies(
         trainer_path = Path(trainer_path)
         if not trainer_path.exists():
             raise click.ClickException(f"Trainer package path '{trainer_path}' does not exist.")
-        trainer_id = create_trainer_package(
-            trainer_path,
-            cfg.get_platform_endpoint("trainers"),
-            cfg.api_key,
+        response = create_trainer_package(
+            source_dir=trainer_path,
+            cfg=cfg,
         )
-        return trainer_id
+        return response["id"]
 
     if trainer_id:
-        trainer_response = get_trainer_package_by_id(
-            id=trainer_id, endpoint=cfg.get_platform_endpoint("trainers"), api_key=cfg.api_key
-        )
+        trainer_response = get_trainer_package_by_id(id=trainer_id, cfg=cfg)
         return trainer_response["id"]
 
     raise click.MissingParameter(

hafnia_cli/profile_cmds.py

@@ -6,10 +6,15 @@ import hafnia_cli.consts as consts
 from hafnia_cli.config import Config, ConfigSchema
 
 
-@click.group()
-def profile():
+@click.group(invoke_without_command=True)
+@click.pass_context
+def profile(ctx):
     """Manage profile."""
-    pass
+    if ctx.invoked_subcommand is None:
+        # No subcommand provided, show active profile and help
+        cfg = ctx.obj
+        profile_show(cfg)
+        click.echo("\n" + ctx.get_help())
 
 
 @profile.command("ls")

hafnia_cli/trainer_package_cmds.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import click
 
@@ -21,12 +21,60 @@ def cmd_list_trainer_packages(cfg: Config, limit: Optional[int]) -> None:
 
     from hafnia.platform.trainer_package import get_trainer_packages, pretty_print_trainer_packages
 
-    endpoint = cfg.get_platform_endpoint("trainers")
-    trainers = get_trainer_packages(endpoint, cfg.api_key)
+    trainers = get_trainer_packages(cfg=cfg)
 
     pretty_print_trainer_packages(trainers, limit=limit)
 
 
+@trainer_package.command(name="create")
+@click.pass_obj
+@click.argument(
+    "path",
+    type=Path,
+)
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    default=None,
+    help="Name of the trainer package.",
+)
+@click.option(
+    "-d",
+    "--description",
+    type=str,
+    default=None,
+    help="Description of the trainer package.",
+)
+@click.option(
+    "--cmd",
+    type=Optional[str],
+    default=None,
+    show_default=True,
+    help="Default command to run the trainer package.",
+)
+def cmd_create_trainer_package(
+    cfg: Config,
+    path: Path,
+    cmd: Optional[str] = None,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> Dict:
+    """Create a trainer package on the platform"""
+    from hafnia.platform.trainer_package import create_trainer_package
+
+    path_trainer = Path(path).resolve()
+    trainer_response = create_trainer_package(
+        source_dir=path_trainer,
+        name=name,
+        description=description,
+        cmd=cmd,
+        cfg=cfg,
+    )
+
+    return trainer_response
+
+
 @trainer_package.command(name="create-zip")
 @click.argument("source")
 @click.option(
@@ -46,7 +94,7 @@ def cmd_create_trainer_package_zip(source: str, output: str) -> None:
         raise click.ClickException(consts.ERROR_TRAINER_PACKAGE_FILE_FORMAT)
 
     path_source = Path(source)
-    path_output_zip = archive_dir(path_source, path_output_zip)
+    path_output_zip, _ = archive_dir(path_source, path_output_zip)
 
 
 @trainer_package.command(name="view-zip")

hafnia/data/__init__.py

@@ -1,3 +0,0 @@
-from hafnia.data.factory import get_dataset_path, load_dataset
-
-__all__ = ["load_dataset", "get_dataset_path"]

hafnia/data/factory.py

@@ -1,22 +0,0 @@
-from pathlib import Path
-from typing import Any
-
-from hafnia import utils
-from hafnia.dataset.hafnia_dataset import HafniaDataset, get_or_create_dataset_path_from_recipe
-
-
-def load_dataset(recipe: Any, force_redownload: bool = False) -> HafniaDataset:
-    """Load a dataset either from a local path or from the Hafnia platform."""
-
-    path_dataset = get_dataset_path(recipe, force_redownload=force_redownload)
-    dataset = HafniaDataset.from_path(path_dataset)
-    return dataset
-
-
-def get_dataset_path(recipe: Any, force_redownload: bool = False) -> Path:
-    if utils.is_hafnia_cloud_job():
-        return utils.get_dataset_path_in_hafnia_cloud()
-
-    path_dataset = get_or_create_dataset_path_from_recipe(recipe, force_redownload=force_redownload)
-
-    return path_dataset