hafnia 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

hafnia/dataset/dataset_names.py

@@ -18,11 +18,14 @@ class DeploymentStage(Enum):
     PRODUCTION = "production"
 
 
+TAG_IS_SAMPLE = "sample"
+
+OPS_REMOVE_CLASS = "__REMOVE__"
+
+
 class FieldName:
     CLASS_NAME: str = "class_name"  # Name of the class this primitive is associated with, e.g. "car" for Bbox
-    CLASS_IDX: str = (
-        "class_idx"  # Index of the class this primitive is associated with, e.g. 0 for "car" if it is the first class
-    )
+    CLASS_IDX: str = "class_idx"  # Index of the class this primitive is associated with, e.g. 0 for "car" if it is the first class  # noqa: E501
     OBJECT_ID: str = "object_id"  # Unique identifier for the object, e.g. "12345123"
     CONFIDENCE: str = "confidence"  # Confidence score (0-1.0) for the primitive, e.g. 0.95 for Bbox
 
@@ -50,8 +53,9 @@ class ColumnName:
     HEIGHT: str = "height"
     WIDTH: str = "width"
     SPLIT: str = "split"
-    IS_SAMPLE: str = "is_sample"
     REMOTE_PATH: str = "remote_path"  # Path to the file in remote storage, e.g. S3
+    ATTRIBUTION: str = "attribution"  # Attribution for the sample (image/video), e.g. creator, license, source, etc.
+    TAGS: str = "tags"
     META: str = "meta"
 
 

hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Type, Union
 
 from pydantic import (
     field_serializer,
@@ -14,9 +14,7 @@ from hafnia import utils
 from hafnia.dataset.dataset_recipe import recipe_transforms
 from hafnia.dataset.dataset_recipe.recipe_types import RecipeCreation, RecipeTransform, Serializable
 from hafnia.dataset.hafnia_dataset import HafniaDataset
-
-if TYPE_CHECKING:
-    from hafnia.dataset.hafnia_dataset import HafniaDataset
+from hafnia.dataset.primitives.primitive import Primitive
 
 
 class DatasetRecipe(Serializable):
@@ -76,6 +74,42 @@ class DatasetRecipe(Serializable):
         json_str = path_json.read_text(encoding="utf-8")
         return DatasetRecipe.from_json_str(json_str)
 
+    @staticmethod
+    def from_dict(data: Dict[str, Any]) -> "DatasetRecipe":
+        """Deserialize from a dictionary."""
+        dataset_recipe = Serializable.from_dict(data)
+        return dataset_recipe
+
+    @staticmethod
+    def from_recipe_id(recipe_id: str) -> "DatasetRecipe":
+        """Loads a dataset recipe by id from the hafnia platform."""
+        from cli.config import Config
+        from hafnia.platform.dataset_recipe import get_dataset_recipe_by_id
+
+        cfg = Config()
+        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
+        recipe_dict = get_dataset_recipe_by_id(recipe_id, endpoint=endpoint_dataset, api_key=cfg.api_key)
+        recipe_dict = recipe_dict["template"]["body"]
+        if isinstance(recipe_dict, str):
+            return DatasetRecipe.from_implicit_form(recipe_dict)
+
+        recipe = DatasetRecipe.from_dict(recipe_dict)
+        return recipe
+
+    @staticmethod
+    def from_recipe_name(name: str) -> "DatasetRecipe":
+        """Loads a dataset recipe by name from the hafnia platform"""
+        from cli.config import Config
+        from hafnia.platform.dataset_recipe import get_dataset_recipe_by_name
+
+        cfg = Config()
+        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
+        recipe = get_dataset_recipe_by_name(name=name, endpoint=endpoint_dataset, api_key=cfg.api_key)
+        if not recipe:
+            raise ValueError(f"Dataset recipe '{name}' not found.")
+        recipe_id = recipe["id"]
+        return DatasetRecipe.from_recipe_id(recipe_id)
+
     @staticmethod
     def from_implicit_form(recipe: Any) -> DatasetRecipe:
         """
@@ -152,6 +186,59 @@ class DatasetRecipe(Serializable):
 
         raise ValueError(f"Unsupported recipe type: {type(recipe)}")
 
+    ### Upload, store and recipe conversions ###
+    def as_python_code(self, keep_default_fields: bool = False, as_kwargs: bool = True) -> str:
+        str_operations = [self.creation.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs)]
+        if self.operations:
+            for op in self.operations:
+                str_operations.append(op.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs))
+        operations_str = ".".join(str_operations)
+        return operations_str
+
+    def as_short_name(self) -> str:
+        """Return a short name for the transforms."""
+
+        creation_name = self.creation.as_short_name()
+        if self.operations is None or len(self.operations) == 0:
+            return creation_name
+        short_names = [creation_name]
+        for operation in self.operations:
+            short_names.append(operation.as_short_name())
+        transforms_str = ",".join(short_names)
+        return f"Recipe({transforms_str})"
+
+    def as_json_str(self, indent: int = 2) -> str:
+        """Serialize the dataset recipe to a JSON string."""
+        dict_data = self.as_dict()
+        return json.dumps(dict_data, indent=indent, ensure_ascii=False)
+
+    def as_json_file(self, path_json: Path, indent: int = 2) -> None:
+        """Serialize the dataset recipe to a JSON file."""
+        path_json.parent.mkdir(parents=True, exist_ok=True)
+        json_str = self.as_json_str(indent=indent)
+        path_json.write_text(json_str, encoding="utf-8")
+
+    def as_dict(self) -> dict:
+        """Serialize the dataset recipe to a dictionary."""
+        return self.model_dump(mode="json")
+
+    def as_platform_recipe(self, recipe_name: Optional[str]) -> Dict:
+        """Uploads dataset recipe to the hafnia platform."""
+        from cli.config import Config
+        from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe
+
+        recipe = self.as_dict()
+        cfg = Config()
+        endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
+        recipe_dict = get_or_create_dataset_recipe(
+            recipe=recipe,
+            endpoint=endpoint_dataset,
+            api_key=cfg.api_key,
+            name=recipe_name,
+        )
+
+        return recipe_dict
+
     ### Dataset Recipe Transformations ###
     def shuffle(recipe: DatasetRecipe, seed: int = 42) -> DatasetRecipe:
         operation = recipe_transforms.Shuffle(seed=seed)
@@ -184,37 +271,36 @@ class DatasetRecipe(Serializable):
         recipe.append_operation(operation)
         return recipe
 
-    ### Conversions ###
-    def as_python_code(self, keep_default_fields: bool = False, as_kwargs: bool = True) -> str:
-        str_operations = [self.creation.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs)]
-        if self.operations:
-            for op in self.operations:
-                str_operations.append(op.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs))
-        operations_str = ".".join(str_operations)
-        return operations_str
-
-    def as_short_name(self) -> str:
-        """Return a short name for the transforms."""
-
-        creation_name = self.creation.as_short_name()
-        if self.operations is None or len(self.operations) == 0:
-            return creation_name
-        short_names = [creation_name]
-        for operation in self.operations:
-            short_names.append(operation.as_short_name())
-        transforms_str = ",".join(short_names)
-        return f"Recipe({transforms_str})"
+    def class_mapper(
+        recipe: DatasetRecipe,
+        class_mapping: Dict[str, str],
+        method: str = "strict",
+        primitive: Optional[Type[Primitive]] = None,
+        task_name: Optional[str] = None,
+    ) -> DatasetRecipe:
+        operation = recipe_transforms.ClassMapper(
+            class_mapping=class_mapping,
+            method=method,
+            primitive=primitive,
+            task_name=task_name,
+        )
+        recipe.append_operation(operation)
+        return recipe
 
-    def as_json_str(self, indent: int = 2) -> str:
-        """Serialize the dataset recipe to a JSON string."""
-        data = self.model_dump(mode="json")
-        # data = type_as_first_key(data)
-        return json.dumps(data, indent=indent, ensure_ascii=False)
+    def rename_task(recipe: DatasetRecipe, old_task_name: str, new_task_name: str) -> DatasetRecipe:
+        operation = recipe_transforms.RenameTask(old_task_name=old_task_name, new_task_name=new_task_name)
+        recipe.append_operation(operation)
+        return recipe
 
-    def as_json_file(self, path_json: Path, indent: int = 2) -> None:
-        """Serialize the dataset recipe to a JSON file."""
-        json_str = self.as_json_str(indent=indent)
-        path_json.write_text(json_str, encoding="utf-8")
+    def select_samples_by_class_name(
+        recipe: DatasetRecipe,
+        name: Union[List[str], str],
+        task_name: Optional[str] = None,
+        primitive: Optional[Type[Primitive]] = None,
+    ) -> DatasetRecipe:
+        operation = recipe_transforms.SelectSamplesByClassName(name=name, task_name=task_name, primitive=primitive)
+        recipe.append_operation(operation)
+        return recipe
 
     ### Helper methods ###
     def get_dataset_names(self) -> List[str]:

hafnia/dataset/dataset_recipe/recipe_transforms.py

@@ -1,10 +1,8 @@
-from typing import TYPE_CHECKING, Callable, Dict
+from typing import Callable, Dict, List, Optional, Type, Union
 
 from hafnia.dataset.dataset_recipe.recipe_types import RecipeTransform
 from hafnia.dataset.hafnia_dataset import HafniaDataset
-
-if TYPE_CHECKING:
-    pass
+from hafnia.dataset.primitives.primitive import Primitive
 
 
 class Shuffle(RecipeTransform):
@@ -51,3 +49,33 @@ class DefineSampleSetBySize(RecipeTransform):
     @staticmethod
     def get_function() -> Callable[..., "HafniaDataset"]:
         return HafniaDataset.define_sample_set_by_size
+
+
+class ClassMapper(RecipeTransform):
+    class_mapping: Dict[str, str]
+    method: str = "strict"
+    primitive: Optional[Type[Primitive]] = None
+    task_name: Optional[str] = None
+
+    @staticmethod
+    def get_function() -> Callable[..., "HafniaDataset"]:
+        return HafniaDataset.class_mapper
+
+
+class RenameTask(RecipeTransform):
+    old_task_name: str
+    new_task_name: str
+
+    @staticmethod
+    def get_function() -> Callable[..., "HafniaDataset"]:
+        return HafniaDataset.rename_task
+
+
+class SelectSamplesByClassName(RecipeTransform):
+    name: Union[List[str], str]
+    task_name: Optional[str] = None
+    primitive: Optional[Type[Primitive]] = None
+
+    @staticmethod
+    def get_function() -> Callable[..., "HafniaDataset"]:
+        return HafniaDataset.select_samples_by_class_name

hafnia/dataset/dataset_recipe/recipe_types.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel, computed_field
 
 from hafnia import utils
 
-if TYPE_CHECKING:
+if TYPE_CHECKING:  # Using 'TYPE_CHECKING' to avoid circular imports during type checking
     from hafnia.dataset.hafnia_dataset import HafniaDataset