hafnia 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

cli/__main__.py

@@ -20,19 +20,15 @@ def configure(cfg: Config) -> None:
 
     profile_name = click.prompt("Profile Name", type=str, default=consts.DEFAULT_PROFILE_NAME)
     profile_name = profile_name.strip()
-    try:
-        cfg.add_profile(profile_name, ConfigSchema(), set_active=True)
-    except ValueError:
-        raise click.ClickException(consts.ERROR_CREATE_PROFILE)
+
+    cfg.check_profile_name(profile_name)
 
     api_key = click.prompt("Hafnia API Key", type=str, hide_input=True)
-    try:
-        cfg.api_key = api_key.strip()
-    except ValueError as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        return
+
     platform_url = click.prompt("Hafnia Platform URL", type=str, default=consts.DEFAULT_API_URL)
-    cfg.platform_url = platform_url.strip()
+
+    cfg_profile = ConfigSchema(api_key=api_key, platform_url=platform_url)
+    cfg.add_profile(profile_name, cfg_profile, set_active=True)
     cfg.save_config()
     profile_cmds.profile_show(cfg)
 

cli/config.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional
 from pydantic import BaseModel, field_validator
 
 import cli.consts as consts
-from hafnia.log import user_logger
+from hafnia.log import sys_logger, user_logger
 
 PLATFORM_API_MAPPING = {
     "recipes": "/api/v1/recipes",
@@ -23,9 +23,17 @@ class ConfigSchema(BaseModel):
     api_key: Optional[str] = None
 
     @field_validator("api_key")
-    def validate_api_key(cls, value: str) -> str:
-        if value is not None and len(value) < 10:
+    def validate_api_key(cls, value: Optional[str]) -> Optional[str]:
+        if value is None:
+            return value
+
+        if len(value) < 10:
             raise ValueError("API key is too short.")
+
+        if not value.startswith("ApiKey "):
+            sys_logger.warning("API key is missing the 'ApiKey ' prefix. Prefix is being added automatically.")
+            value = f"ApiKey {value}"
+
         return value
 
 
@@ -51,6 +59,7 @@ class Config:
         if profile_name not in self.config_data.profiles:
             raise ValueError(f"Profile '{profile_name}' does not exist.")
         self.config_data.active_profile = profile_name
+        self.save_config()
 
     @property
     def config(self) -> ConfigSchema:
@@ -92,13 +101,18 @@ class Config:
 
         return Path.home() / ".hafnia" / "config.json"
 
-    def add_profile(self, profile_name: str, profile: ConfigSchema, set_active: bool = False) -> None:
-        profile_name = profile_name.strip()
+    def check_profile_name(self, profile_name: str) -> None:
+        if not profile_name or not isinstance(profile_name, str):
+            raise ValueError("Profile name must be a non-empty string.")
+
         if profile_name in self.config_data.profiles:
             user_logger.warning(
                 f"Profile with name '{profile_name}' already exists, it will be overwritten by the new one."
             )
 
+    def add_profile(self, profile_name: str, profile: ConfigSchema, set_active: bool = False) -> None:
+        profile_name = profile_name.strip()
+        self.check_profile_name(profile_name)
         self.config_data.profiles[profile_name] = profile
         if set_active:
             self.config_data.active_profile = profile_name

cli/profile_cmds.py

@@ -56,6 +56,7 @@ def profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate:
     cfg_profile = ConfigSchema(platform_url=api_url, api_key=api_key)
 
     cfg.add_profile(profile_name=name, profile=cfg_profile, set_active=activate)
+    profile_show(cfg)
 
 
 @profile.command("rm")
@@ -87,7 +88,7 @@ def profile_active(cfg: Config) -> None:
 
 
 def profile_show(cfg: Config) -> None:
-    masked_key = f"{cfg.api_key[:4]}...{cfg.api_key[-4:]}" if len(cfg.api_key) > 8 else "****"
+    masked_key = f"{cfg.api_key[:11]}...{cfg.api_key[-4:]}" if len(cfg.api_key) > 20 else "****"
     console = Console()
 
     table = Table(title=f"{consts.PROFILE_TABLE_HEADER} {cfg.active_profile}", show_header=False)

hafnia/dataset/dataset_helpers.py

@@ -1,6 +1,7 @@
 import io
 import math
 import random
+import shutil
 from pathlib import Path
 from typing import Dict, List
 
@@ -21,7 +22,7 @@ def create_split_name_list_from_ratios(split_ratios: Dict[str, float], n_items:
 
 
 def hash_file_xxhash(path: Path, chunk_size: int = 262144) -> str:
-    hasher = xxhash.xxh3_64()
+    hasher = xxhash.xxh3_128()
 
     with open(path, "rb") as f:
         for chunk in iter(lambda: f.read(chunk_size), b""):  # 8192, 16384, 32768, 65536
@@ -30,7 +31,7 @@ def hash_file_xxhash(path: Path, chunk_size: int = 262144) -> str:
 
 
 def hash_from_bytes(data: bytes) -> str:
-    hasher = xxhash.xxh3_64()
+    hasher = xxhash.xxh3_128()
     hasher.update(data)
     return hasher.hexdigest()
 
@@ -40,14 +41,46 @@ def save_image_with_hash_name(image: np.ndarray, path_folder: Path) -> Path:
     buffer = io.BytesIO()
     pil_image.save(buffer, format="PNG")
     hash_value = hash_from_bytes(buffer.getvalue())
-    path_image = Path(path_folder) / f"{hash_value}.png"
+    path_image = Path(path_folder) / relative_path_from_hash(hash=hash_value, suffix=".png")
+    path_image.parent.mkdir(parents=True, exist_ok=True)
     pil_image.save(path_image)
     return path_image
 
 
-def filename_as_hash_from_path(path_image: Path) -> str:
-    hash = hash_file_xxhash(path_image)
-    return f"{hash}{path_image.suffix}"
+def copy_and_rename_file_to_hash_value(path_source: Path, path_dataset_root: Path) -> Path:
+    """
+    Copies a file to a dataset root directory with a hash-based name and sub-directory structure.
+
+    E.g. for an "image.png" with hash "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4", the image will be copied to
+    'path_dataset_root / "data" / "dfe" / "dfe8f3b1c2a4f5b6c7d8e9f0a1b2c3d4.png"'
+    Notice that the hash is used for both the filename and the subfolder name.
+
+    Placing image/video files into multiple sub-folders (instead of one large folder) is seemingly
+    unnecessary, but it is actually a requirement when the dataset is later downloaded from S3.
+
+    The reason is that AWS has a rate limit of 3500 ops/sec per prefix (sub-folder) in S3 - meaning we can "only"
+    download 3500 files per second from a single folder (prefix) in S3.
+
+    For even a single user, we found that this limit was being reached when files are stored in single folder (prefix)
+    in S3. To support multiple users and concurrent experiments, we are required to separate files into
+    multiple sub-folders (prefixes) in S3 to not hit the rate limit.
+    """
+
+    if not path_source.exists():
+        raise FileNotFoundError(f"Source file {path_source} does not exist.")
+
+    hash_value = hash_file_xxhash(path_source)
+    path_file = path_dataset_root / relative_path_from_hash(hash=hash_value, suffix=path_source.suffix)
+    path_file.parent.mkdir(parents=True, exist_ok=True)
+    if not path_file.exists():
+        shutil.copy2(path_source, path_file)
+
+    return path_file
+
+
+def relative_path_from_hash(hash: str, suffix: str) -> Path:
+    path_file = Path("data") / hash[:3] / f"{hash}{suffix}"
+    return path_file
 
 
 def split_sizes_from_ratios(n_items: int, split_ratios: Dict[str, float]) -> Dict[str, int]:

hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -216,6 +216,16 @@ class DatasetRecipe(Serializable):
         json_str = self.as_json_str(indent=indent)
         path_json.write_text(json_str, encoding="utf-8")
 
+    ### Helper methods ###
+    def get_dataset_names(self) -> List[str]:
+        """
+        Get all dataset names added with 'from_name'.
+        Function recursively gathers dataset names.
+        """
+        if self.creation is None:
+            return []
+        return self.creation.get_dataset_names()
+
     ### Validation and Serialization ###
     @field_validator("creation", mode="plain")
     @classmethod
@@ -282,7 +292,10 @@ class FromPath(RecipeCreation):
         return HafniaDataset.from_path
 
     def as_short_name(self) -> str:
-        return f"'{self.path_folder}'".replace(os.sep, "|")
+        return f"'{self.path_folder}'".replace(os.sep, "-")
+
+    def get_dataset_names(self) -> List[str]:
+        return []  # Only counts 'from_name' datasets
 
 
 class FromName(RecipeCreation):
@@ -297,6 +310,9 @@ class FromName(RecipeCreation):
     def as_short_name(self) -> str:
         return self.name
 
+    def get_dataset_names(self) -> List[str]:
+        return [self.name]
+
 
 class FromMerge(RecipeCreation):
     recipe0: DatasetRecipe
@@ -310,6 +326,11 @@ class FromMerge(RecipeCreation):
         merger = FromMerger(recipes=[self.recipe0, self.recipe1])
         return merger.as_short_name()
 
+    def get_dataset_names(self) -> List[str]:
+        """Get the dataset names from the merged recipes."""
+        names = [*self.recipe0.creation.get_dataset_names(), *self.recipe1.creation.get_dataset_names()]
+        return names
+
 
 class FromMerger(RecipeCreation):
     recipes: List[DatasetRecipe]
@@ -325,3 +346,40 @@ class FromMerger(RecipeCreation):
 
     def as_short_name(self) -> str:
         return f"Merger({','.join(recipe.as_short_name() for recipe in self.recipes)})"
+
+    def get_dataset_names(self) -> List[str]:
+        """Get the dataset names from the merged recipes."""
+        names = []
+        for recipe in self.recipes:
+            names.extend(recipe.creation.get_dataset_names())
+        return names
+
+
+def extract_dataset_names_from_json_dict(data: dict) -> list[str]:
+    """
+    Extract dataset names recursively from a JSON dictionary added with 'from_name'.
+
+    Even if the same functionality is achieved with `DatasetRecipe.get_dataset_names()`,
+    we want to keep this function in 'dipdatalib' to extract dataset names from json dictionaries
+    directly.
+    """
+    creation_field = data.get("creation")
+    if creation_field is None:
+        return []
+    if creation_field.get("__type__") == "FromName":
+        return [creation_field["name"]]
+    elif creation_field.get("__type__") == "FromMerge":
+        recipe_names = ["recipe0", "recipe1"]
+        dataset_name = []
+        for recipe_name in recipe_names:
+            recipe = creation_field.get(recipe_name)
+            if recipe is None:
+                continue
+            dataset_name.extend(extract_dataset_names_from_json_dict(recipe))
+        return dataset_name
+    elif creation_field.get("__type__") == "FromMerger":
+        dataset_name = []
+        for recipe in creation_field.get("recipes", []):
+            dataset_name.extend(extract_dataset_names_from_json_dict(recipe))
+        return dataset_name
+    return []

hafnia/dataset/dataset_recipe/recipe_types.py

@@ -108,6 +108,10 @@ class RecipeCreation(Serializable):
     def get_function() -> Callable[..., "HafniaDataset"]:
         pass
 
+    @abstractmethod
+    def get_dataset_names(self) -> List[str]:
+        pass
+
     def build(self) -> "HafniaDataset":
         from hafnia.dataset.dataset_recipe.dataset_recipe import DatasetRecipe
 

hafnia/dataset/hafnia_dataset.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 import shutil
 from dataclasses import dataclass
 from pathlib import Path
@@ -182,9 +181,8 @@ class HafniaDataset:
         table = read_table_from_path(path_folder)
 
         # Convert from relative paths to absolute paths
-        table = table.with_columns(
-            pl.concat_str([pl.lit(str(path_folder.absolute()) + os.sep), pl.col("file_name")]).alias("file_name")
-        )
+        dataset_root = path_folder.absolute().as_posix() + "/"
+        table = table.with_columns((dataset_root + pl.col("file_name")).alias("file_name"))
         if check_for_images:
             check_image_paths(table)
         return HafniaDataset(samples=table, info=dataset_info)
@@ -413,30 +411,18 @@ class HafniaDataset:
 
         return True
 
-    def write(self, path_folder: Path, name_by_hash: bool = True, add_version: bool = False) -> None:
+    def write(self, path_folder: Path, add_version: bool = False) -> None:
         user_logger.info(f"Writing dataset to {path_folder}...")
         if not path_folder.exists():
             path_folder.mkdir(parents=True)
-        path_folder_images = path_folder / "data"
-        path_folder_images.mkdir(parents=True, exist_ok=True)
 
         new_relative_paths = []
         for org_path in tqdm(self.samples["file_name"].to_list(), desc="- Copy images"):
-            org_path = Path(org_path)
-            if not org_path.exists():
-                raise FileNotFoundError(f"File {org_path} does not exist in the dataset.")
-            if name_by_hash:
-                filename = dataset_helpers.filename_as_hash_from_path(org_path)
-            else:
-                filename = Path(org_path).name
-            new_path = path_folder_images / filename
-            if not new_path.exists():
-                shutil.copy2(org_path, new_path)
-
-            if not new_path.exists():
-                raise FileNotFoundError(f"File {new_path} does not exist in the dataset.")
+            new_path = dataset_helpers.copy_and_rename_file_to_hash_value(
+                path_source=Path(org_path),
+                path_dataset_root=path_folder,
+            )
             new_relative_paths.append(str(new_path.relative_to(path_folder)))
-
         table = self.samples.with_columns(pl.Series(new_relative_paths).alias("file_name"))
         table.write_ndjson(path_folder / FILENAME_ANNOTATIONS_JSONL)  # Json for readability
         table.write_parquet(path_folder / FILENAME_ANNOTATIONS_PARQUET)  # Parquet for speed

hafnia/platform/datasets.py

@@ -2,6 +2,7 @@ import os
 import shutil
 import subprocess
 import tempfile
+import uuid
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -61,7 +62,12 @@ def download_or_get_dataset_path(
     dataset_id = get_dataset_id(dataset_name=dataset_name, endpoint=endpoint_dataset, api_key=api_key)
     if dataset_id is None:
         sys_logger.error(f"Dataset '{dataset_name}' not found on the Hafnia platform.")
-    access_dataset_endpoint = f"{endpoint_dataset}/{dataset_id}/temporary-credentials"
+
+    if utils.is_hafnia_cloud_job():
+        credentials_endpoint_suffix = "temporary-credentials-hidden"  # Access to hidden datasets
+    else:
+        credentials_endpoint_suffix = "temporary-credentials"  # Access to sample dataset
+    access_dataset_endpoint = f"{endpoint_dataset}/{dataset_id}/{credentials_endpoint_suffix}"
 
     download_dataset_from_access_endpoint(
         endpoint=access_dataset_endpoint,
@@ -80,7 +86,7 @@ def download_dataset_from_access_endpoint(
 ) -> None:
     resource_credentials = get_resource_credentials(endpoint, api_key)
 
-    local_dataset_paths = [str(path_dataset / filename) for filename in DATASET_FILENAMES_REQUIRED]
+    local_dataset_paths = [(path_dataset / filename).as_posix() for filename in DATASET_FILENAMES_REQUIRED]
     s3_uri = resource_credentials.s3_uri()
     s3_dataset_files = [f"{s3_uri}/{filename}" for filename in DATASET_FILENAMES_REQUIRED]
 
@@ -94,7 +100,6 @@ def download_dataset_from_access_endpoint(
 
     if not download_files:
         return
-
     dataset = HafniaDataset.from_path(path_dataset, check_for_images=False)
     fast_copy_files_s3(
         src_paths=dataset.samples[ColumnName.REMOTE_PATH].to_list(),
@@ -124,8 +129,10 @@ def execute_s5cmd_commands(
     description: str = "Executing s5cmd commands",
 ) -> List[str]:
     append_envs = append_envs or {}
-    with tempfile.NamedTemporaryFile(suffix=".txt") as tmp_file:
-        tmp_file_path = Path(tmp_file.name)
+    # In Windows default "Temp" directory can not be deleted that is why we need to create a
+    # temporary directory.
+    with tempfile.TemporaryDirectory() as temp_dir:
+        tmp_file_path = Path(temp_dir, f"{uuid.uuid4().hex}.txt")
         tmp_file_path.write_text("\n".join(commands))
         run_cmds = [
             "s5cmd",

{hafnia-0.2.1.dist-info → hafnia-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.2.1
+Version: 0.2.3
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE

{hafnia-0.2.1.dist-info → hafnia-0.2.3.dist-info}/RECORD

@@ -1,27 +1,26 @@
 cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cli/__main__.py,sha256=pKYvJOk0toACDMrgEeUYT3P5EQPFmXdMRIQuLZLd3dc,1603
-cli/config.py,sha256=-BTdljjC42hXHb1P0yewea9knzgSBFsb909qJ5DEkCo,5531
+cli/__main__.py,sha256=WPOiwolX6J5qLBQGv_b64PGYWScrwqbdVh5zs5AbzVk,1436
+cli/config.py,sha256=hkVd1WyrRqLBgJbKWJkXBzRWlvBRr8dt_8f722yZiiM,6063
 cli/consts.py,sha256=sj0MRwbbCT2Yl77FPddck1VWkFxp7QY6I9l1o75j_aE,963
 cli/dataset_cmds.py,sha256=VUMhnHGYPtNNJUK9aobKTx2zpVzLex4gTMmyQXuzCVw,1623
 cli/experiment_cmds.py,sha256=L-k_ZJ4B7I4cA8OvHcheSwXM6nx9aTF9G7eKBzAcOzQ,1961
-cli/profile_cmds.py,sha256=-HQcFgYI6Rqaefi0Nj-91KhiqPKUj7zOaiJWbHx_bac,3196
+cli/profile_cmds.py,sha256=qop9hW4EjbTEQ5d28tiIHCaG6iUM9opQcj289qI-tkg,3220
 cli/recipe_cmds.py,sha256=qnMfF-te47HXNkgyA0hm9X3etDQsqMnrVEGDCrzVjZU,1462
 cli/runc_cmds.py,sha256=QqhQe2sd7tK1Bl2aGfIWRyJjpP6F7Tducg7HULrHsZ4,4958
 hafnia/__init__.py,sha256=Zphq-cQoX95Z11zm4lkrU-YiAJxddR7IBfwDkxeHoDE,108
-hafnia/helper_testing.py,sha256=GnaNhXdY81arjCT9M2RUAmvn2-aIzRqlCtbWwGbOIaY,3901
 hafnia/http.py,sha256=HoPB03IL6e-nglTrw1NGT6sDx1T8VNas5HjTT1QZHnU,3035
 hafnia/log.py,sha256=sWF8tz78yBtwZ9ddzm19L1MBSBJ3L4G704IGeT1_OEU,784
 hafnia/torch_helpers.py,sha256=ho65B0WIu_SjbaKPRL4wabDNrnVumWH8QSXVH4r7NAY,11605
 hafnia/utils.py,sha256=aTZaeHldXn4Jx_AR2BYATxtLCRrBKBjjDFmpSZTSvV4,5138
 hafnia/data/__init__.py,sha256=o9QjiGbEcNa6r-qDmwwmxPXf-1UitNl5-WxFNcujqsg,111
 hafnia/data/factory.py,sha256=OY6l6c9UKk6OUDhG4Akb2VgcSaTRLHlbSndAe1HuW2U,813
-hafnia/dataset/dataset_helpers.py,sha256=WVCpbUfNbHy7MZJqJ3OyJF8k1hSObo3kScxpXT17Sj8,3510
+hafnia/dataset/dataset_helpers.py,sha256=x6jub_aLWJn-sWSvXT_0-nwzzPG1xMM9yBMNDx6Nufw,5190
 hafnia/dataset/dataset_names.py,sha256=mp7A_TOqgoqHUEBCPC4ReKNJ93cxwQB451owoCqD6yM,2120
 hafnia/dataset/dataset_upload_helper.py,sha256=D1BGaeEar4McpUvXj4Yy8nk1tr12IEVhP_Ma47OoWmU,21150
-hafnia/dataset/hafnia_dataset.py,sha256=4SJUq7pAqLkcFzgnOUUx8ERraE_sABctOAsONBJExME,27664
-hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=DbPLlmshF6DC98Cwko04XtBaXgSg966LZKR6JXD_9Sg,13632
+hafnia/dataset/hafnia_dataset.py,sha256=6yy13mU9OnJfIoG8R1ZS7mmkNR_VKOD9B4L2KjdS76I,27078
+hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=3rMMd1xlfQzElxF9P2uNyKqK-GjbCADZgcFDoPviTmU,15796
 hafnia/dataset/dataset_recipe/recipe_transforms.py,sha256=wh1y2XyX0PwOwfuzJ3_17KKng2Rk0zLlgdfSHfS1SyM,1305
-hafnia/dataset/dataset_recipe/recipe_types.py,sha256=6LxfanhX9ihof1gGSonoC-56zSWsI8k2aS4Uw_QgXoM,5176
+hafnia/dataset/dataset_recipe/recipe_types.py,sha256=breT8x81FcmiZ82U_D9FBut0F-eWwOeBWBOse9kNAYU,5256
 hafnia/dataset/operations/dataset_stats.py,sha256=tSHPmkXt4WNgjf5-j3jIrsSy1Ajld3619AkUHaesXb4,445
 hafnia/dataset/operations/dataset_transformations.py,sha256=4ibC11upEtRGJgoFLv8lUnglv2xANZVfNdsvI1BMvfM,2960
 hafnia/dataset/operations/table_transformations.py,sha256=kCLbLRdiFSx1JG0IWtaKkhWcMtM7hy8zgm0Ehz0zO_g,7639
@@ -38,13 +37,13 @@ hafnia/experiment/__init__.py,sha256=OEFE6HqhO5zcTCLZcPcPVjIg7wMFFnvZ1uOtAVhRz7M
 hafnia/experiment/hafnia_logger.py,sha256=dnV3VPzJK7DSeUh0g4Hk9w1g-eSXcVqJD9If0h2d2GE,6885
 hafnia/platform/__init__.py,sha256=zJsR6Hy_0iUcC9xL-lBnqR0mLfF4EUr_VXa_XQA7SlA,455
 hafnia/platform/builder.py,sha256=_g8ykQWETz5Y4Np9QU1a6wIzbbJwXCkbiOCA6JcF5Rc,5742
-hafnia/platform/datasets.py,sha256=J252hrejrBWUdS6hY4lRc9_SbYy7CMD92068lLHjPC8,6953
+hafnia/platform/datasets.py,sha256=mRv8A0JSMYdBr3_0qqrw21kKSSPMSYUFWyPVNxYqZrA,7344
 hafnia/platform/download.py,sha256=oJzdxSIDTuw1an7maC6I7A5nZvDaZPhUkuAmyRwN9Kc,6843
 hafnia/platform/experiment.py,sha256=-nAfTmn1c8sE6pHDCTNZvWDTopkXndarJAPIGvsnk60,2389
 hafnia/visualizations/colors.py,sha256=003eAJVnBal4abaYIIpsrT7erIOIjTUHHYVJ1Tj1CDc,5226
 hafnia/visualizations/image_visualizations.py,sha256=RuFFj2fJCm9dxl2Lq0MumJHF81ZnX-IsDsTxm8ZFV9A,7313
-hafnia-0.2.1.dist-info/METADATA,sha256=A1_OEYNslARBFGoYBPm7_-3YivfUeA8adwUUbsM3UsY,19040
-hafnia-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hafnia-0.2.1.dist-info/entry_points.txt,sha256=FCJVIQ8GP2VE9I3eeGVF5eLxVDNW_01pOJCpG_CGnMM,45
-hafnia-0.2.1.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
-hafnia-0.2.1.dist-info/RECORD,,
+hafnia-0.2.3.dist-info/METADATA,sha256=7ZwSgwrbliqkHYTy11Sg01su1z-aZiWSGs9Y6qHBQ4k,19040
+hafnia-0.2.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hafnia-0.2.3.dist-info/entry_points.txt,sha256=FCJVIQ8GP2VE9I3eeGVF5eLxVDNW_01pOJCpG_CGnMM,45
+hafnia-0.2.3.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
+hafnia-0.2.3.dist-info/RECORD,,

hafnia/helper_testing.py

@@ -1,108 +0,0 @@
-from inspect import getmembers, isfunction, signature
-from pathlib import Path
-from types import FunctionType
-from typing import Any, Callable, Dict, Union, get_origin
-
-from hafnia import utils
-from hafnia.dataset.dataset_names import FILENAME_ANNOTATIONS_JSONL, DatasetVariant
-from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
-
-MICRO_DATASETS = {
-    "tiny-dataset": utils.PATH_DATASETS / "tiny-dataset",
-    "coco-2017": utils.PATH_DATASETS / "coco-2017",
-}
-
-
-def get_path_workspace() -> Path:
-    return Path(__file__).parents[2]
-
-
-def get_path_expected_images() -> Path:
-    return get_path_workspace() / "tests" / "data" / "expected_images"
-
-
-def get_path_test_data() -> Path:
-    return get_path_workspace() / "tests" / "data"
-
-
-def get_path_micro_hafnia_dataset_no_check() -> Path:
-    return get_path_test_data() / "micro_test_datasets"
-
-
-def get_path_micro_hafnia_dataset(dataset_name: str, force_update=False) -> Path:
-    import pytest
-
-    if dataset_name not in MICRO_DATASETS:
-        raise ValueError(f"Dataset name '{dataset_name}' is not recognized. Available options: {list(MICRO_DATASETS)}")
-    path_dataset = MICRO_DATASETS[dataset_name]
-
-    path_test_dataset = get_path_micro_hafnia_dataset_no_check() / dataset_name
-    path_test_dataset_annotations = path_test_dataset / FILENAME_ANNOTATIONS_JSONL
-    if path_test_dataset_annotations.exists() and not force_update:
-        return path_test_dataset
-
-    hafnia_dataset = HafniaDataset.from_path(path_dataset / DatasetVariant.SAMPLE.value)
-    hafnia_dataset = hafnia_dataset.select_samples(n_samples=3, seed=42)
-    hafnia_dataset.write(path_test_dataset)
-
-    if force_update:
-        pytest.fail(
-            "Sample image and metadata have been updated using 'force_update=True'. Set 'force_update=False' and rerun the test."
-        )
-    pytest.fail("Missing test sample image. Please rerun the test.")
-    return path_test_dataset
-
-
-def get_sample_micro_hafnia_dataset(dataset_name: str, force_update=False) -> Sample:
-    micro_dataset = get_micro_hafnia_dataset(dataset_name=dataset_name, force_update=force_update)
-    sample_dict = micro_dataset[0]
-    sample = Sample(**sample_dict)
-    return sample
-
-
-def get_micro_hafnia_dataset(dataset_name: str, force_update: bool = False) -> HafniaDataset:
-    path_dataset = get_path_micro_hafnia_dataset(dataset_name=dataset_name, force_update=force_update)
-    hafnia_dataset = HafniaDataset.from_path(path_dataset)
-    return hafnia_dataset
-
-
-def is_hafnia_configured() -> bool:
-    """
-    Check if Hafnia is configured by verifying if the API key is set.
-    """
-    from cli.config import Config
-
-    return Config().is_configured()
-
-
-def is_typing_type(annotation: Any) -> bool:
-    return get_origin(annotation) is not None
-
-
-def annotation_as_string(annotation: Union[type, str]) -> str:
-    """Convert type annotation to string."""
-    if isinstance(annotation, str):
-        return annotation.replace("'", "")
-    if is_typing_type(annotation):  # Is using typing types like List, Dict, etc.
-        return str(annotation).replace("typing.", "")
-    if hasattr(annotation, "__name__"):
-        return annotation.__name__
-    return str(annotation)
-
-
-def get_hafnia_functions_from_module(python_module) -> Dict[str, FunctionType]:
-    def dataset_is_first_arg(func: Callable) -> bool:
-        """
-        Check if the function has 'HafniaDataset' as the first parameter.
-        """
-        func_signature = signature(func)
-        params = func_signature.parameters
-        if len(params) == 0:
-            return False
-        first_argument_type = list(params.values())[0]
-
-        annotation_as_str = annotation_as_string(first_argument_type.annotation)
-        return annotation_as_str == "HafniaDataset"
-
-    functions = {func[0]: func[1] for func in getmembers(python_module, isfunction) if dataset_is_first_arg(func[1])}
-    return functions