hafnia 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

{hafnia-0.2.0.dist-info → hafnia-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.2.0
+Version: 0.2.1
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -147,22 +147,20 @@ The `HafniaDataset` object provides a convenient way to interact with the datase
 creating splits, accessing samples, printing statistics, saving to and loading from disk.
 
 In essence, the `HafniaDataset` class contains `dataset.info` with dataset information 
-and `dataset.table` with annotations as a polars DataFrame
+and `dataset.samples` with annotations as a polars DataFrame
 
 ```python
 # Annotations are stored in a polars DataFrame
-print(dataset.table.head(2))
+print(dataset.samples.head(2))
 shape: (2, 14)
-┌──────────┬────────────────────────────────┬────────┬───────┬───┬───────────────────────────────┬──────────┬──────────┬───────────────────────────────┐
-│ image_id ┆ file_name                      ┆ height ┆ width ┆ … ┆ objects                       ┆ bitmasks ┆ polygons ┆ meta                          │
-│ ---      ┆ ---                            ┆ ---    ┆ ---   ┆   ┆ ---                           ┆ ---      ┆ ---      ┆ ---                           │
-│ str      ┆ str                            ┆ i64    ┆ i64   ┆   ┆ list[struct[12]]              ┆ null     ┆ null     ┆ struct[5]                     │
-╞══════════╪════════════════════════════════╪════════╪═══════╪═══╪═══════════════════════════════╪══════════╪══════════╪═══════════════════════════════╡
-│ 7800     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23," ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ V…                            ┆          ┆          ┆ 0…                            │
-│ 7900     ┆ /home/ubuntu/code/hafnia/.data ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0 ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30: │
-│          ┆ …                              ┆        ┆       ┆   ┆ .…                            ┆          ┆          ┆ 0…                            │
-└──────────┴────────────────────────────────┴────────┴───────┴───┴───────────────────────────────┴──────────┴──────────┴───────────────────────────────┘
+┌──────────────┬─────────────────────────────────┬────────┬───────┬───┬─────────────────────────────────┬──────────┬──────────┬─────────────────────────────────┐
+│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ objects                         ┆ bitmasks ┆ polygons ┆ meta                            │
+│ ---          ┆ ---                             ┆ ---    ┆ ---   ┆   ┆ ---                             ┆ ---      ┆ ---      ┆ ---                             │
+│ u32          ┆ str                             ┆ i64    ┆ i64   ┆   ┆ list[struct[11]]                ┆ null     ┆ null     ┆ struct[5]                       │
+╞══════════════╪═════════════════════════════════╪════════╪═══════╪═══╪═════════════════════════════════╪══════════╪══════════╪═════════════════════════════════╡
+│ 0            ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.0492,0.0357,0.2083,0.23,"V… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+│ 100          ┆ /home/ubuntu/code/hafnia/.data… ┆ 1080   ┆ 1920  ┆ … ┆ [{0.146382,0.078704,0.42963,0.… ┆ null     ┆ null     ┆ {120.0,1.0,"2024-07-10T18:30:0… │
+└──────────────┴─────────────────────────────────┴────────┴───────┴───┴─────────────────────────────────┴──────────┴──────────┴─────────────────────────────────┘
 ```
 
 ```python
@@ -235,16 +233,15 @@ It also contain annotations as primitive types such as `Bbox`, `Classification`.
 ```python
 rich.print(sample)
 Sample(
-    image_id='7800',
-    file_name='data/video_0026a86b-2f43-49f2-a17c-59244d10a585_1fps_mp4_frame_0
-0000.png',
+    sample_index=120,
+    file_name='/home/ubuntu/code/hafnia/.data/datasets/midwest-vehicle-detection/data/343403325f27e390.png',
     height=1080,
     width=1920,
-    split='test',
+    split='train',
     is_sample=True,
-    frame_number=None,
-    video_name=None,
-    remote_path=None,
+    collection_index=None,
+    collection_id=None,
+    remote_path='s3://mdi-production-midwest-vehicle-detection/sample/data/343403325f27e390.png',
     classifications=[
         Classification(
             class_name='Clear',

{hafnia-0.2.0.dist-info → hafnia-0.2.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cli/__main__.py,sha256=pKYvJOk0toACDMrgEeUYT3P5EQPFmXdMRIQuLZLd3dc,1603
-cli/config.py,sha256=R9w0NKIOtIxRKNs7ieeUrIKwRkrTlK5PqOVjc5VYljE,4923
+cli/config.py,sha256=-BTdljjC42hXHb1P0yewea9knzgSBFsb909qJ5DEkCo,5531
 cli/consts.py,sha256=sj0MRwbbCT2Yl77FPddck1VWkFxp7QY6I9l1o75j_aE,963
 cli/dataset_cmds.py,sha256=VUMhnHGYPtNNJUK9aobKTx2zpVzLex4gTMmyQXuzCVw,1623
 cli/experiment_cmds.py,sha256=L-k_ZJ4B7I4cA8OvHcheSwXM6nx9aTF9G7eKBzAcOzQ,1961
@@ -8,19 +8,23 @@ cli/profile_cmds.py,sha256=-HQcFgYI6Rqaefi0Nj-91KhiqPKUj7zOaiJWbHx_bac,3196
 cli/recipe_cmds.py,sha256=qnMfF-te47HXNkgyA0hm9X3etDQsqMnrVEGDCrzVjZU,1462
 cli/runc_cmds.py,sha256=QqhQe2sd7tK1Bl2aGfIWRyJjpP6F7Tducg7HULrHsZ4,4958
 hafnia/__init__.py,sha256=Zphq-cQoX95Z11zm4lkrU-YiAJxddR7IBfwDkxeHoDE,108
-hafnia/helper_testing.py,sha256=4pIG5sA1FycbZRFmfG7CyhZMdQgWob86JCzX8ALuHfQ,2344
+hafnia/helper_testing.py,sha256=GnaNhXdY81arjCT9M2RUAmvn2-aIzRqlCtbWwGbOIaY,3901
 hafnia/http.py,sha256=HoPB03IL6e-nglTrw1NGT6sDx1T8VNas5HjTT1QZHnU,3035
 hafnia/log.py,sha256=sWF8tz78yBtwZ9ddzm19L1MBSBJ3L4G704IGeT1_OEU,784
 hafnia/torch_helpers.py,sha256=ho65B0WIu_SjbaKPRL4wabDNrnVumWH8QSXVH4r7NAY,11605
-hafnia/utils.py,sha256=DFMALTCOxGsTWZ6qIVbm55Wr8xxwukj87sb0oYB6w64,4660
+hafnia/utils.py,sha256=aTZaeHldXn4Jx_AR2BYATxtLCRrBKBjjDFmpSZTSvV4,5138
 hafnia/data/__init__.py,sha256=o9QjiGbEcNa6r-qDmwwmxPXf-1UitNl5-WxFNcujqsg,111
-hafnia/data/factory.py,sha256=RC36LSPCULJqHn93vGeadzNXoItHhsYZA8YxURY1nd0,717
+hafnia/data/factory.py,sha256=OY6l6c9UKk6OUDhG4Akb2VgcSaTRLHlbSndAe1HuW2U,813
 hafnia/dataset/dataset_helpers.py,sha256=WVCpbUfNbHy7MZJqJ3OyJF8k1hSObo3kScxpXT17Sj8,3510
-hafnia/dataset/dataset_names.py,sha256=DhnFDlaq-PDNeGpvE1pFmhZ2M7HKeQNyRo6PwuchVe8,2074
-hafnia/dataset/dataset_transformation.py,sha256=LyXt4LOQaZ4EkYspKlqp_W3IbHJxB1AxVNkuuShiKg0,7761
+hafnia/dataset/dataset_names.py,sha256=mp7A_TOqgoqHUEBCPC4ReKNJ93cxwQB451owoCqD6yM,2120
 hafnia/dataset/dataset_upload_helper.py,sha256=D1BGaeEar4McpUvXj4Yy8nk1tr12IEVhP_Ma47OoWmU,21150
-hafnia/dataset/hafnia_dataset.py,sha256=IJfPQivzM0wAZgGP7fx7-Mpoe8mhrEpLzBJ9JP3N6D4,19970
-hafnia/dataset/table_transformations.py,sha256=WuAbEXaNKyacmY4FShNkUoG3-rVC5pTKntO8Z7CJNoI,7628
+hafnia/dataset/hafnia_dataset.py,sha256=4SJUq7pAqLkcFzgnOUUx8ERraE_sABctOAsONBJExME,27664
+hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=DbPLlmshF6DC98Cwko04XtBaXgSg966LZKR6JXD_9Sg,13632
+hafnia/dataset/dataset_recipe/recipe_transforms.py,sha256=wh1y2XyX0PwOwfuzJ3_17KKng2Rk0zLlgdfSHfS1SyM,1305
+hafnia/dataset/dataset_recipe/recipe_types.py,sha256=6LxfanhX9ihof1gGSonoC-56zSWsI8k2aS4Uw_QgXoM,5176
+hafnia/dataset/operations/dataset_stats.py,sha256=tSHPmkXt4WNgjf5-j3jIrsSy1Ajld3619AkUHaesXb4,445
+hafnia/dataset/operations/dataset_transformations.py,sha256=4ibC11upEtRGJgoFLv8lUnglv2xANZVfNdsvI1BMvfM,2960
+hafnia/dataset/operations/table_transformations.py,sha256=kCLbLRdiFSx1JG0IWtaKkhWcMtM7hy8zgm0Ehz0zO_g,7639
 hafnia/dataset/primitives/__init__.py,sha256=LAdTeK5GgmaF1se8f0Yj1lOTNqplXGL87kLeOnv1D8Q,627
 hafnia/dataset/primitives/bbox.py,sha256=HXYYy5BLNZwh-bO7aiAWg3z0OurUev8ISa-vYey8b8A,6055
 hafnia/dataset/primitives/bitmask.py,sha256=mq_wchMqGupJDc-a-mJh9uBO_mjHcXpLH49g591doAM,7619
@@ -31,16 +35,16 @@ hafnia/dataset/primitives/primitive.py,sha256=7jxcyFADVGf95pjeQHEOqAnR9eucLpxA2h
 hafnia/dataset/primitives/segmentation.py,sha256=jUMjOmYr9j4An3YSCw5CJC1W8ihXAbus3CXaTOpc7Xw,1905
 hafnia/dataset/primitives/utils.py,sha256=3gT1as-xXEj8CamoIuBb9gQwUN9Ae9qnqtqF_uEe0zo,1993
 hafnia/experiment/__init__.py,sha256=OEFE6HqhO5zcTCLZcPcPVjIg7wMFFnvZ1uOtAVhRz7M,85
-hafnia/experiment/hafnia_logger.py,sha256=TNGsb5aP_FE0t2NQofik6chK6dDWckYJMikiGS9GAd0,6855
+hafnia/experiment/hafnia_logger.py,sha256=dnV3VPzJK7DSeUh0g4Hk9w1g-eSXcVqJD9If0h2d2GE,6885
 hafnia/platform/__init__.py,sha256=zJsR6Hy_0iUcC9xL-lBnqR0mLfF4EUr_VXa_XQA7SlA,455
 hafnia/platform/builder.py,sha256=_g8ykQWETz5Y4Np9QU1a6wIzbbJwXCkbiOCA6JcF5Rc,5742
-hafnia/platform/datasets.py,sha256=Sc2vkT3QYMwaSWp2Pf2E2BLW4YG40Tqdhn6vgkTgpQk,6643
+hafnia/platform/datasets.py,sha256=J252hrejrBWUdS6hY4lRc9_SbYy7CMD92068lLHjPC8,6953
 hafnia/platform/download.py,sha256=oJzdxSIDTuw1an7maC6I7A5nZvDaZPhUkuAmyRwN9Kc,6843
 hafnia/platform/experiment.py,sha256=-nAfTmn1c8sE6pHDCTNZvWDTopkXndarJAPIGvsnk60,2389
 hafnia/visualizations/colors.py,sha256=003eAJVnBal4abaYIIpsrT7erIOIjTUHHYVJ1Tj1CDc,5226
-hafnia/visualizations/image_visualizations.py,sha256=mNlRfG-ooJw0vqKyNCuAZOBpjiFuqJH3mjd6spInJs4,7318
-hafnia-0.2.0.dist-info/METADATA,sha256=sYxIR2FnabM4KApFnNw_zKHPEDjQ9abU3dpon4RTXOY,19153
-hafnia-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hafnia-0.2.0.dist-info/entry_points.txt,sha256=FCJVIQ8GP2VE9I3eeGVF5eLxVDNW_01pOJCpG_CGnMM,45
-hafnia-0.2.0.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
-hafnia-0.2.0.dist-info/RECORD,,
+hafnia/visualizations/image_visualizations.py,sha256=RuFFj2fJCm9dxl2Lq0MumJHF81ZnX-IsDsTxm8ZFV9A,7313
+hafnia-0.2.1.dist-info/METADATA,sha256=A1_OEYNslARBFGoYBPm7_-3YivfUeA8adwUUbsM3UsY,19040
+hafnia-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hafnia-0.2.1.dist-info/entry_points.txt,sha256=FCJVIQ8GP2VE9I3eeGVF5eLxVDNW_01pOJCpG_CGnMM,45
+hafnia-0.2.1.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
+hafnia-0.2.1.dist-info/RECORD,,

hafnia/dataset/dataset_transformation.py

@@ -1,187 +0,0 @@
-import hashlib
-import shutil
-from pathlib import Path
-from random import Random
-from typing import TYPE_CHECKING, Callable, Dict
-
-import cv2
-import numpy as np
-import polars as pl
-from PIL import Image
-from tqdm import tqdm
-
-from hafnia.dataset import dataset_helpers
-from hafnia.dataset.dataset_names import ColumnName
-from hafnia.log import user_logger
-
-if TYPE_CHECKING:
-    from hafnia.dataset.hafnia_dataset import HafniaDataset
-
-
-### Image transformations ###
-class AnonymizeByPixelation:
-    def __init__(self, resize_factor: float = 0.10):
-        self.resize_factor = resize_factor
-
-    def __call__(self, frame: np.ndarray) -> np.ndarray:
-        org_size = frame.shape[:2]
-        frame = cv2.resize(frame, (0, 0), fx=self.resize_factor, fy=self.resize_factor)
-        frame = cv2.resize(frame, org_size[::-1], interpolation=cv2.INTER_NEAREST)
-        return frame
-
-
-def splits_by_ratios(dataset: "HafniaDataset", split_ratios: Dict[str, float], seed: int = 42) -> "HafniaDataset":
-    """
-    Divides the dataset into splits based on the provided ratios.
-
-    Example: Defining split ratios and applying the transformation
-
-    >>> dataset = HafniaDataset.read_from_path(Path("path/to/dataset"))
-    >>> split_ratios = {SplitName.TRAIN: 0.8, SplitName.VAL: 0.1, SplitName.TEST: 0.1}
-    >>> dataset_with_splits = splits_by_ratios(dataset, split_ratios, seed=42)
-    Or use the function as a
-    >>> dataset_with_splits = dataset.splits_by_ratios(split_ratios, seed=42)
-    """
-    n_items = len(dataset)
-    split_name_column = dataset_helpers.create_split_name_list_from_ratios(
-        split_ratios=split_ratios, n_items=n_items, seed=seed
-    )
-    table = dataset.samples.with_columns(pl.Series(split_name_column).alias("split"))
-    return dataset.update_table(table)
-
-
-def divide_split_into_multiple_splits(
-    dataset: "HafniaDataset",
-    divide_split_name: str,
-    split_ratios: Dict[str, float],
-) -> "HafniaDataset":
-    """
-    Divides a dataset split ('divide_split_name') into multiple splits based on the provided split
-    ratios ('split_ratios'). This is especially useful for some open datasets where they have only provide
-    two splits or only provide annotations for two splits. This function allows you to create additional
-    splits based on the provided ratios.
-
-    Example: Defining split ratios and applying the transformation
-    >>> dataset = HafniaDataset.read_from_path(Path("path/to/dataset"))
-    >>> divide_split_name = SplitName.TEST
-    >>> split_ratios = {SplitName.TEST: 0.8, SplitName.VAL: 0.2}
-    >>> dataset_with_splits = divide_split_into_multiple_splits(dataset, divide_split_name, split_ratios)
-    """
-    dataset_split_to_be_divided = dataset.create_split_dataset(split_name=divide_split_name)
-    if len(dataset_split_to_be_divided) == 0:
-        split_counts = dict(dataset.samples.select(pl.col(ColumnName.SPLIT).value_counts()).iter_rows())
-        raise ValueError(
-            f"No samples in the '{divide_split_name}' split to divide into multiple splits. {split_counts=}"
-        )
-    assert len(dataset_split_to_be_divided) > 0, f"No samples in the '{divide_split_name}' split!"
-    dataset_split_to_be_divided = dataset_split_to_be_divided.split_by_ratios(split_ratios=split_ratios, seed=42)
-
-    remaining_data = dataset.samples.filter(pl.col(ColumnName.SPLIT).is_in([divide_split_name]).not_())
-    new_table = pl.concat([remaining_data, dataset_split_to_be_divided.samples], how="vertical")
-    dataset_new = dataset.update_table(new_table)
-    return dataset_new
-
-
-def shuffle_dataset(dataset: "HafniaDataset", seed: int = 42) -> "HafniaDataset":
-    table = dataset.samples.sample(n=len(dataset), with_replacement=False, seed=seed, shuffle=True)
-    return dataset.update_table(table)
-
-
-def sample(dataset: "HafniaDataset", n_samples: int, shuffle: bool = True, seed: int = 42) -> "HafniaDataset":
-    table = dataset.samples.sample(n=n_samples, with_replacement=False, seed=seed, shuffle=shuffle)
-    return dataset.update_table(table)
-
-
-def define_sample_set_by_size(dataset: "HafniaDataset", n_samples: int, seed: int = 42) -> "HafniaDataset":
-    is_sample_indices = Random(seed).sample(range(len(dataset)), n_samples)
-    is_sample_column = [False for _ in range(len(dataset))]
-    for idx in is_sample_indices:
-        is_sample_column[idx] = True
-
-    table = dataset.samples.with_columns(pl.Series(is_sample_column).alias("is_sample"))
-    return dataset.update_table(table)
-
-
-def transform_images(
-    dataset: "HafniaDataset",
-    transform: Callable[[np.ndarray], np.ndarray],
-    path_output: Path,
-) -> "HafniaDataset":
-    new_paths = []
-    path_image_folder = path_output / "data"
-    path_image_folder.mkdir(parents=True, exist_ok=True)
-
-    for org_path in tqdm(dataset.samples["file_name"].to_list(), desc="Transform images"):
-        org_path = Path(org_path)
-        if not org_path.exists():
-            raise FileNotFoundError(f"File {org_path} does not exist in the dataset.")
-
-        image = np.array(Image.open(org_path))
-        image_transformed = transform(image)
-        new_path = dataset_helpers.save_image_with_hash_name(image_transformed, path_image_folder)
-
-        if not new_path.exists():
-            raise FileNotFoundError(f"Transformed file {new_path} does not exist in the dataset.")
-        new_paths.append(str(new_path))
-
-    table = dataset.samples.with_columns(pl.Series(new_paths).alias("file_name"))
-    return dataset.update_table(table)
-
-
-def rename_to_unique_image_names(dataset: "HafniaDataset", path_output: Path) -> "HafniaDataset":
-    user_logger.info(f"Copy images to have unique filenames. New path is '{path_output}'")
-    shutil.rmtree(path_output, ignore_errors=True)  # Remove the output folder if it exists
-    new_paths = []
-    for org_path in tqdm(dataset.samples["file_name"].to_list(), desc="- Rename/copy images"):
-        org_path = Path(org_path)
-        if not org_path.exists():
-            raise FileNotFoundError(f"File {org_path} does not exist in the dataset.")
-
-        hash_name = hashlib.md5(str(org_path).encode()).hexdigest()[
-            :6
-        ]  # Generate a unique name based on the original file name
-        new_path = path_output / "data" / f"{hash_name}_{org_path.name}"
-        if not new_path.parent.exists():
-            new_path.parent.mkdir(parents=True, exist_ok=True)
-
-        shutil.copyfile(org_path, new_path)  # Copy the original file to the new path
-        new_paths.append(str(new_path))
-
-    table = dataset.samples.with_columns(pl.Series(new_paths).alias("file_name"))
-    return dataset.update_table(table)
-
-
-### Hafnia Dataset Transformations ###
-class SplitsByRatios:
-    def __init__(self, split_ratios: dict, seed: int = 42):
-        self.split_ratios = split_ratios
-        self.seed = seed
-
-    def __call__(self, dataset: "HafniaDataset") -> "HafniaDataset":
-        return splits_by_ratios(dataset, self.split_ratios, self.seed)
-
-
-class ShuffleDataset:
-    def __init__(self, seed: int = 42):
-        self.seed = seed
-
-    def __call__(self, dataset: "HafniaDataset") -> "HafniaDataset":
-        return shuffle_dataset(dataset, self.seed)
-
-
-class SampleSetBySize:
-    def __init__(self, n_samples: int, seed: int = 42):
-        self.n_samples = n_samples
-        self.seed = seed
-
-    def __call__(self, dataset: "HafniaDataset") -> "HafniaDataset":
-        return define_sample_set_by_size(dataset, self.n_samples, self.seed)
-
-
-class TransformImages:
-    def __init__(self, transform: Callable[[np.ndarray], np.ndarray], path_output: Path):
-        self.transform = transform
-        self.path_output = path_output
-
-    def __call__(self, dataset: "HafniaDataset") -> "HafniaDataset":
-        return transform_images(dataset, self.transform, self.path_output)