hafnia 0.4.0__tar.gz → 0.4.2__tar.gz

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/Dockerfile

@@ -1,5 +1,8 @@
 ARG PYTHON_VERSION
 FROM python:${PYTHON_VERSION}-slim
+
+RUN apt-get update && apt-get install -y pigz && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /opt/ml/processing
 
 COPY dist/*.whl .
@@ -8,4 +11,4 @@ RUN pip install --no-cache-dir *.whl && \
 
 RUN mkdir -p /opt/ml/processing/input \
              /opt/ml/processing/output \
-             /opt/ml/processing/tmp
+             /opt/ml/processing/tmp

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/build.yaml

@@ -6,6 +6,10 @@ on:
       python-version-file:
         required: true
         type: string
+      pyproject-toml-file:
+        required: false
+        type: string
+        default: "pyproject.toml"
     outputs:
       package-version:
         description: "The extracted package version"
@@ -25,11 +29,11 @@ jobs:
       - name: Extract package version
         id: extract-version
         run: |
-          VERSION=$(grep -m 1 'version = '  ${{ inputs.python-version-file }} | sed -e 's/version = "\(.*\)"/\1/')
+          VERSION=$(grep -m 1 'version = '  ${{ inputs.pyproject-toml-file }} | sed -e 's/version = "\(.*\)"/\1/')
           echo "package_version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           version: 0.6.8
 
@@ -45,7 +49,7 @@ jobs:
         run: uv build
 
       - name: Upload package artifact
-        uses: actions/upload-artifact@v4.6.2
+        uses: actions/upload-artifact@v5.0.0
         with:
           name: python-package
           path: dist/

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/check_release.yaml

@@ -20,7 +20,7 @@ jobs:
       make_release: ${{ steps.check_release.outputs.make_release }}
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v5.0.0
+        uses: actions/download-artifact@v6.0.0
         with:
             name: python-package
             path: dist/

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/ci_cd.yaml

@@ -5,14 +5,14 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
-    paths: ["src/**", "tests/**", "pyproject.toml"]
+    paths: ["src/**", "tests/**", "pyproject.toml", ".python-version", "uv.lock", ".github/workflows/**"]
 
 jobs:
   lint:
     name: Lint Code
     uses: ./.github/workflows/lint.yaml
     with:
-        python-version-file: "pyproject.toml"
+        python-version-file: ".python-version"
 
   security-scan:
     name: Security Scan
@@ -36,14 +36,15 @@ jobs:
     secrets: inherit
     uses: ./.github/workflows/tests.yaml
     with:
-        python-version-file: "pyproject.toml"
+        python-version-file: ".python-version"
 
   build:
     name: Build Package
     needs: [test, security-scan]
     uses: ./.github/workflows/build.yaml
     with:
-        python-version-file: "pyproject.toml"
+        python-version-file: ".python-version"
+        pyproject-toml-file: "pyproject.toml"
 
   publish-docker-staging:
     name: Publish Docker Image to Staging
@@ -52,7 +53,7 @@ jobs:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     uses: ./.github/workflows/publish_docker.yaml
     with:
-      python-version-file: "pyproject.toml"
+      python-version-file: ".python-version"
       package-version: ${{ needs.build.outputs.package-version }}
       environment: "staging"
 
@@ -80,7 +81,7 @@ jobs:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     uses: ./.github/workflows/publish_docker.yaml
     with:
-      python-version-file: "pyproject.toml"
+      python-version-file: ".python-version"
       package-version: ${{ needs.build.outputs.package-version }}
       environment: "production"
 

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/publish_docker.yaml

@@ -31,7 +31,7 @@ jobs:
           python-version-file: ${{ inputs.python-version-file }}
 
       - name: Download package artifact
-        uses: actions/download-artifact@v5.0.0
+        uses: actions/download-artifact@v6.0.0
         with:
           name: python-package
           path: dist/

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/publish_pypi.yaml

@@ -17,7 +17,7 @@ jobs:
       contents: read
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v5.0.0
+        uses: actions/download-artifact@v6.0.0
         with:
           name: python-package
           path: dist/

{hafnia-0.4.0 → hafnia-0.4.2}/.github/workflows/tests.yaml

@@ -20,7 +20,7 @@ jobs:
         with:
           python-version-file: ${{ inputs.python-version-file }}
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           version: 0.6.8
       - name: Install the project

{hafnia-0.4.0 → hafnia-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.4.0
+Version: 0.4.2
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -158,7 +158,7 @@ and `dataset.samples` with annotations as a polars DataFrame
 print(dataset.samples.head(2))
 shape: (2, 14)
 ┌──────────────┬─────────────────────────────────┬────────┬───────┬───┬─────────────────────────────────┬──────────┬──────────┬─────────────────────────────────┐
-│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ objects                         ┆ bitmasks ┆ polygons ┆ meta                            │
+│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ bboxes                          ┆ bitmasks ┆ polygons ┆ meta                            │
 │ ---          ┆ ---                             ┆ ---    ┆ ---   ┆   ┆ ---                             ┆ ---      ┆ ---      ┆ ---                             │
 │ u32          ┆ str                             ┆ i64    ┆ i64   ┆   ┆ list[struct[11]]                ┆ null     ┆ null     ┆ struct[5]                       │
 ╞══════════════╪═════════════════════════════════╪════════╪═══════╪═══╪═════════════════════════════════╪══════════╪══════════╪═════════════════════════════════╡
@@ -218,7 +218,7 @@ sample_dict = dataset[0]
 
 for sample_dict in dataset:
     sample = Sample(**sample_dict)
-    print(sample.sample_id, sample.objects)
+    print(sample.sample_id, sample.bboxes)
     break
 ```
 Not that it is possible to create a `Sample` object from the sample dictionary.
@@ -421,7 +421,7 @@ pil_image.save("visualized_labels.png")
 
 # Create DataLoaders - using TorchVisionCollateFn
 collate_fn = torch_helpers.TorchVisionCollateFn(
-    skip_stacking=["objects.bbox", "objects.class_idx"]
+    skip_stacking=["bboxes.bbox", "bboxes.class_idx"]
 )
 train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
 ```

{hafnia-0.4.0 → hafnia-0.4.2}/README.md

@@ -129,7 +129,7 @@ and `dataset.samples` with annotations as a polars DataFrame
 print(dataset.samples.head(2))
 shape: (2, 14)
 ┌──────────────┬─────────────────────────────────┬────────┬───────┬───┬─────────────────────────────────┬──────────┬──────────┬─────────────────────────────────┐
-│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ objects                         ┆ bitmasks ┆ polygons ┆ meta                            │
+│ sample_index ┆ file_name                       ┆ height ┆ width ┆ … ┆ bboxes                          ┆ bitmasks ┆ polygons ┆ meta                            │
 │ ---          ┆ ---                             ┆ ---    ┆ ---   ┆   ┆ ---                             ┆ ---      ┆ ---      ┆ ---                             │
 │ u32          ┆ str                             ┆ i64    ┆ i64   ┆   ┆ list[struct[11]]                ┆ null     ┆ null     ┆ struct[5]                       │
 ╞══════════════╪═════════════════════════════════╪════════╪═══════╪═══╪═════════════════════════════════╪══════════╪══════════╪═════════════════════════════════╡
@@ -189,7 +189,7 @@ sample_dict = dataset[0]
 
 for sample_dict in dataset:
     sample = Sample(**sample_dict)
-    print(sample.sample_id, sample.objects)
+    print(sample.sample_id, sample.bboxes)
     break
 ```
 Not that it is possible to create a `Sample` object from the sample dictionary.
@@ -392,7 +392,7 @@ pil_image.save("visualized_labels.png")
 
 # Create DataLoaders - using TorchVisionCollateFn
 collate_fn = torch_helpers.TorchVisionCollateFn(
-    skip_stacking=["objects.bbox", "objects.class_idx"]
+    skip_stacking=["bboxes.bbox", "bboxes.class_idx"]
 )
 train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
 ```

{hafnia-0.4.0 → hafnia-0.4.2}/examples/example_dataset_recipe.py

@@ -129,26 +129,28 @@ mapping_midwest = {
     "Vehicle*": "Vehicle",  # Wildcard mapping. Selects class names starting with 'Vehicle.' e.g. 'Vehicle.Bicycle', "Vehicle.Car', etc.
     "Vehicle.Trailer": OPS_REMOVE_CLASS,  # Use this to remove a class
 }
-coco_remapped = coco.class_mapper(class_mapping=mappings_coco, method="remove_undefined", task_name="bboxes")
-midwest_remapped = midwest.class_mapper(class_mapping=mapping_midwest, task_name="bboxes")
+coco_remapped = coco.class_mapper(class_mapping=mappings_coco, method="remove_undefined", task_name="object_detection")
+midwest_remapped = midwest.class_mapper(class_mapping=mapping_midwest, task_name="object_detection")
 
 # 2b) Merge datasets
 merged_dataset_all_images = HafniaDataset.from_merge(dataset0=coco_remapped, dataset1=midwest_remapped)
 
 # 2c) Remove images without 'Person' or 'Vehicle' annotations
-merged_dataset = merged_dataset_all_images.select_samples_by_class_name(name=["Person", "Vehicle"], task_name="bboxes")
+merged_dataset = merged_dataset_all_images.select_samples_by_class_name(
+    name=["Person", "Vehicle"], task_name="object_detection"
+)
 merged_dataset.print_stats()
 
 # 3) Once you have verified operations using the 'HafniaDataset' interface, you can convert
 # the operations to a single 'DatasetRecipe'
 merged_recipe = DatasetRecipe.from_merge(
     recipe0=DatasetRecipe.from_name("coco-2017").class_mapper(
-        class_mapping=mappings_coco, method="remove_undefined", task_name="bboxes"
+        class_mapping=mappings_coco, method="remove_undefined", task_name="object_detection"
     ),
     recipe1=DatasetRecipe.from_name("midwest-vehicle-detection").class_mapper(
-        class_mapping=mapping_midwest, task_name="bboxes"
+        class_mapping=mapping_midwest, task_name="object_detection"
     ),
-).select_samples_by_class_name(name=["Person", "Vehicle"], task_name="bboxes")
+).select_samples_by_class_name(name=["Person", "Vehicle"], task_name="object_detection")
 
 # 3a) Verify again on the sample datasets, that the recipe works and can build as a dataset
 merged_dataset = merged_recipe.build()

{hafnia-0.4.0 → hafnia-0.4.2}/examples/example_hafnia_dataset.py

@@ -33,8 +33,8 @@ dataset.print_class_distribution()
 dataset.print_stats()  # Print verbose dataset statistics
 
 # Get dataset stats
-dataset.class_counts_all()  # Get class counts for all tasks
-dataset.class_counts_for_task(primitive=Classification)  # Get class counts for a specific task
+dataset.calculate_class_counts()  # Get class counts for all tasks
+dataset.calculate_task_class_counts(primitive=Classification)  # Get class counts for a specific task
 
 # Create a dataset split for training
 dataset_train = dataset.create_split_dataset("train")
@@ -86,9 +86,19 @@ dataset.write(path_dataset)
 # Load dataset from disk
 dataset_again = HafniaDataset.from_path(path_dataset)
 
+## Dataset importers and exporters ##
+dataset_coco = HafniaDataset.from_name("coco-2017").select_samples(n_samples=5, seed=42)
+path_yolo_format = Path(".data/tmp/yolo_dataset")
 
+# Export dataset to YOLO format
+dataset_coco.to_yolo_format(path_export_yolo_dataset=path_yolo_format)
+
+# Import dataset from YOLO format
+dataset_coco_imported = HafniaDataset.from_yolo_format(path_yolo_format)
+
+## Custom dataset operations and statistics ##
 # Want custom dataset transformations or statistics? Use the polars table (dataset.samples) directly
-n_objects = dataset.samples["objects"].list.len().sum()
+n_objects = dataset.samples["bboxes"].list.len().sum()
 n_objects = dataset.samples[Bbox.column_name()].list.len().sum()  # Use Bbox.column_name() to avoid magic variables
 n_classifications = dataset.samples[Classification.column_name()].list.len().sum()
 
@@ -106,7 +116,7 @@ for sample_dict in dataset_train:
 # Unpack dict into a Sample-object! Important for data validation, useability, IDE completion and mypy hints
 sample: Sample = Sample(**sample_dict)
 
-objects: List[Bbox] = sample.objects  # Use 'sample.objects' access bounding boxes as a list of Bbox objects
+bboxes: List[Bbox] = sample.bboxes  # Use 'sample.bboxes' access bounding boxes as a list of Bbox objects
 bitmasks: List[Bitmask] = sample.bitmasks  # Use 'sample.bitmasks' to access bitmasks as a list of Bitmask objects
 polygons: List[Polygon] = sample.polygons  # Use 'sample.polygons' to access polygons as a list of Polygon objects
 classifications: List[Classification] = sample.classifications  # As a list of Classification objects
@@ -134,7 +144,7 @@ for i_fake_sample in range(5):
         width=640,
         split="train",
         tags=["sample"],
-        objects=bboxes,
+        bboxes=bboxes,
         classifications=classifications,
     )
     fake_samples.append(sample)

{hafnia-0.4.0 → hafnia-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hafnia"
-version = "0.4.0"
+version = "0.4.2"
 description = "Python SDK for communication with Hafnia platform."
 readme = "README.md"
 authors = [
@@ -44,14 +44,14 @@ dev = [
 test = ["pytest>=8.3.4", "pre-commit>=4.2.0", "ruff>=0.9.1"]
 
 [project.scripts]
-hafnia = 'cli.__main__:main'
+hafnia = 'hafnia_cli.__main__:main'
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/cli", "src/hafnia"]
+packages = ["src/hafnia_cli", "src/hafnia"]
 
 [tool.uv]
 default-groups = ["test"]

{hafnia-0.4.0 → hafnia-0.4.2}/src/hafnia/__init__.py

@@ -3,4 +3,4 @@ from importlib.metadata import version
 __package_name__ = "hafnia"
 __version__ = version(__package_name__)
 
-__dataset_format_version__ = "0.1.0"  # Hafnia dataset format version
+__dataset_format_version__ = "0.2.0"  # Hafnia dataset format version

hafnia-0.4.2/src/hafnia/dataset/dataset_names.py

@@ -0,0 +1,190 @@
+from enum import Enum
+from typing import Dict, List, Optional
+
+import boto3
+from pydantic import BaseModel, field_validator
+
+FILENAME_RECIPE_JSON = "recipe.json"
+FILENAME_DATASET_INFO = "dataset_info.json"
+FILENAME_ANNOTATIONS_JSONL = "annotations.jsonl"
+FILENAME_ANNOTATIONS_PARQUET = "annotations.parquet"
+
+DATASET_FILENAMES_REQUIRED = [
+    FILENAME_DATASET_INFO,
+    FILENAME_ANNOTATIONS_JSONL,
+    FILENAME_ANNOTATIONS_PARQUET,
+]
+
+
+class DeploymentStage(Enum):
+    STAGING = "staging"
+    PRODUCTION = "production"
+
+
+TAG_IS_SAMPLE = "sample"
+
+OPS_REMOVE_CLASS = "__REMOVE__"
+
+
+class PrimitiveField:
+    CLASS_NAME: str = "class_name"  # Name of the class this primitive is associated with, e.g. "car" for Bbox
+    CLASS_IDX: str = "class_idx"  # Index of the class this primitive is associated with, e.g. 0 for "car" if it is the first class  # noqa: E501
+    OBJECT_ID: str = "object_id"  # Unique identifier for the object, e.g. "12345123"
+    CONFIDENCE: str = "confidence"  # Confidence score (0-1.0) for the primitive, e.g. 0.95 for Bbox
+
+    META: str = "meta"  # Contains metadata about each primitive, e.g. attributes color, occluded, iscrowd, etc.
+    TASK_NAME: str = "task_name"  # Name of the task this primitive is associated with, e.g. "bboxes" for Bbox
+
+    @staticmethod
+    def fields() -> List[str]:
+        """
+        Returns a list of expected field names for primitives.
+        """
+        return [
+            PrimitiveField.CLASS_NAME,
+            PrimitiveField.CLASS_IDX,
+            PrimitiveField.OBJECT_ID,
+            PrimitiveField.CONFIDENCE,
+            PrimitiveField.META,
+            PrimitiveField.TASK_NAME,
+        ]
+
+
+class SampleField:
+    FILE_PATH: str = "file_path"
+    HEIGHT: str = "height"
+    WIDTH: str = "width"
+    SPLIT: str = "split"
+    TAGS: str = "tags"
+
+    CLASSIFICATIONS: str = "classifications"
+    BBOXES: str = "bboxes"
+    BITMASKS: str = "bitmasks"
+    POLYGONS: str = "polygons"
+
+    STORAGE_FORMAT: str = "storage_format"  # E.g. "image", "video", "zip"
+    COLLECTION_INDEX: str = "collection_index"
+    COLLECTION_ID: str = "collection_id"
+    REMOTE_PATH: str = "remote_path"  # Path to the file in remote storage, e.g. S3
+    SAMPLE_INDEX: str = "sample_index"
+
+    ATTRIBUTION: str = "attribution"  # Attribution for the sample (image/video), e.g. creator, license, source, etc.
+    META: str = "meta"
+    DATASET_NAME: str = "dataset_name"
+
+
+class StorageFormat:
+    IMAGE: str = "image"
+    VIDEO: str = "video"
+    ZIP: str = "zip"
+
+
+class SplitName:
+    TRAIN: str = "train"
+    VAL: str = "validation"
+    TEST: str = "test"
+    UNDEFINED: str = "UNDEFINED"
+
+    @staticmethod
+    def valid_splits() -> List[str]:
+        return [SplitName.TRAIN, SplitName.VAL, SplitName.TEST]
+
+    @staticmethod
+    def all_split_names() -> List[str]:
+        return [*SplitName.valid_splits(), SplitName.UNDEFINED]
+
+
+class DatasetVariant(Enum):
+    DUMP = "dump"
+    SAMPLE = "sample"
+    HIDDEN = "hidden"
+
+
+class AwsCredentials(BaseModel):
+    access_key: str
+    secret_key: str
+    session_token: str
+    region: Optional[str]
+
+    def aws_credentials(self) -> Dict[str, str]:
+        """
+        Returns the AWS credentials as a dictionary.
+        """
+        environment_vars = {
+            "AWS_ACCESS_KEY_ID": self.access_key,
+            "AWS_SECRET_ACCESS_KEY": self.secret_key,
+            "AWS_SESSION_TOKEN": self.session_token,
+        }
+        if self.region:
+            environment_vars["AWS_REGION"] = self.region
+
+        return environment_vars
+
+    @staticmethod
+    def from_session(session: boto3.Session) -> "AwsCredentials":
+        """
+        Creates AwsCredentials from a Boto3 session.
+        """
+        frozen_credentials = session.get_credentials().get_frozen_credentials()
+        return AwsCredentials(
+            access_key=frozen_credentials.access_key,
+            secret_key=frozen_credentials.secret_key,
+            session_token=frozen_credentials.token,
+            region=session.region_name,
+        )
+
+
+ARN_PREFIX = "arn:aws:s3:::"
+
+
+class ResourceCredentials(AwsCredentials):
+    s3_arn: str
+
+    @staticmethod
+    def fix_naming(payload: Dict[str, str]) -> "ResourceCredentials":
+        """
+        The endpoint returns a payload with a key called 's3_path', but it
+        is actually an ARN path (starts with arn:aws:s3::). This method renames it to 's3_arn' for consistency.
+        """
+        if "s3_path" in payload and payload["s3_path"].startswith(ARN_PREFIX):
+            payload["s3_arn"] = payload.pop("s3_path")
+
+        if "region" not in payload:
+            payload["region"] = "eu-west-1"
+        return ResourceCredentials(**payload)
+
+    @field_validator("s3_arn")
+    @classmethod
+    def validate_s3_arn(cls, value: str) -> str:
+        """Validate s3_arn to ensure it starts with 'arn:aws:s3:::'"""
+        if not value.startswith("arn:aws:s3:::"):
+            raise ValueError(f"Invalid S3 ARN: {value}. It should start with 'arn:aws:s3:::'")
+        return value
+
+    def s3_path(self) -> str:
+        """
+        Extracts the S3 path from the ARN.
+        Example: arn:aws:s3:::my-bucket/my-prefix -> my-bucket/my-prefix
+        """
+        return self.s3_arn[len(ARN_PREFIX) :]
+
+    def s3_uri(self) -> str:
+        """
+        Converts the S3 ARN to a URI format.
+        Example: arn:aws:s3:::my-bucket/my-prefix -> s3://my-bucket/my-prefix
+        """
+        return f"s3://{self.s3_path()}"
+
+    def bucket_name(self) -> str:
+        """
+        Extracts the bucket name from the S3 ARN.
+        Example: arn:aws:s3:::my-bucket/my-prefix -> my-bucket
+        """
+        return self.s3_path().split("/")[0]
+
+    def object_key(self) -> str:
+        """
+        Extracts the object key from the S3 ARN.
+        Example: arn:aws:s3:::my-bucket/my-prefix -> my-prefix
+        """
+        return "/".join(self.s3_path().split("/")[1:])

{hafnia-0.4.0 → hafnia-0.4.2}/src/hafnia/dataset/dataset_recipe/dataset_recipe.py

@@ -98,8 +98,8 @@ class DatasetRecipe(Serializable):
     @staticmethod
     def from_recipe_id(recipe_id: str) -> "DatasetRecipe":
         """Loads a dataset recipe by id from the hafnia platform."""
-        from cli.config import Config
         from hafnia.platform.dataset_recipe import get_dataset_recipe_by_id
+        from hafnia_cli.config import Config
 
         cfg = Config()
         endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
@@ -114,8 +114,8 @@ class DatasetRecipe(Serializable):
     @staticmethod
     def from_recipe_name(name: str) -> "DatasetRecipe":
         """Loads a dataset recipe by name from the hafnia platform"""
-        from cli.config import Config
         from hafnia.platform.dataset_recipe import get_dataset_recipe_by_name
+        from hafnia_cli.config import Config
 
         cfg = Config()
         endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
@@ -239,8 +239,8 @@ class DatasetRecipe(Serializable):
 
     def as_platform_recipe(self, recipe_name: Optional[str], overwrite: bool = False) -> Dict:
         """Uploads dataset recipe to the hafnia platform."""
-        from cli.config import Config
         from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe
+        from hafnia_cli.config import Config
 
         recipe = self.as_dict()
         cfg = Config()