hafnia 0.2.4__tar.gz → 0.4.0__tar.gz

{hafnia-0.2.4 → hafnia-0.4.0}/.github/workflows/build.yaml

@@ -18,7 +18,7 @@ jobs:
       package-version: ${{ steps.extract-version.outputs.package_version }}
     steps:
       - uses: actions/checkout@v5.0.0
-      - uses: actions/setup-python@v5.6.0
+      - uses: actions/setup-python@v6.0.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
 

{hafnia-0.2.4 → hafnia-0.4.0}/.github/workflows/ci_cd.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5.0.0
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.32.0
+        uses: aquasecurity/trivy-action@0.33.1
         with:
           scan-type: 'fs'
           scan-ref: '.'
@@ -33,6 +33,7 @@ jobs:
   test:
     name: Run Tests
     needs: lint
+    secrets: inherit
     uses: ./.github/workflows/tests.yaml
     with:
         python-version-file: "pyproject.toml"

{hafnia-0.2.4 → hafnia-0.4.0}/.github/workflows/lint.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5.0.0
-      - uses: actions/setup-python@v5.6.0
+      - uses: actions/setup-python@v6.0.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
       - uses: pre-commit/action@v3.0.1

{hafnia-0.2.4 → hafnia-0.4.0}/.github/workflows/publish_docker.yaml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5.0.0
-      - uses: actions/setup-python@v5.6.0
+      - uses: actions/setup-python@v6.0.0
         id: python
         with:
           python-version-file: ${{ inputs.python-version-file }}
@@ -47,7 +47,7 @@ jobs:
             echo "aws_region=${{ secrets.STAGE_AWS_REGION }}" >> $GITHUB_OUTPUT
           fi
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4.3.1
+        uses: aws-actions/configure-aws-credentials@v5.1.0
         with:
           role-to-assume: arn:aws:iam::${{ steps.env-vars.outputs.aws_account_id }}:role/${{ secrets.AWS_ROLE_NAME }}
           aws-region: ${{ steps.env-vars.outputs.aws_region }} 
@@ -63,7 +63,7 @@ jobs:
         uses: docker/build-push-action@v6.18.0
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: mdi-runtime
+          ECR_REPOSITORY: platform_sdk_runtime
         with:
           context: .
           file: .github/workflows/Dockerfile
@@ -77,4 +77,4 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |
-            PYTHON_VERSION=${{ steps.python.outputs.python-version }}
+            PYTHON_VERSION=${{ steps.python.outputs.python-version }}

{hafnia-0.2.4 → hafnia-0.4.0}/.github/workflows/tests.yaml

@@ -9,10 +9,14 @@ on:
         type: string
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 1
+      matrix:
+        os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v5.0.0
-      - uses: actions/setup-python@v5.6.0
+      - uses: actions/setup-python@v6.0.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
       - name: Install uv
@@ -21,5 +25,13 @@ jobs:
           version: 0.6.8
       - name: Install the project
         run: uv sync --group dev
+      - name: Mount secrets config
+        shell: bash
+        env:
+          HAFNIA_CONFIG: ${{ secrets.HAFNIA_CONFIG }}
+        run: |
+          mkdir -p ~/.hafnia
+          echo "$HAFNIA_CONFIG" | jq . > ~/.hafnia/config.json
       - name: Run tests
-        run: uv run pytest tests 
+        run: uv run pytest tests 
+

{hafnia-0.2.4 → hafnia-0.4.0}/.gitignore

@@ -163,4 +163,4 @@ cython_debug/
 /pypiprivate/
 /packaging/
 /.data/
-/recipe.zip
+/trainer.zip

hafnia-0.4.0/.trivyignore

@@ -0,0 +1,3 @@
+# Ignore 'CVE-2024-37059' issue https://avd.aquasec.com/nvd/2024/cve-2024-37059/
+# The vulnerability does not apply to our platform as models are not loaded on our platform.
+CVE-2024-37059

{hafnia-0.2.4 → hafnia-0.4.0}/.vscode/extensions.json

@@ -5,7 +5,8 @@
         "ms-python.mypy-type-checker",
         "charliermarsh.ruff",
         "tamasfe.even-better-toml",
-        "streetsidesoftware.code-spell-checker"
+        "streetsidesoftware.code-spell-checker",
+        "ryanluker.vscode-coverage-gutters"
     ]
   }
   

{hafnia-0.2.4 → hafnia-0.4.0}/.vscode/launch.json

@@ -48,17 +48,45 @@
             ],
         },
         {
-            "name": "debug (hafnia dataset X)",
+            "name": "cmd: 'hafnia dataset [X]'",
             "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/src/cli/__main__.py",
             "args": [
                 "dataset",
+                //"ls",
                 "download",
                 "mnist",
-                //"./.data",
                 "--force"
             ]
+        },
+        {
+            "name": "cmd: 'hafnia experiment [X]'",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/cli/__main__.py",
+            "args": [
+                "experiment",
+                "create",
+                // "--trainer-path",
+                // "${workspaceFolder}/../trainer-classification",
+                //"--trainer-id",
+                //"e47d701d-c5ed-4014-9480-434f04e9459b",
+                "--trainer-path",
+                "${workspaceFolder}/../trainer-classification",
+                "--dataset",
+                "mnist",
+            ]
+        },    
+        {
+            "name": "cmd: 'hafnia train-recipe [X]'",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/cli/__main__.py",
+            "args": [
+                "trainer",
+                "ls"
+            ]
         }
     ]
 }

{hafnia-0.2.4 → hafnia-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.2.4
+Version: 0.4.0
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -9,6 +9,9 @@ Requires-Dist: boto3>=1.35.91
 Requires-Dist: click>=8.1.8
 Requires-Dist: emoji>=2.14.1
 Requires-Dist: flatten-dict>=0.4.2
+Requires-Dist: keyring>=25.6.0
+Requires-Dist: mcp==1.16.0
+Requires-Dist: mlflow>=3.4.0
 Requires-Dist: more-itertools>=10.7.0
 Requires-Dist: opencv-python-headless>=4.11.0.86
 Requires-Dist: pathspec>=0.12.1
@@ -19,20 +22,20 @@ Requires-Dist: pycocotools>=2.0.10
 Requires-Dist: pydantic>=2.10.4
 Requires-Dist: rich>=13.9.4
 Requires-Dist: s5cmd>=0.2.0
+Requires-Dist: sagemaker-mlflow>=0.1.0
 Requires-Dist: seedir>=0.5.0
-Requires-Dist: tqdm>=4.67.1
 Requires-Dist: xxhash>=3.5.0
 Description-Content-Type: text/markdown
 
 # Hafnia
 
-The `hafnia` python package is a collection of tools to create and run model training recipes on
+The `hafnia` python sdk and cli is a collection of tools to create and run model trainer packages on
 the [Hafnia Platform](https://hafnia.milestonesys.com/). 
 
 The package includes the following interfaces: 
 
 - `cli`: A Command Line Interface (CLI) to 1) configure/connect to Hafnia's [Training-aaS](https://hafnia.readme.io/docs/training-as-a-service) and 2) create and 
-launch recipe scripts.
+launch trainer packages.
 - `hafnia`: A python package including `HafniaDataset` to manage datasets and `HafniaLogger` to do 
 experiment tracking.
 
@@ -42,19 +45,19 @@ experiment tracking.
 and *hidden* datasets. Hidden datasets refers to datasets that can be used for 
 training, but are not available for download or direct access. 
 
-This is a key feature of the Hafnia platform, as a hidden dataset ensures data 
+This is a key for the Hafnia platform, as a hidden dataset ensures data 
 privacy, and allow models to be trained compliantly and ethically by third parties (you).
 
 The `script2model` approach is a Training-aaS concept, where you package your custom training 
-script as a *training recipe* and use the recipe to train models on the hidden datasets.
+project or script as a *trainer package* and use the package to train models on the hidden datasets.
 
-To support local development of a training recipe, we have introduced a **sample dataset** 
+To support local development of a trainer package, we have introduced a **sample dataset** 
 for each dataset available in the Hafnia [data library](https://hafnia.milestonesys.com/training-aas/datasets). The sample dataset is a small 
-and anonymized subset of the full dataset and available for download. 
+and an anonymized subset of the full dataset and available for download. 
 
 With the sample dataset, you can seamlessly switch between local development and Training-aaS. 
-Locally, you can create, validate and debug your training recipe. The recipe is then 
-launched with Training-aaS, where the recipe runs on the full dataset and can be scaled to run on
+Locally, you can create, validate and debug your trainer package. The trainer package is then 
+launched with Training-aaS, where the package runs on the full dataset and can be scaled to run on
 multiple GPUs and instances if needed. 
 
 ## Getting started: Configuration
@@ -78,6 +81,7 @@ Copy the key and save it for later use.
     Hafnia API Key:  # Pass your HAFNIA API key
     Hafnia Platform URL [https://api.mdi.milestonesys.com]:  # Press [Enter]
     ```
+
 1. Download `mnist` from terminal to verify that your configuration is working.  
 
     ```bash
@@ -89,7 +93,7 @@ With Hafnia configured on your local machine, it is now possible to download
 and explore the dataset sample with a python script:
 
 ```python
-from hafnia.data import load_dataset, get_dataset_path
+from hafnia.data import get_dataset_path
 from hafnia.dataset.hafnia_dataset import HafniaDataset
 
 # To download the sample dataset use:
@@ -122,19 +126,19 @@ midwest-vehicle-detection
 You can interact with data as you want, but we also provide `HafniaDataset`
 for loading/saving, managing and interacting with the dataset.
 
-We recommend to visit and potentially execute the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py) 
-to see how to use the `HafniaDataset` class and its methods.
+We recommend the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py) 
+for a short introduction on the `HafniaDataset`.
 
 Below is a short introduction to the `HafniaDataset` class.
 
 ```python
 from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
 
-# Load dataset
+# Load dataset from path
 dataset = HafniaDataset.read_from_path(path_dataset)
 
-# Alternatively, you can use the 'load_dataset' function to download and load dataset in one go.
-# dataset = load_dataset("midwest-vehicle-detection")
+# Or get dataset directly by name
+dataset = HafniaDataset.from_name("midwest-vehicle-detection")
 
 # Print dataset information
 dataset.print_stats()
@@ -199,6 +203,8 @@ DatasetInfo(
         'duration_average': 120.0,
         ...
     }
+    "format_version": "0.0.2",
+    "updated_at": "2025-09-24T21:50:20.231263"
 )
 ```
 
@@ -238,7 +244,7 @@ Sample(
     height=1080,
     width=1920,
     split='train',
-    is_sample=True,
+    tags=["sample"],
     collection_index=None,
     collection_id=None,
     remote_path='s3://mdi-production-midwest-vehicle-detection/sample/data/343403325f27e390.png',
@@ -302,10 +308,10 @@ Sample(
 )
 ```
 
-To learn more, view and potentially execute the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py). 
+To learn more, we recommend the `HafniaDataset` example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py). 
 
 ### Dataset Locally vs. Training-aaS
-An important feature of `load_dataset` is that it will return the full dataset 
+An important feature of `HafniaDataset.from_name` is that it will return the full dataset 
 when loaded with Training-aaS on the Hafnia platform. 
 
 This enables seamlessly switching between running/validating a training script 
@@ -316,7 +322,7 @@ Available datasets with corresponding sample datasets can be found in [data libr
 
 
 ## Getting started: Experiment Tracking with HafniaLogger
-The `HafniaLogger` is an important part of the recipe script and enables you to track, log and
+The `HafniaLogger` is an important part of the trainer and enables you to track, log and
 reproduce your experiments.
 
 When integrated into your training script, the `HafniaLogger` is responsible for collecting:
@@ -356,7 +362,7 @@ logger.log_scalar("validation/loss", value=0.1, step=100)
 logger.log_metric("validation/accuracy", value=0.95, step=100)
 ```
 
-Similar to `load_dataset`, the tracker behaves differently when running locally or in the cloud. 
+The tracker behaves differently when running locally or in the cloud. 
 Locally, experiment data is stored in a local folder `.data/experiments/{DATE_TIME}`. 
 
 In the cloud, the experiment data will be available in the Hafnia platform under 
@@ -380,7 +386,7 @@ and datasets available in the data library.
 
 ```python
 # Load Hugging Face dataset
-dataset_splits = load_dataset("midwest-vehicle-detection")
+dataset_splits = HafniaDataset.from_name("midwest-vehicle-detection")
 
 # Define transforms
 train_transforms = v2.Compose(
@@ -422,25 +428,25 @@ train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=
 
 
 ## Example: Training-aaS
-By combining logging and dataset loading, we can now construct our model training recipe. 
+By combining logging and dataset loading, we can now construct our model trainer package. 
 
-To demonstrate this, we have provided a recipe project that serves as a template for creating and structuring training recipes
-[recipe-classification](https://github.com/milestone-hafnia/recipe-classification)
+To demonstrate this, we have provided a trainer package project that serves as a template for creating and structuring trainers. The example repo is called
+[trainer-classification](https://github.com/milestone-hafnia/trainer-classification)
 
-The project also contains additional information on how to structure your training recipe, use the `HafniaLogger`, the `load_dataset` function and different approach for launching 
-the training recipe on the Hafnia platform.
+The project also contains additional information on how to structure your trainer package, use the `HafniaLogger`, loading a dataset and different approach for launching 
+the trainer on the Hafnia platform.
 
 
-## Create, Build and Run `recipe.zip` locally
-In order to test recipe compatibility with Hafnia cloud use the following command to build and 
+## Create, Build and Run `trainer.zip` locally
+In order to test trainer package compatibility with Hafnia cloud use the following command to build and 
 start the job locally.
 
 ```bash
-    # Create 'recipe.zip' from source folder '.'
-    hafnia recipe create .
-    
-    # Build the docker image locally from a 'recipe.zip' file
-    hafnia runc build-local recipe.zip
+    # Create 'trainer.zip' in the root folder of your training trainer project '../trainer/classification'
+    hafnia trainer create-zip ../trainer-classification
+
+    # Build the docker image locally from a 'trainer.zip' file
+    hafnia runc build-local trainer.zip
 
     # Execute the docker image locally with a desired dataset
     hafnia runc launch-local --dataset mnist  "python scripts/train.py"

{hafnia-0.2.4 → hafnia-0.4.0}/README.md

@@ -1,12 +1,12 @@
 # Hafnia
 
-The `hafnia` python package is a collection of tools to create and run model training recipes on
+The `hafnia` python sdk and cli is a collection of tools to create and run model trainer packages on
 the [Hafnia Platform](https://hafnia.milestonesys.com/). 
 
 The package includes the following interfaces: 
 
 - `cli`: A Command Line Interface (CLI) to 1) configure/connect to Hafnia's [Training-aaS](https://hafnia.readme.io/docs/training-as-a-service) and 2) create and 
-launch recipe scripts.
+launch trainer packages.
 - `hafnia`: A python package including `HafniaDataset` to manage datasets and `HafniaLogger` to do 
 experiment tracking.
 
@@ -16,19 +16,19 @@ experiment tracking.
 and *hidden* datasets. Hidden datasets refers to datasets that can be used for 
 training, but are not available for download or direct access. 
 
-This is a key feature of the Hafnia platform, as a hidden dataset ensures data 
+This is a key for the Hafnia platform, as a hidden dataset ensures data 
 privacy, and allow models to be trained compliantly and ethically by third parties (you).
 
 The `script2model` approach is a Training-aaS concept, where you package your custom training 
-script as a *training recipe* and use the recipe to train models on the hidden datasets.
+project or script as a *trainer package* and use the package to train models on the hidden datasets.
 
-To support local development of a training recipe, we have introduced a **sample dataset** 
+To support local development of a trainer package, we have introduced a **sample dataset** 
 for each dataset available in the Hafnia [data library](https://hafnia.milestonesys.com/training-aas/datasets). The sample dataset is a small 
-and anonymized subset of the full dataset and available for download. 
+and an anonymized subset of the full dataset and available for download. 
 
 With the sample dataset, you can seamlessly switch between local development and Training-aaS. 
-Locally, you can create, validate and debug your training recipe. The recipe is then 
-launched with Training-aaS, where the recipe runs on the full dataset and can be scaled to run on
+Locally, you can create, validate and debug your trainer package. The trainer package is then 
+launched with Training-aaS, where the package runs on the full dataset and can be scaled to run on
 multiple GPUs and instances if needed. 
 
 ## Getting started: Configuration
@@ -52,6 +52,7 @@ Copy the key and save it for later use.
     Hafnia API Key:  # Pass your HAFNIA API key
     Hafnia Platform URL [https://api.mdi.milestonesys.com]:  # Press [Enter]
     ```
+
 1. Download `mnist` from terminal to verify that your configuration is working.  
 
     ```bash
@@ -63,7 +64,7 @@ With Hafnia configured on your local machine, it is now possible to download
 and explore the dataset sample with a python script:
 
 ```python
-from hafnia.data import load_dataset, get_dataset_path
+from hafnia.data import get_dataset_path
 from hafnia.dataset.hafnia_dataset import HafniaDataset
 
 # To download the sample dataset use:
@@ -96,19 +97,19 @@ midwest-vehicle-detection
 You can interact with data as you want, but we also provide `HafniaDataset`
 for loading/saving, managing and interacting with the dataset.
 
-We recommend to visit and potentially execute the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py) 
-to see how to use the `HafniaDataset` class and its methods.
+We recommend the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py) 
+for a short introduction on the `HafniaDataset`.
 
 Below is a short introduction to the `HafniaDataset` class.
 
 ```python
 from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
 
-# Load dataset
+# Load dataset from path
 dataset = HafniaDataset.read_from_path(path_dataset)
 
-# Alternatively, you can use the 'load_dataset' function to download and load dataset in one go.
-# dataset = load_dataset("midwest-vehicle-detection")
+# Or get dataset directly by name
+dataset = HafniaDataset.from_name("midwest-vehicle-detection")
 
 # Print dataset information
 dataset.print_stats()
@@ -173,6 +174,8 @@ DatasetInfo(
         'duration_average': 120.0,
         ...
     }
+    "format_version": "0.0.2",
+    "updated_at": "2025-09-24T21:50:20.231263"
 )
 ```
 
@@ -212,7 +215,7 @@ Sample(
     height=1080,
     width=1920,
     split='train',
-    is_sample=True,
+    tags=["sample"],
     collection_index=None,
     collection_id=None,
     remote_path='s3://mdi-production-midwest-vehicle-detection/sample/data/343403325f27e390.png',
@@ -276,10 +279,10 @@ Sample(
 )
 ```
 
-To learn more, view and potentially execute the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py). 
+To learn more, we recommend the `HafniaDataset` example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py). 
 
 ### Dataset Locally vs. Training-aaS
-An important feature of `load_dataset` is that it will return the full dataset 
+An important feature of `HafniaDataset.from_name` is that it will return the full dataset 
 when loaded with Training-aaS on the Hafnia platform. 
 
 This enables seamlessly switching between running/validating a training script 
@@ -290,7 +293,7 @@ Available datasets with corresponding sample datasets can be found in [data libr
 
 
 ## Getting started: Experiment Tracking with HafniaLogger
-The `HafniaLogger` is an important part of the recipe script and enables you to track, log and
+The `HafniaLogger` is an important part of the trainer and enables you to track, log and
 reproduce your experiments.
 
 When integrated into your training script, the `HafniaLogger` is responsible for collecting:
@@ -330,7 +333,7 @@ logger.log_scalar("validation/loss", value=0.1, step=100)
 logger.log_metric("validation/accuracy", value=0.95, step=100)
 ```
 
-Similar to `load_dataset`, the tracker behaves differently when running locally or in the cloud. 
+The tracker behaves differently when running locally or in the cloud. 
 Locally, experiment data is stored in a local folder `.data/experiments/{DATE_TIME}`. 
 
 In the cloud, the experiment data will be available in the Hafnia platform under 
@@ -354,7 +357,7 @@ and datasets available in the data library.
 
 ```python
 # Load Hugging Face dataset
-dataset_splits = load_dataset("midwest-vehicle-detection")
+dataset_splits = HafniaDataset.from_name("midwest-vehicle-detection")
 
 # Define transforms
 train_transforms = v2.Compose(
@@ -396,25 +399,25 @@ train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=
 
 
 ## Example: Training-aaS
-By combining logging and dataset loading, we can now construct our model training recipe. 
+By combining logging and dataset loading, we can now construct our model trainer package. 
 
-To demonstrate this, we have provided a recipe project that serves as a template for creating and structuring training recipes
-[recipe-classification](https://github.com/milestone-hafnia/recipe-classification)
+To demonstrate this, we have provided a trainer package project that serves as a template for creating and structuring trainers. The example repo is called
+[trainer-classification](https://github.com/milestone-hafnia/trainer-classification)
 
-The project also contains additional information on how to structure your training recipe, use the `HafniaLogger`, the `load_dataset` function and different approach for launching 
-the training recipe on the Hafnia platform.
+The project also contains additional information on how to structure your trainer package, use the `HafniaLogger`, loading a dataset and different approach for launching 
+the trainer on the Hafnia platform.
 
 
-## Create, Build and Run `recipe.zip` locally
-In order to test recipe compatibility with Hafnia cloud use the following command to build and 
+## Create, Build and Run `trainer.zip` locally
+In order to test trainer package compatibility with Hafnia cloud use the following command to build and 
 start the job locally.
 
 ```bash
-    # Create 'recipe.zip' from source folder '.'
-    hafnia recipe create .
-    
-    # Build the docker image locally from a 'recipe.zip' file
-    hafnia runc build-local recipe.zip
+    # Create 'trainer.zip' in the root folder of your training trainer project '../trainer/classification'
+    hafnia trainer create-zip ../trainer-classification
+
+    # Build the docker image locally from a 'trainer.zip' file
+    hafnia runc build-local trainer.zip
 
     # Execute the docker image locally with a desired dataset
     hafnia runc launch-local --dataset mnist  "python scripts/train.py"