hafnia 0.1.23__tar.gz → 0.1.25__tar.gz

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/build.yaml

@@ -18,7 +18,7 @@ jobs:
       package-version: ${{ steps.extract-version.outputs.package_version }}
     steps:
       - uses: actions/checkout@v4.2.2
-      - uses: actions/setup-python@v5.4.0
+      - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
 
@@ -29,7 +29,7 @@ jobs:
           echo "package_version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           version: 0.6.8
 

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/check_release.yaml

@@ -20,7 +20,7 @@ jobs:
       make_release: ${{ steps.check_release.outputs.make_release }}
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v4.2.1
+        uses: actions/download-artifact@v4.3.0
         with:
             name: python-package
             path: dist/

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/ci_cd.yaml

@@ -44,17 +44,9 @@ jobs:
     with:
         python-version-file: "pyproject.toml"
 
-  publish-pypi-test:
-    name: Publish Package to TestPyPI
-    needs: build
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    uses: ./.github/workflows/publish_pypi.yaml
-    with:
-      environment: testpypi
-        
   publish-docker-staging:
     name: Publish Docker Image to Staging
-    needs: [build, publish-pypi-test]
+    needs: build
     secrets: inherit
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     uses: ./.github/workflows/publish_docker.yaml

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/lint.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.2.2
-      - uses: actions/setup-python@v5.4.0
+      - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
       - uses: pre-commit/action@v3.0.1

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/publish_docker.yaml

@@ -25,13 +25,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.2.2
-      - uses: actions/setup-python@v5.4.0
+      - uses: actions/setup-python@v5.6.0
         id: python
         with:
           python-version-file: ${{ inputs.python-version-file }}
 
       - name: Download package artifact
-        uses: actions/download-artifact@v4.2.1
+        uses: actions/download-artifact@v4.3.0
         with:
           name: python-package
           path: dist/
@@ -60,7 +60,7 @@ jobs:
         uses: docker/setup-buildx-action@v3.10.0
 
       - name: Build and push
-        uses: docker/build-push-action@v6.15.0
+        uses: docker/build-push-action@v6.16.0
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: mdi-runtime

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/publish_pypi.yaml

@@ -17,7 +17,7 @@ jobs:
       contents: read
     steps:
       - name: Download package artifact
-        uses: actions/download-artifact@v4.2.1
+        uses: actions/download-artifact@v4.3.0
         with:
           name: python-package
           path: dist/

{hafnia-0.1.23 → hafnia-0.1.25}/.github/workflows/tests.yaml

@@ -12,11 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.2.2
-      - uses: actions/setup-python@v5.4.0
+      - uses: actions/setup-python@v5.6.0
         with:
           python-version-file: ${{ inputs.python-version-file }}
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           version: 0.6.8
       - name: Install the project

{hafnia-0.1.23 → hafnia-0.1.25}/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.4
+  rev: v0.11.8
   hooks:
     - id: ruff
       types_or: [python, pyi]

{hafnia-0.1.23 → hafnia-0.1.25}/.vscode/launch.json

@@ -21,6 +21,32 @@
                 "ls"
             ],
         },
+        {
+            "name": "cmd: hafnia runc launch-local",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/cli/__main__.py",
+            "args": [
+                "runc",
+                "launch-local",
+                "--dataset",
+                "midwest-vehicle-detection-tiny",
+                "train --config-name yolov4-hafnia.yaml"
+            ],
+        },
+        {
+            "name": "cmd: hafnia runc build-local",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/cli/__main__.py",
+            "args": [
+                "runc",
+                "build-local",
+                "train",
+                "--dataset",
+                "mnist",
+            ],
+        },
         {
             "name": "debug (hafnia data download mnist)",
             "type": "debugpy",

{hafnia-0.1.23 → hafnia-0.1.25}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 Data-insight-Platform
+Copyright (c) 2025 Milestone Systems A/S
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{hafnia-0.1.23 → hafnia-0.1.25}/PKG-INFO

@@ -1,18 +1,21 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.1.23
-Summary: Python tools for communication with Hafnia platform.
-Author-email: Ivan Sahumbaiev <ivsa@milestone.dk>
+Version: 0.1.25
+Summary: Python SDK for communication with Hafnia platform.
+Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: boto3>=1.35.91
 Requires-Dist: click>=8.1.8
 Requires-Dist: datasets>=3.2.0
+Requires-Dist: emoji>=2.14.1
 Requires-Dist: flatten-dict>=0.4.2
+Requires-Dist: pathspec>=0.12.1
 Requires-Dist: pillow>=11.1.0
 Requires-Dist: pyarrow>=18.1.0
 Requires-Dist: pydantic>=2.10.4
 Requires-Dist: rich>=13.9.4
+Requires-Dist: seedir>=0.5.0
 Requires-Dist: tqdm>=4.67.1
 Provides-Extra: torch
 Requires-Dist: flatten-dict>=0.4.2; extra == 'torch'
@@ -167,31 +170,49 @@ and datasets available in the data library.
 By combining logging and dataset loading, we can now construct our model training recipe. 
 
 To demonstrate this, we have provided a recipe project that serves as a template for creating and structuring training recipes
-[recipe-classification](https://github.com/Data-insight-Platform/recipe-classification)
+[recipe-classification](https://github.com/milestone-hafnia/recipe-classification)
 
 The project also contains additional information on how to structure your training recipe, use the `HafniaLogger`, the `load_dataset` function and different approach for launching 
 the training recipe on the Hafnia platform.
 
+
+## Create, Build and Run `recipe.zip` locally
+In order to test recipe compatibility with Hafnia cloud use the following command to build and 
+start the job locally.
+
+```bash
+    # Create 'recipe.zip' from source folder '.'
+    hafnia recipe create .
+    
+    # Build the docker image locally from a 'recipe.zip' file
+    hafnia runc build-local recipe.zip
+
+    # Execute the docker image locally with a desired dataset
+    hafnia runc launch-local --dataset mnist  "python scripts/train.py"
+```
+
 ## Detailed Documentation
 For more information, go to our [documentation page](https://hafnia.readme.io/docs/welcome-to-hafnia) 
 or in below markdown pages. 
 
 - [CLI](docs/cli.md) - Detailed guide for the Hafnia command-line interface
-- [Script2Model Documentation](docs/s2m.md) - Detailed guide for script2model
 - [Release lifecycle](docs/release.md) - Details about package release lifecycle.
 
 ## Development
 For development, we are using an uv based virtual python environment.
 
 Install uv
-
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-
+```bash 
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
 
 Install python dependencies including developer (`--dev`) and optional dependencies (`--all-extras`).
 
-    uv sync --all-extras --dev
+```bash
+uv sync --all-extras --dev
+```
 
  Run tests:
-
-    uv run pytest tests
+```bash
+uv run pytest tests
+```

{hafnia-0.1.23 → hafnia-0.1.25}/README.md

@@ -145,31 +145,49 @@ and datasets available in the data library.
 By combining logging and dataset loading, we can now construct our model training recipe. 
 
 To demonstrate this, we have provided a recipe project that serves as a template for creating and structuring training recipes
-[recipe-classification](https://github.com/Data-insight-Platform/recipe-classification)
+[recipe-classification](https://github.com/milestone-hafnia/recipe-classification)
 
 The project also contains additional information on how to structure your training recipe, use the `HafniaLogger`, the `load_dataset` function and different approach for launching 
 the training recipe on the Hafnia platform.
 
+
+## Create, Build and Run `recipe.zip` locally
+In order to test recipe compatibility with Hafnia cloud use the following command to build and 
+start the job locally.
+
+```bash
+    # Create 'recipe.zip' from source folder '.'
+    hafnia recipe create .
+    
+    # Build the docker image locally from a 'recipe.zip' file
+    hafnia runc build-local recipe.zip
+
+    # Execute the docker image locally with a desired dataset
+    hafnia runc launch-local --dataset mnist  "python scripts/train.py"
+```
+
 ## Detailed Documentation
 For more information, go to our [documentation page](https://hafnia.readme.io/docs/welcome-to-hafnia) 
 or in below markdown pages. 
 
 - [CLI](docs/cli.md) - Detailed guide for the Hafnia command-line interface
-- [Script2Model Documentation](docs/s2m.md) - Detailed guide for script2model
 - [Release lifecycle](docs/release.md) - Details about package release lifecycle.
 
 ## Development
 For development, we are using an uv based virtual python environment.
 
 Install uv
-
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-
+```bash 
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
 
 Install python dependencies including developer (`--dev`) and optional dependencies (`--all-extras`).
 
-    uv sync --all-extras --dev
+```bash
+uv sync --all-extras --dev
+```
 
  Run tests:
-
-    uv run pytest tests
+```bash
+uv run pytest tests
+```

{hafnia-0.1.23 → hafnia-0.1.25}/docs/cli.md

@@ -95,4 +95,5 @@ Available environment variables:
 - `MDI_CONFIG_PATH` - Custom path to the configuration file
 - `MDI_API_KEY_SECRET_NAME` - Name of the AWS Secrets Manager secret containing the API key
 - `AWS_REGION` - AWS region for ECR and Secrets Manager operations
-- `RECIPE_DIR` - Directory containing recipe code (used by the `runc launch` command)
+- `RECIPE_DIR` - Directory containing recipe code (used by the `runc launch` command
+- `HAFNIA_CLOUD` – Allow emulate cloud behaviour

{hafnia-0.1.23 → hafnia-0.1.25}/pyproject.toml

@@ -1,19 +1,22 @@
 [project]
 name = "hafnia"
-version = "0.1.23"
-description = "Python tools for communication with Hafnia platform."
+version = "0.1.25"
+description = "Python SDK for communication with Hafnia platform."
 readme = "README.md"
-authors = [{ name = "Ivan Sahumbaiev", email = "ivsa@milestone.dk" }]
+authors = [{ name = "Milestone Systems", email = "hafniaplatform@milestone.dk" }]
 requires-python = ">=3.10"
 dependencies = [
     "boto3>=1.35.91",
     "click>=8.1.8",
     "datasets>=3.2.0",
+    "emoji>=2.14.1",
     "flatten-dict>=0.4.2",
+    "pathspec>=0.12.1",
     "pillow>=11.1.0",
     "pyarrow>=18.1.0",
     "pydantic>=2.10.4",
     "rich>=13.9.4",
+    "seedir>=0.5.0",
     "tqdm>=4.67.1",
 ]
 

{hafnia-0.1.23 → hafnia-0.1.25}/src/cli/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import click
 
-from cli import consts, data_cmds, experiment_cmds, profile_cmds, runc_cmds
+from cli import consts, data_cmds, experiment_cmds, profile_cmds, recipe_cmds, runc_cmds
 from cli.config import Config, ConfigSchema
 
 
@@ -54,6 +54,7 @@ main.add_command(profile_cmds.profile)
 main.add_command(data_cmds.data)
 main.add_command(runc_cmds.runc)
 main.add_command(experiment_cmds.experiment)
+main.add_command(recipe_cmds.recipe)
 
 if __name__ == "__main__":
     main()

{hafnia-0.1.23 → hafnia-0.1.25}/src/cli/consts.py

@@ -8,6 +8,7 @@ ERROR_CREATE_PROFILE: str = "Failed to create profile. Profile name must be uniq
 ERROR_GET_RESOURCE: str = "Failed to get the data from platform. Verify url or api key."
 
 ERROR_EXPERIMENT_DIR: str = "Source directory does not exist"
+ERROR_RECIPE_FILE_FORMAT: str = "Recipe filename must be a '.zip' file"
 
 PROFILE_SWITCHED_SUCCESS: str = "Switched to profile:"
 PROFILE_REMOVED_SUCCESS: str = "Removed profile:"

{hafnia-0.1.23 → hafnia-0.1.25}/src/cli/data_cmds.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Optional
 
 import click
@@ -35,20 +36,18 @@ def data_get(cfg: Config, url: str, destination: click.Path) -> None:
 @click.argument("destination", default=None, required=False)
 @click.option("--force", is_flag=True, default=False, help="Force download")
 @click.pass_obj
-def data_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> None:
+def data_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> Path:
     """Download dataset from Hafnia platform"""
 
     from hafnia.data.factory import download_or_get_dataset_path
 
     try:
-        endpoint_dataset = cfg.get_platform_endpoint("datasets")
-        api_key = cfg.api_key
-        download_or_get_dataset_path(
+        path_dataset = download_or_get_dataset_path(
             dataset_name=dataset_name,
-            endpoint=endpoint_dataset,
-            api_key=api_key,
+            cfg=cfg,
             output_dir=destination,
             force_redownload=force,
         )
     except Exception:
         raise click.ClickException(consts.ERROR_GET_RESOURCE)
+    return path_dataset

{hafnia-0.1.23 → hafnia-0.1.25}/src/cli/experiment_cmds.py

@@ -13,32 +13,6 @@ def experiment() -> None:
     pass
 
 
-@experiment.command(name="create_recipe")
-@click.option("--source_folder", default=".", type=Path, help="Path to the source folder", show_default=True)
-@click.option(
-    "--recipe_filename",
-    default="recipe.zip",
-    type=Path,
-    help="Recipe filename. Should have a '.zip' suffix",
-    show_default=True,
-)
-def create_recipe(source_folder: str, recipe_filename: str) -> None:
-    """Build recipe from local path as image with prefix - localhost"""
-
-    from hafnia.platform.builder import validate_recipe
-    from hafnia.utils import archive_dir
-
-    path_output_zip = Path(recipe_filename)
-
-    if path_output_zip.suffix != ".zip":
-        raise click.ClickException("Recipe filename must be a '.zip' file")
-
-    path_source = Path(source_folder)
-
-    path_output_zip = archive_dir(path_source, path_output_zip)
-    validate_recipe(path_output_zip)
-
-
 @experiment.command(name="create")
 @click.argument("name")
 @click.argument("source_dir", type=Path)

hafnia-0.1.25/src/cli/recipe_cmds.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+import click
+
+import cli.consts as consts
+
+
+@click.group(name="recipe")
+def recipe() -> None:
+    """Hafnia Recipe management commands"""
+    pass
+
+
+@recipe.command(name="create")
+@click.argument("source")
+@click.option(
+    "--output", type=click.Path(writable=True), default="./recipe.zip", show_default=True, help="Output recipe path."
+)
+def create(source: str, output: str) -> None:
+    """Create HRF from local path"""
+
+    from hafnia.platform.builder import validate_recipe
+    from hafnia.utils import archive_dir
+
+    path_output_zip = Path(output)
+    if path_output_zip.suffix != ".zip":
+        raise click.ClickException(consts.ERROR_RECIPE_FILE_FORMAT)
+
+    path_source = Path(source)
+    path_output_zip = archive_dir(path_source, path_output_zip)
+    validate_recipe(path_output_zip)
+
+
+@recipe.command(name="view")
+@click.option("--path", type=str, default="./recipe.zip", show_default=True, help="Path of recipe.zip.")
+@click.option("--depth-limit", type=int, default=3, help="Limit the depth of the tree view.", show_default=True)
+def view(path: str, depth_limit: int) -> None:
+    """View the content of a recipe zip file."""
+    from hafnia.utils import view_recipe_content
+
+    path_recipe = Path(path)
+    if not path_recipe.exists():
+        raise click.ClickException(
+            f"Recipe file '{path_recipe}' does not exist. Please provide a valid path. "
+            f"To create a recipe, use the 'hafnia recipe create' command."
+        )
+
+    tree_str = view_recipe_content(path_recipe, depth_limit=depth_limit)
+    click.echo(tree_str)

hafnia-0.1.25/src/cli/runc_cmds.py

@@ -0,0 +1,143 @@
+import json
+import subprocess
+import zipfile
+from hashlib import sha256
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Optional
+
+import click
+
+from cli.config import Config
+
+
+@click.group(name="runc")
+def runc():
+    """Experiment management commands"""
+    pass
+
+
+@runc.command(name="launch")
+@click.argument("task", required=True)
+def launch(task: str) -> None:
+    """Launch a job within the image."""
+    from hafnia.platform.executor import handle_launch
+
+    handle_launch(task)
+
+
+@runc.command(name="launch-local")
+@click.argument("exec_cmd", type=str)
+@click.option(
+    "--dataset",
+    type=str,
+    help="Hafnia dataset name e.g. mnist, midwest-vehicle-detection or a path to a local dataset",
+    required=True,
+)
+@click.option(
+    "--image_name",
+    type=Optional[str],
+    default=None,
+    help=(
+        "Docker image name to use for the launch. "
+        "By default, it will use image name from '.state.json' "
+        "file generated by the 'hafnia runc build-local' command"
+    ),
+)
+@click.pass_obj
+def launch_local(cfg: Config, exec_cmd: str, dataset: str, image_name: str) -> None:
+    """Launch a job within the image."""
+    from hafnia.data.factory import download_or_get_dataset_path
+
+    is_local_dataset = "/" in dataset
+    if is_local_dataset:
+        click.echo(f"Using local dataset: {dataset}")
+        path_dataset = Path(dataset)
+        if not path_dataset.exists():
+            raise click.ClickException(f"Dataset path does not exist: {path_dataset}")
+    else:
+        click.echo(f"Using Hafnia dataset: {dataset}")
+        path_dataset = download_or_get_dataset_path(dataset_name=dataset, cfg=cfg, force_redownload=False)
+
+    if image_name is None:
+        # Load image name from state.json
+        path_state_file = Path("state.json")
+        if not path_state_file.exists():
+            raise click.ClickException("State file does not exist. Please build the image first.")
+        state_dict = json.loads(path_state_file.read_text())
+        if "mdi_tag" not in state_dict:
+            raise click.ClickException("mdi_tag not found in state file. Please build the image first.")
+        image_name = state_dict["mdi_tag"]
+
+    docker_cmds = [
+        "docker",
+        "run",
+        "--rm",
+        "-v",
+        f"{path_dataset.absolute()}:/opt/ml/input/data/training",
+        "-e",
+        "HAFNIA_CLOUD=true",
+        "-e",
+        "PYTHONPATH=src",
+        "--runtime",
+        "nvidia",
+        image_name,
+    ] + exec_cmd.split(" ")
+
+    # Use the "hafnia runc launch" cmd when we have moved to the new folder structure and
+    # direct commands.
+    # Replace '+ exec_cmd.split(" ")' with '["hafnia", "runc", "launch"] + exec_cmd.split(" ")'
+
+    click.echo(f"Running command: \n\t{' '.join(docker_cmds)}")
+    subprocess.run(docker_cmds, check=True)
+
+
+@runc.command(name="build")
+@click.argument("recipe_url")
+@click.argument("state_file", default="state.json")
+@click.argument("ecr_repository", default="localhost")
+@click.argument("image_name", default="recipe")
+@click.pass_obj
+def build(cfg: Config, recipe_url: str, state_file: str, ecr_repository: str, image_name: str) -> None:
+    """Build docker image with a given recipe."""
+    from hafnia.platform.builder import build_image, prepare_recipe
+
+    with TemporaryDirectory() as temp_dir:
+        image_info = prepare_recipe(recipe_url, Path(temp_dir), cfg.api_key)
+        image_info["name"] = image_name
+        build_image(image_info, ecr_repository, state_file)
+
+
+@runc.command(name="build-local")
+@click.argument("recipe")
+@click.argument("state_file", default="state.json")
+@click.argument("image_name", default="recipe")
+def build_local(recipe: str, state_file: str, image_name: str) -> None:
+    """Build recipe from local path as image with prefix - localhost"""
+
+    from hafnia.platform.builder import build_image, validate_recipe
+    from hafnia.utils import archive_dir
+
+    recipe_zip = Path(recipe)
+    recipe_created = False
+    if not recipe_zip.suffix == ".zip" and recipe_zip.is_dir():
+        recipe_zip = archive_dir(recipe_zip)
+        recipe_created = True
+
+    validate_recipe(recipe_zip)
+    click.echo("Recipe successfully validated")
+    with TemporaryDirectory() as temp_dir:
+        temp_dir_path = Path(temp_dir)
+        with zipfile.ZipFile(recipe_zip, "r") as zip_ref:
+            zip_ref.extractall(temp_dir_path)
+
+        image_info = {
+            "name": image_name,
+            "dockerfile": (temp_dir_path / "Dockerfile").as_posix(),
+            "docker_context": temp_dir_path.as_posix(),
+            "hash": sha256(recipe_zip.read_bytes()).hexdigest()[:8],
+        }
+        click.echo("Start building image")
+        build_image(image_info, "localhost", state_file=state_file)
+        if recipe_created:
+            recipe_zip.unlink()