hafnia 0.5.2__tar.gz → 0.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hafnia-0.5.2 → hafnia-0.5.3}/.vscode/launch.json +17 -2
- {hafnia-0.5.2 → hafnia-0.5.3}/PKG-INFO +14 -10
- {hafnia-0.5.2 → hafnia-0.5.3}/README.md +13 -9
- {hafnia-0.5.2 → hafnia-0.5.3}/examples/example_dataset_recipe.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/examples/example_torchvision_dataloader.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/pyproject.toml +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_recipe/dataset_recipe.py +56 -101
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/hafnia_dataset.py +9 -49
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/hafnia_dataset_types.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/operations/dataset_stats.py +2 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/classification.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/segmentation.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/experiment/command_builder.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/dataset_recipe.py +30 -18
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/datasets.py +8 -4
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/experiment.py +12 -8
- hafnia-0.5.3/src/hafnia/platform/trainer_package.py +103 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/utils.py +7 -5
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/dataset_recipe_cmds.py +4 -8
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/experiment_cmds.py +15 -25
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/profile_cmds.py +8 -3
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/trainer_package_cmds.py +52 -4
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/conftest.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/helper_testing.py +1 -2
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_cli_integration.py +19 -3
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_dataset_recipes_with_platform.py +4 -5
- hafnia-0.5.3/tests/integration/test_dataset_versioning.py +47 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_samples.py +2 -45
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/dataset_recipe/test_dataset_recipes.py +65 -81
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_colors.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_utils.py +3 -3
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_visualizations.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/uv.lock +1 -1
- hafnia-0.5.2/src/hafnia/data/__init__.py +0 -3
- hafnia-0.5.2/src/hafnia/data/factory.py +0 -22
- hafnia-0.5.2/src/hafnia/platform/trainer_package.py +0 -57
- hafnia-0.5.2/tests/unit/dataset/dataset_recipe/test_dataset_recipe_helpers.py +0 -129
- {hafnia-0.5.2 → hafnia-0.5.3}/.devcontainer/devcontainer.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.devcontainer/hooks/post_create +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/dependabot.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/Dockerfile +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/build.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/check_release.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/ci_cd.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/lint.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/publish_docker.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/publish_pypi.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.github/workflows/tests.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.gitignore +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.pre-commit-config.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.python-version +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.trivyignore +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.vscode/extensions.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/.vscode/settings.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/LICENSE +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/docs/cli.md +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/docs/release.md +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/examples/example_hafnia_dataset.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/examples/example_logger.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/__init__.py +0 -0
- {hafnia-0.5.2/src/hafnia/visualizations → hafnia-0.5.3/src/hafnia/dataset}/colors.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_details_uploader.py +1 -1
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_helpers.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_names.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_recipe/recipe_transforms.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/dataset_recipe/recipe_types.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/format_conversions/format_coco.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/format_conversions/format_helpers.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/format_conversions/format_image_classification_folder.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/format_conversions/format_yolo.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/format_conversions/torchvision_datasets.py +0 -0
- {hafnia-0.5.2/src/hafnia/visualizations → hafnia-0.5.3/src/hafnia/dataset}/image_visualizations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/license_types.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/operations/dataset_s3_storage.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/operations/dataset_transformations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/operations/table_transformations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/__init__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/bbox.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/bitmask.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/point.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/polygon.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/primitive.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/dataset/primitives/utils.py +0 -0
- {hafnia-0.5.2/src/hafnia → hafnia-0.5.3/src/hafnia/dataset}/torch_helpers.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/experiment/__init__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/experiment/hafnia_logger.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/http.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/log.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/__init__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/builder.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/download.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia/platform/s5cmd_utils.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/__init__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/__main__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/config.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/consts.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/dataset_cmds.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/keychain.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/src/hafnia_cli/runc_cmds.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/__init__.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/train/000000000632.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/train/000000000724.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/train/_annotations.coco.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/valid/000000000139.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/valid/000000000285.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_coco_roboflow/valid/_annotations.coco.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/obj.names +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/train/data/000000000139.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/train/data/000000000139.txt +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/train/data/000000000285.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/train/data/000000000285.txt +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/train/images.txt +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/validation/data/000000000632.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/validation/data/000000000632.txt +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_formats/format_yolo/validation/images.txt +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/dataset_image_metadata_schema.yaml +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_dataset_transformations/test_video_storage_format_read_image.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_convert_segmentation_to_rle_list[polygon].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_convert_segmentation_to_rle_list[rle_as_ints].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_convert_segmentation_to_rle_list[rle_compressed_bytes].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_convert_segmentation_to_rle_list[rle_compressed_str].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_from_coco_format_visualized.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_coco/test_to_coco_format_visualized.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_yolo/test_format_yolo_import_export_tiny_dataset.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_format_yolo/test_import_yolo_format_visualized.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[caltech-101].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[caltech-256].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[cifar100].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[cifar10].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[coco-2017].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[midwest-vehicle-detection].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[mnist].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_check_dataset[tiny-dataset].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[caltech-101].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[caltech-256].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[cifar100].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[cifar10].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[coco-2017].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[midwest-vehicle-detection].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[mnist].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[tiny-dataset].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_blur_anonymization[micro-coco-2017].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_blur_anonymization[micro-tiny-dataset].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_draw_annotations[micro-coco-2017].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_draw_annotations[micro-tiny-dataset].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_mask_region[micro-coco-2017].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_mask_region[micro-tiny-dataset].png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/expected_images/test_visualizations/test_polygon_to_bitmask_conversion.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/annotations.jsonl +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/annotations.parquet +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/data/253/253925d334c002ce6662d8133535dd4c.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/data/b1a/b1a09f4d922f8f6904bab0c1caf172ab.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/data/f67/f675c8a1e862b5e00203ab888ac7fff4.jpg +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-coco-2017/dataset_info.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/annotations.jsonl +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/annotations.parquet +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/data/25c/25c3a206e7b60ab50245ee3d52d97f11.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/data/962/962fd865fdd45f169d5ca8c8f284d68d.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/data/ec6/ec60f2f4fb854b59c97e16b45c713de0.png +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/data/micro_test_datasets/micro-tiny-dataset/dataset_info.json +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/helper_testing_datasets.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_bring_your_own_data.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_check_example_scripts.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_dataset_merges.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/integration/test_torchvision_datasets.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/dataset_recipe/test_recipe_transformations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/format_conversions/test_format_coco.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/format_conversions/test_format_image_classification_folder.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/format_conversions/test_format_yolo.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/operations/test_dataset_stats.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/operations/test_dataset_transformations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/operations/test_table_transformations.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_dataset_details_uploader.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_dataset_helpers.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_dataset_names.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_hafnia_dataset.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_hafnia_dataset_types.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/dataset/test_shape_primitives.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_builder.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_cli.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_command_builder.py +0 -0
- {hafnia-0.5.2 → hafnia-0.5.3}/tests/unit/test_hafnia_logger.py +0 -0
|
@@ -77,9 +77,24 @@
|
|
|
77
77
|
"--dataset",
|
|
78
78
|
"mnist",
|
|
79
79
|
]
|
|
80
|
-
},
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"name": "cmd: 'hafnia trainer create [X]'",
|
|
83
|
+
"type": "debugpy",
|
|
84
|
+
"request": "launch",
|
|
85
|
+
"program": "${workspaceFolder}/src/hafnia_cli/__main__.py",
|
|
86
|
+
"args": [
|
|
87
|
+
"trainer",
|
|
88
|
+
"create",
|
|
89
|
+
"${workspaceFolder}/../trainer-classification",
|
|
90
|
+
"--name",
|
|
91
|
+
"Classification Trainer Package Launched in debug mode from vc-code",
|
|
92
|
+
"--description",
|
|
93
|
+
"A trainer package for image classification tasks launched in debug mode."
|
|
94
|
+
]
|
|
95
|
+
},
|
|
81
96
|
{
|
|
82
|
-
"name": "cmd: 'hafnia
|
|
97
|
+
"name": "cmd: 'hafnia trainer [X]'",
|
|
83
98
|
"type": "debugpy",
|
|
84
99
|
"request": "launch",
|
|
85
100
|
"program": "${workspaceFolder}/src/hafnia_cli/__main__.py",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hafnia
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.3
|
|
4
4
|
Summary: Python SDK for communication with Hafnia platform.
|
|
5
5
|
Author-email: Milestone Systems <hafniaplatform@milestone.dk>
|
|
6
6
|
License-File: LICENSE
|
|
@@ -64,9 +64,16 @@ multiple GPUs and instances if needed.
|
|
|
64
64
|
## Getting started: Configuration
|
|
65
65
|
To get started with Hafnia:
|
|
66
66
|
|
|
67
|
-
1. Install `hafnia` with your favorite python package manager
|
|
67
|
+
1. Install `hafnia` with your favorite python package manager:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
# With uv package manager
|
|
71
|
+
uv add hafnia
|
|
72
|
+
|
|
73
|
+
# With pip
|
|
74
|
+
pip install hafnia
|
|
75
|
+
```
|
|
68
76
|
|
|
69
|
-
`pip install hafnia`
|
|
70
77
|
1. Sign in to the [Hafnia Platform](https://hafnia.milestonesys.com/).
|
|
71
78
|
1. Create an API KEY for Training aaS. For more instructions, follow this
|
|
72
79
|
[guide](https://hafnia.readme.io/docs/create-an-api-key).
|
|
@@ -94,11 +101,9 @@ With Hafnia configured on your local machine, it is now possible to download
|
|
|
94
101
|
and explore the dataset sample with a python script:
|
|
95
102
|
|
|
96
103
|
```python
|
|
97
|
-
from hafnia.data import get_dataset_path
|
|
98
104
|
from hafnia.dataset.hafnia_dataset import HafniaDataset
|
|
99
105
|
|
|
100
|
-
|
|
101
|
-
path_dataset = get_dataset_path("midwest-vehicle-detection")
|
|
106
|
+
dataset = HafniaDataset.from_name("midwest-vehicle-detection")
|
|
102
107
|
```
|
|
103
108
|
|
|
104
109
|
This will download the dataset sample `midwest-vehicle-detection` to the local `.data/datasets/` folder
|
|
@@ -124,11 +129,10 @@ midwest-vehicle-detection
|
|
|
124
129
|
3 directories, 217 files
|
|
125
130
|
```
|
|
126
131
|
|
|
127
|
-
|
|
128
|
-
for loading/saving, managing and interacting with the dataset.
|
|
132
|
+
We provide the `HafniaDataset` format for loading/saving, managing and interacting with the dataset.
|
|
129
133
|
|
|
130
134
|
We recommend the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py)
|
|
131
|
-
for a
|
|
135
|
+
for a quick introduction on the `HafniaDataset`.
|
|
132
136
|
|
|
133
137
|
Below is a short introduction to the `HafniaDataset` class.
|
|
134
138
|
|
|
@@ -136,7 +140,7 @@ Below is a short introduction to the `HafniaDataset` class.
|
|
|
136
140
|
from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
|
|
137
141
|
|
|
138
142
|
# Load dataset from path
|
|
139
|
-
dataset = HafniaDataset.
|
|
143
|
+
dataset = HafniaDataset.from_path(path_dataset)
|
|
140
144
|
|
|
141
145
|
# Or get dataset directly by name
|
|
142
146
|
dataset = HafniaDataset.from_name("midwest-vehicle-detection")
|
|
@@ -34,9 +34,16 @@ multiple GPUs and instances if needed.
|
|
|
34
34
|
## Getting started: Configuration
|
|
35
35
|
To get started with Hafnia:
|
|
36
36
|
|
|
37
|
-
1. Install `hafnia` with your favorite python package manager
|
|
37
|
+
1. Install `hafnia` with your favorite python package manager:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
# With uv package manager
|
|
41
|
+
uv add hafnia
|
|
42
|
+
|
|
43
|
+
# With pip
|
|
44
|
+
pip install hafnia
|
|
45
|
+
```
|
|
38
46
|
|
|
39
|
-
`pip install hafnia`
|
|
40
47
|
1. Sign in to the [Hafnia Platform](https://hafnia.milestonesys.com/).
|
|
41
48
|
1. Create an API KEY for Training aaS. For more instructions, follow this
|
|
42
49
|
[guide](https://hafnia.readme.io/docs/create-an-api-key).
|
|
@@ -64,11 +71,9 @@ With Hafnia configured on your local machine, it is now possible to download
|
|
|
64
71
|
and explore the dataset sample with a python script:
|
|
65
72
|
|
|
66
73
|
```python
|
|
67
|
-
from hafnia.data import get_dataset_path
|
|
68
74
|
from hafnia.dataset.hafnia_dataset import HafniaDataset
|
|
69
75
|
|
|
70
|
-
|
|
71
|
-
path_dataset = get_dataset_path("midwest-vehicle-detection")
|
|
76
|
+
dataset = HafniaDataset.from_name("midwest-vehicle-detection")
|
|
72
77
|
```
|
|
73
78
|
|
|
74
79
|
This will download the dataset sample `midwest-vehicle-detection` to the local `.data/datasets/` folder
|
|
@@ -94,11 +99,10 @@ midwest-vehicle-detection
|
|
|
94
99
|
3 directories, 217 files
|
|
95
100
|
```
|
|
96
101
|
|
|
97
|
-
|
|
98
|
-
for loading/saving, managing and interacting with the dataset.
|
|
102
|
+
We provide the `HafniaDataset` format for loading/saving, managing and interacting with the dataset.
|
|
99
103
|
|
|
100
104
|
We recommend the example script [examples/example_hafnia_dataset.py](examples/example_hafnia_dataset.py)
|
|
101
|
-
for a
|
|
105
|
+
for a quick introduction on the `HafniaDataset`.
|
|
102
106
|
|
|
103
107
|
Below is a short introduction to the `HafniaDataset` class.
|
|
104
108
|
|
|
@@ -106,7 +110,7 @@ Below is a short introduction to the `HafniaDataset` class.
|
|
|
106
110
|
from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample
|
|
107
111
|
|
|
108
112
|
# Load dataset from path
|
|
109
|
-
dataset = HafniaDataset.
|
|
113
|
+
dataset = HafniaDataset.from_path(path_dataset)
|
|
110
114
|
|
|
111
115
|
# Or get dataset directly by name
|
|
112
116
|
dataset = HafniaDataset.from_name("midwest-vehicle-detection")
|
|
@@ -36,7 +36,7 @@ rprint(dataset_recipe)
|
|
|
36
36
|
|
|
37
37
|
# Example: Saving and loading a dataset recipe from file.
|
|
38
38
|
path_recipe = Path(".data/dataset_recipes/example_recipe.json")
|
|
39
|
-
|
|
39
|
+
dataset_recipe.as_json_file(path_recipe)
|
|
40
40
|
dataset_recipe_again: DatasetRecipe = DatasetRecipe.from_json_file(path_recipe)
|
|
41
41
|
|
|
42
42
|
# Verify that the loaded recipe is identical to the original recipe.
|
|
@@ -6,7 +6,7 @@ import torchvision.transforms.functional
|
|
|
6
6
|
from torch.utils.data import DataLoader
|
|
7
7
|
from torchvision.transforms import v2
|
|
8
8
|
|
|
9
|
-
from hafnia import torch_helpers
|
|
9
|
+
from hafnia.dataset import torch_helpers
|
|
10
10
|
from hafnia.dataset.hafnia_dataset import HafniaDataset
|
|
11
11
|
|
|
12
12
|
if __name__ == "__main__":
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
|
+
import shutil
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
|
|
7
8
|
|
|
@@ -12,6 +13,7 @@ from pydantic import (
|
|
|
12
13
|
|
|
13
14
|
from hafnia import utils
|
|
14
15
|
from hafnia.dataset.dataset_helpers import dataset_name_and_version_from_string
|
|
16
|
+
from hafnia.dataset.dataset_names import FILENAME_RECIPE_JSON
|
|
15
17
|
from hafnia.dataset.dataset_recipe import recipe_transforms
|
|
16
18
|
from hafnia.dataset.dataset_recipe.recipe_types import (
|
|
17
19
|
RecipeCreation,
|
|
@@ -22,6 +24,7 @@ from hafnia.dataset.hafnia_dataset import (
|
|
|
22
24
|
HafniaDataset,
|
|
23
25
|
available_dataset_versions_from_name,
|
|
24
26
|
)
|
|
27
|
+
from hafnia.dataset.hafnia_dataset_types import DatasetMetadataFilePaths
|
|
25
28
|
from hafnia.dataset.primitives.primitive import Primitive
|
|
26
29
|
from hafnia.log import user_logger
|
|
27
30
|
|
|
@@ -117,6 +120,23 @@ class DatasetRecipe(Serializable):
|
|
|
117
120
|
json_str = path_json.read_text(encoding="utf-8")
|
|
118
121
|
return DatasetRecipe.from_json_str(json_str)
|
|
119
122
|
|
|
123
|
+
@staticmethod
|
|
124
|
+
def from_recipe_field(recipe_field: Union[str, Dict[str, Any]]) -> "DatasetRecipe":
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
Deserialize from a recipe field which can be either a string or a dictionary.
|
|
128
|
+
|
|
129
|
+
string: A dataset name and version string in the format 'name:version'.
|
|
130
|
+
dict: A dictionary representation of the DatasetRecipe.
|
|
131
|
+
|
|
132
|
+
"""
|
|
133
|
+
if isinstance(recipe_field, str):
|
|
134
|
+
return DatasetRecipe.from_name_and_version_string(recipe_field)
|
|
135
|
+
elif isinstance(recipe_field, dict):
|
|
136
|
+
return DatasetRecipe.from_dict(recipe_field)
|
|
137
|
+
|
|
138
|
+
raise TypeError(f"Expected str or dict for recipe_field, got {type(recipe_field).__name__}.")
|
|
139
|
+
|
|
120
140
|
@staticmethod
|
|
121
141
|
def from_dict(data: Dict[str, Any]) -> "DatasetRecipe":
|
|
122
142
|
"""Deserialize from a dictionary."""
|
|
@@ -130,14 +150,9 @@ class DatasetRecipe(Serializable):
|
|
|
130
150
|
from hafnia_cli.config import Config
|
|
131
151
|
|
|
132
152
|
cfg = Config()
|
|
133
|
-
|
|
134
|
-
recipe_dict = get_dataset_recipe_by_id(recipe_id, endpoint=endpoint_dataset, api_key=cfg.api_key)
|
|
153
|
+
recipe_dict = get_dataset_recipe_by_id(recipe_id, cfg=cfg)
|
|
135
154
|
recipe_dict = recipe_dict["template"]["body"]
|
|
136
|
-
|
|
137
|
-
return DatasetRecipe.from_implicit_form(recipe_dict)
|
|
138
|
-
|
|
139
|
-
recipe = DatasetRecipe.from_dict(recipe_dict)
|
|
140
|
-
return recipe
|
|
155
|
+
return DatasetRecipe.from_recipe_field(recipe_dict)
|
|
141
156
|
|
|
142
157
|
@staticmethod
|
|
143
158
|
def from_recipe_name(name: str) -> "DatasetRecipe":
|
|
@@ -146,8 +161,7 @@ class DatasetRecipe(Serializable):
|
|
|
146
161
|
from hafnia_cli.config import Config
|
|
147
162
|
|
|
148
163
|
cfg = Config()
|
|
149
|
-
|
|
150
|
-
recipe = get_dataset_recipe_by_name(name=name, endpoint=endpoint_dataset, api_key=cfg.api_key)
|
|
164
|
+
recipe = get_dataset_recipe_by_name(name=name, cfg=cfg)
|
|
151
165
|
if not recipe:
|
|
152
166
|
raise ValueError(f"Dataset recipe '{name}' not found.")
|
|
153
167
|
recipe_id = recipe["id"]
|
|
@@ -168,82 +182,6 @@ class DatasetRecipe(Serializable):
|
|
|
168
182
|
|
|
169
183
|
return DatasetRecipe.from_name(name=dataset_name, version=version)
|
|
170
184
|
|
|
171
|
-
@staticmethod
|
|
172
|
-
def from_implicit_form(recipe: Any) -> DatasetRecipe:
|
|
173
|
-
"""
|
|
174
|
-
Recursively convert from implicit recipe to explicit form.
|
|
175
|
-
Handles mixed implicit/explicit recipes.
|
|
176
|
-
|
|
177
|
-
Conversion rules:
|
|
178
|
-
- str: Will get a dataset by name -> DatasetRecipeFromName
|
|
179
|
-
- Path: Will get a dataset from path -> DatasetRecipeFromPath
|
|
180
|
-
- tuple: Will merge datasets specified in the tuple -> RecipeMerger
|
|
181
|
-
- list: Will define a list of transformations -> RecipeTransforms
|
|
182
|
-
|
|
183
|
-
Example: DataRecipe from dataset name:
|
|
184
|
-
```python
|
|
185
|
-
recipe_implicit = "mnist"
|
|
186
|
-
recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
|
|
187
|
-
>>> recipe_explicit
|
|
188
|
-
DatasetRecipeFromName(dataset_name='mnist', force_redownload=False)
|
|
189
|
-
```
|
|
190
|
-
|
|
191
|
-
Example: DataRecipe from tuple (merging multiple recipes):
|
|
192
|
-
```python
|
|
193
|
-
recipe_implicit = ("dataset1", "dataset2")
|
|
194
|
-
recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
|
|
195
|
-
>>> recipe_explicit
|
|
196
|
-
RecipeMerger(
|
|
197
|
-
recipes=[
|
|
198
|
-
DatasetRecipeFromName(dataset_name='dataset1', force_redownload=False),
|
|
199
|
-
DatasetRecipeFromName(dataset_name='dataset2', force_redownload=False)
|
|
200
|
-
]
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
Example: DataRecipe from list (recipe and transformations):
|
|
204
|
-
```python
|
|
205
|
-
recipe_implicit = ["mnist", SelectSamples(n_samples=20), Shuffle(seed=123)]
|
|
206
|
-
recipe_explicit = DatasetRecipe.from_implicit_form(recipe_implicit)
|
|
207
|
-
>>> recipe_explicit
|
|
208
|
-
Transforms(
|
|
209
|
-
recipe=DatasetRecipeFromName(dataset_name='mnist', force_redownload=False),
|
|
210
|
-
transforms=[SelectSamples(n_samples=20), Shuffle(seed=123)]
|
|
211
|
-
)
|
|
212
|
-
```
|
|
213
|
-
|
|
214
|
-
"""
|
|
215
|
-
if isinstance(recipe, DatasetRecipe): # type: ignore
|
|
216
|
-
# It is possible to do an early return if recipe is a 'DataRecipe'-type even for nested and
|
|
217
|
-
# potentially mixed recipes. If you (really) think about it, this might surprise you,
|
|
218
|
-
# as this will bypass the conversion logic for nested recipes.
|
|
219
|
-
# However, this is not a problem as 'DataRecipe' classes are also pydantic models,
|
|
220
|
-
# so if a user introduces a 'DataRecipe'-class in the recipe (in potentially
|
|
221
|
-
# some nested and mixed implicit/explicit form) it will (due to pydantic validation) force
|
|
222
|
-
# the user to specify all nested recipes to be converted to explicit form.
|
|
223
|
-
return recipe
|
|
224
|
-
|
|
225
|
-
if isinstance(recipe, str): # str-type is convert to DatasetFromName
|
|
226
|
-
return DatasetRecipe.from_name_and_version_string(string=recipe, resolve_missing_version=True)
|
|
227
|
-
|
|
228
|
-
if isinstance(recipe, Path): # Path-type is convert to DatasetFromPath
|
|
229
|
-
return DatasetRecipe.from_path(path_folder=recipe)
|
|
230
|
-
|
|
231
|
-
if isinstance(recipe, tuple): # tuple-type is convert to DatasetMerger
|
|
232
|
-
recipes = [DatasetRecipe.from_implicit_form(item) for item in recipe]
|
|
233
|
-
return DatasetRecipe.from_merger(recipes=recipes)
|
|
234
|
-
|
|
235
|
-
if isinstance(recipe, list): # list-type is convert to Transforms
|
|
236
|
-
if len(recipe) == 0:
|
|
237
|
-
raise ValueError("List of recipes cannot be empty")
|
|
238
|
-
|
|
239
|
-
dataset_recipe = recipe[0] # First element is the dataset recipe
|
|
240
|
-
loader = DatasetRecipe.from_implicit_form(dataset_recipe)
|
|
241
|
-
|
|
242
|
-
transforms = recipe[1:] # Remaining items are transformations
|
|
243
|
-
return DatasetRecipe(creation=loader.creation, operations=transforms)
|
|
244
|
-
|
|
245
|
-
raise ValueError(f"Unsupported recipe type: {type(recipe)}")
|
|
246
|
-
|
|
247
185
|
### Upload, store and recipe conversions ###
|
|
248
186
|
def as_python_code(self, keep_default_fields: bool = False, as_kwargs: bool = True) -> str:
|
|
249
187
|
str_operations = [self.creation.as_python_code(keep_default_fields=keep_default_fields, as_kwargs=as_kwargs)]
|
|
@@ -285,17 +223,10 @@ class DatasetRecipe(Serializable):
|
|
|
285
223
|
from hafnia.platform.dataset_recipe import get_or_create_dataset_recipe
|
|
286
224
|
from hafnia_cli.config import Config
|
|
287
225
|
|
|
288
|
-
recipe = self.as_dict()
|
|
289
226
|
cfg = Config()
|
|
290
|
-
endpoint_dataset = cfg.get_platform_endpoint("dataset_recipes")
|
|
291
|
-
recipe_dict = get_or_create_dataset_recipe(
|
|
292
|
-
recipe=recipe,
|
|
293
|
-
endpoint=endpoint_dataset,
|
|
294
|
-
api_key=cfg.api_key,
|
|
295
|
-
name=recipe_name,
|
|
296
|
-
overwrite=overwrite,
|
|
297
|
-
)
|
|
298
227
|
|
|
228
|
+
recipe = self.as_dict()
|
|
229
|
+
recipe_dict = get_or_create_dataset_recipe(recipe=recipe, name=recipe_name, overwrite=overwrite, cfg=cfg)
|
|
299
230
|
return recipe_dict
|
|
300
231
|
|
|
301
232
|
### Dataset Recipe Transformations ###
|
|
@@ -428,13 +359,6 @@ def unique_name_from_recipe(recipe: DatasetRecipe) -> str:
|
|
|
428
359
|
return unique_name
|
|
429
360
|
|
|
430
361
|
|
|
431
|
-
def get_dataset_path_from_recipe(recipe: DatasetRecipe, path_datasets: Optional[Union[Path, str]] = None) -> Path:
|
|
432
|
-
path_datasets = path_datasets or utils.PATH_DATASETS
|
|
433
|
-
path_datasets = Path(path_datasets)
|
|
434
|
-
unique_dataset_name = unique_name_from_recipe(recipe)
|
|
435
|
-
return path_datasets / unique_dataset_name
|
|
436
|
-
|
|
437
|
-
|
|
438
362
|
class FromPath(RecipeCreation):
|
|
439
363
|
path_folder: Path
|
|
440
364
|
check_for_images: bool = True
|
|
@@ -525,3 +449,34 @@ class FromMerger(RecipeCreation):
|
|
|
525
449
|
for recipe in self.recipes:
|
|
526
450
|
names.extend(recipe.creation.get_dataset_names())
|
|
527
451
|
return names
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def get_dataset_path_from_recipe(recipe: DatasetRecipe, path_datasets: Optional[Union[Path, str]] = None) -> Path:
|
|
455
|
+
path_datasets = path_datasets or utils.PATH_DATASETS
|
|
456
|
+
path_datasets = Path(path_datasets)
|
|
457
|
+
unique_dataset_name = unique_name_from_recipe(recipe)
|
|
458
|
+
return path_datasets / unique_dataset_name
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def get_or_create_dataset_path_from_recipe(
|
|
462
|
+
dataset_recipe: DatasetRecipe,
|
|
463
|
+
force_redownload: bool = False,
|
|
464
|
+
path_datasets: Optional[Union[Path, str]] = None,
|
|
465
|
+
) -> Path:
|
|
466
|
+
path_dataset = get_dataset_path_from_recipe(dataset_recipe, path_datasets=path_datasets)
|
|
467
|
+
|
|
468
|
+
if force_redownload:
|
|
469
|
+
shutil.rmtree(path_dataset, ignore_errors=True)
|
|
470
|
+
|
|
471
|
+
dataset_metadata_files = DatasetMetadataFilePaths.from_path(path_dataset)
|
|
472
|
+
if dataset_metadata_files.exists(raise_error=False):
|
|
473
|
+
return path_dataset
|
|
474
|
+
|
|
475
|
+
path_dataset.mkdir(parents=True, exist_ok=True)
|
|
476
|
+
path_recipe_json = path_dataset / FILENAME_RECIPE_JSON
|
|
477
|
+
path_recipe_json.write_text(dataset_recipe.model_dump_json(indent=4))
|
|
478
|
+
|
|
479
|
+
dataset: HafniaDataset = dataset_recipe.build()
|
|
480
|
+
dataset.write(path_dataset)
|
|
481
|
+
|
|
482
|
+
return path_dataset
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import copy
|
|
4
|
-
import shutil
|
|
5
4
|
from dataclasses import dataclass
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from random import Random
|
|
@@ -14,7 +13,6 @@ from hafnia import utils
|
|
|
14
13
|
from hafnia.dataset import dataset_helpers
|
|
15
14
|
from hafnia.dataset.dataset_helpers import is_valid_version_string, version_from_string
|
|
16
15
|
from hafnia.dataset.dataset_names import (
|
|
17
|
-
FILENAME_RECIPE_JSON,
|
|
18
16
|
TAG_IS_SAMPLE,
|
|
19
17
|
PrimitiveField,
|
|
20
18
|
SampleField,
|
|
@@ -146,17 +144,6 @@ class HafniaDataset:
|
|
|
146
144
|
table = table_transformations.add_dataset_name_if_missing(table, dataset_name=info.dataset_name)
|
|
147
145
|
return HafniaDataset(info=info, samples=table)
|
|
148
146
|
|
|
149
|
-
@staticmethod
|
|
150
|
-
def from_recipe(dataset_recipe: Any) -> "HafniaDataset":
|
|
151
|
-
"""
|
|
152
|
-
Load a dataset from a recipe. The recipe can be a string (name of the dataset), a dictionary, or a DataRecipe object.
|
|
153
|
-
"""
|
|
154
|
-
from hafnia.dataset.dataset_recipe.dataset_recipe import DatasetRecipe
|
|
155
|
-
|
|
156
|
-
recipe_explicit = DatasetRecipe.from_implicit_form(dataset_recipe)
|
|
157
|
-
|
|
158
|
-
return recipe_explicit.build() # Build dataset from the recipe
|
|
159
|
-
|
|
160
147
|
@staticmethod
|
|
161
148
|
def from_merge(dataset0: "HafniaDataset", dataset1: "HafniaDataset") -> "HafniaDataset":
|
|
162
149
|
return HafniaDataset.merge(dataset0, dataset1)
|
|
@@ -172,6 +159,8 @@ class HafniaDataset:
|
|
|
172
159
|
If the dataset is already cached, it will be loaded from the cache.
|
|
173
160
|
"""
|
|
174
161
|
|
|
162
|
+
from hafnia.dataset.dataset_recipe.dataset_recipe import get_or_create_dataset_path_from_recipe
|
|
163
|
+
|
|
175
164
|
path_dataset = get_or_create_dataset_path_from_recipe(
|
|
176
165
|
dataset_recipe,
|
|
177
166
|
path_datasets=path_datasets,
|
|
@@ -245,7 +234,7 @@ class HafniaDataset:
|
|
|
245
234
|
|
|
246
235
|
Example: Defining split ratios and applying the transformation
|
|
247
236
|
|
|
248
|
-
>>> dataset = HafniaDataset.
|
|
237
|
+
>>> dataset = HafniaDataset.from_path(Path("path/to/dataset"))
|
|
249
238
|
>>> split_ratios = {SplitName.TRAIN: 0.8, SplitName.VAL: 0.1, SplitName.TEST: 0.1}
|
|
250
239
|
>>> dataset_with_splits = splits_by_ratios(dataset, split_ratios, seed=42)
|
|
251
240
|
Or use the function as a
|
|
@@ -270,7 +259,7 @@ class HafniaDataset:
|
|
|
270
259
|
splits based on the provided ratios.
|
|
271
260
|
|
|
272
261
|
Example: Defining split ratios and applying the transformation
|
|
273
|
-
>>> dataset = HafniaDataset.
|
|
262
|
+
>>> dataset = HafniaDataset.from_path(Path("path/to/dataset"))
|
|
274
263
|
>>> split_name = SplitName.TEST
|
|
275
264
|
>>> split_ratios = {SplitName.TEST: 0.8, SplitName.VAL: 0.2}
|
|
276
265
|
>>> dataset_with_splits = split_into_multiple_splits(dataset, split_name, split_ratios)
|
|
@@ -543,7 +532,7 @@ class HafniaDataset:
|
|
|
543
532
|
primitive: Type[Primitive],
|
|
544
533
|
task_name: Optional[str] = None,
|
|
545
534
|
keep_sample_data: bool = False,
|
|
546
|
-
) -> pl.DataFrame:
|
|
535
|
+
) -> Optional[pl.DataFrame]:
|
|
547
536
|
return table_transformations.create_primitive_table(
|
|
548
537
|
samples_table=self.samples,
|
|
549
538
|
PrimitiveType=primitive,
|
|
@@ -741,36 +730,6 @@ def check_hafnia_dataset_from_path(path_dataset: Path) -> None:
|
|
|
741
730
|
dataset.check_dataset()
|
|
742
731
|
|
|
743
732
|
|
|
744
|
-
def get_or_create_dataset_path_from_recipe(
|
|
745
|
-
dataset_recipe: Any,
|
|
746
|
-
force_redownload: bool = False,
|
|
747
|
-
path_datasets: Optional[Union[Path, str]] = None,
|
|
748
|
-
) -> Path:
|
|
749
|
-
from hafnia.dataset.dataset_recipe.dataset_recipe import (
|
|
750
|
-
DatasetRecipe,
|
|
751
|
-
get_dataset_path_from_recipe,
|
|
752
|
-
)
|
|
753
|
-
|
|
754
|
-
recipe: DatasetRecipe = DatasetRecipe.from_implicit_form(dataset_recipe)
|
|
755
|
-
path_dataset = get_dataset_path_from_recipe(recipe, path_datasets=path_datasets)
|
|
756
|
-
|
|
757
|
-
if force_redownload:
|
|
758
|
-
shutil.rmtree(path_dataset, ignore_errors=True)
|
|
759
|
-
|
|
760
|
-
dataset_metadata_files = DatasetMetadataFilePaths.from_path(path_dataset)
|
|
761
|
-
if dataset_metadata_files.exists(raise_error=False):
|
|
762
|
-
return path_dataset
|
|
763
|
-
|
|
764
|
-
path_dataset.mkdir(parents=True, exist_ok=True)
|
|
765
|
-
path_recipe_json = path_dataset / FILENAME_RECIPE_JSON
|
|
766
|
-
path_recipe_json.write_text(recipe.model_dump_json(indent=4))
|
|
767
|
-
|
|
768
|
-
dataset: HafniaDataset = recipe.build()
|
|
769
|
-
dataset.write(path_dataset)
|
|
770
|
-
|
|
771
|
-
return path_dataset
|
|
772
|
-
|
|
773
|
-
|
|
774
733
|
def available_dataset_versions_from_name(dataset_name: str) -> Dict[Version, "DatasetMetadataFilePaths"]:
|
|
775
734
|
credentials: ResourceCredentials = get_read_credentials_by_name(dataset_name=dataset_name)
|
|
776
735
|
return available_dataset_versions(credentials=credentials)
|
|
@@ -795,12 +754,13 @@ def select_version_from_available_versions(
|
|
|
795
754
|
|
|
796
755
|
if version is None:
|
|
797
756
|
str_versions = [str(v) for v in available_versions]
|
|
798
|
-
raise ValueError(f"Version must be specified. Available versions: {str_versions}")
|
|
799
|
-
|
|
757
|
+
raise ValueError(f"Version must be specified. Available versions: {str_versions}. ")
|
|
758
|
+
|
|
759
|
+
if version == "latest":
|
|
800
760
|
version_casted = max(available_versions)
|
|
801
761
|
user_logger.info(f"'latest' version '{version_casted}' has been selected")
|
|
802
762
|
else:
|
|
803
|
-
version_casted = version_from_string(version)
|
|
763
|
+
version_casted = version_from_string(version, raise_error=True)
|
|
804
764
|
|
|
805
765
|
if version_casted not in available_versions:
|
|
806
766
|
raise ValueError(f"Selected version '{version}' not found in available versions: {available_versions}")
|
|
@@ -470,7 +470,7 @@ class Sample(BaseModel):
|
|
|
470
470
|
return image
|
|
471
471
|
|
|
472
472
|
def draw_annotations(self, image: Optional[np.ndarray] = None) -> np.ndarray:
|
|
473
|
-
from hafnia.
|
|
473
|
+
from hafnia.dataset import image_visualizations
|
|
474
474
|
|
|
475
475
|
if image is None:
|
|
476
476
|
image = self.read_image()
|
|
@@ -104,7 +104,8 @@ def calculate_primitive_counts(dataset: HafniaDataset) -> Dict[str, int]:
|
|
|
104
104
|
name = task.primitive.__name__
|
|
105
105
|
if task.name != task.primitive.default_task_name():
|
|
106
106
|
name = f"{name}.{task.name}"
|
|
107
|
-
|
|
107
|
+
n_objects = 0 if objects is None else len(objects)
|
|
108
|
+
annotation_counts[name] = n_objects
|
|
108
109
|
return annotation_counts
|
|
109
110
|
|
|
110
111
|
|
|
@@ -39,7 +39,7 @@ class Classification(Primitive):
|
|
|
39
39
|
def draw(self, image: np.ndarray, inplace: bool = False, draw_label: bool = True) -> np.ndarray:
|
|
40
40
|
if draw_label is False:
|
|
41
41
|
return image
|
|
42
|
-
from hafnia.
|
|
42
|
+
from hafnia.dataset import image_visualizations
|
|
43
43
|
|
|
44
44
|
class_name = self.get_class_name()
|
|
45
45
|
if self.task_name == self.default_task_name():
|
|
@@ -4,9 +4,9 @@ import cv2
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
from pydantic import Field
|
|
6
6
|
|
|
7
|
+
from hafnia.dataset.colors import get_n_colors
|
|
7
8
|
from hafnia.dataset.primitives.primitive import Primitive
|
|
8
9
|
from hafnia.dataset.primitives.utils import get_class_name
|
|
9
|
-
from hafnia.visualizations.colors import get_n_colors
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class Segmentation(Primitive):
|
|
@@ -103,7 +103,7 @@ def auto_save_command_builder_schema(
|
|
|
103
103
|
bool_handling=bool_handling,
|
|
104
104
|
)
|
|
105
105
|
|
|
106
|
-
path_schema = path_schema or path_of_function(cli_function).with_suffix(".json")
|
|
106
|
+
path_schema = path_schema or path_of_function(cli_function).with_suffix(".schema.json")
|
|
107
107
|
launch_schema.to_json_file(path_schema)
|
|
108
108
|
return path_schema
|
|
109
109
|
|