hafnia 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/build.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/check_release.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/ci_cd.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/lint.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/publish_docker.yaml +3 -3
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/publish_pypi.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/tests.yaml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/PKG-INFO +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/pyproject.toml +1 -1
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/platform/datasets.py +24 -18
- {hafnia-0.2.3 → hafnia-0.2.4}/uv.lock +911 -877
- {hafnia-0.2.3 → hafnia-0.2.4}/.devcontainer/devcontainer.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.devcontainer/hooks/post_create +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/dependabot.yaml +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.github/workflows/Dockerfile +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.gitignore +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.pre-commit-config.yaml +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.python-version +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.vscode/extensions.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.vscode/launch.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/.vscode/settings.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/LICENSE +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/README.md +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/docs/cli.md +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/docs/release.md +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/examples/example_dataset_recipe.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/examples/example_hafnia_dataset.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/examples/example_logger.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/examples/example_torchvision_dataloader.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/__main__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/config.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/consts.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/dataset_cmds.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/experiment_cmds.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/profile_cmds.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/recipe_cmds.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/cli/runc_cmds.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/data/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/data/factory.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_helpers.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_names.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_recipe/dataset_recipe.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_recipe/recipe_transforms.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_recipe/recipe_types.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/dataset_upload_helper.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/hafnia_dataset.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/operations/dataset_stats.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/operations/dataset_transformations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/operations/table_transformations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/bbox.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/bitmask.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/classification.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/point.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/polygon.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/primitive.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/segmentation.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/dataset/primitives/utils.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/experiment/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/experiment/hafnia_logger.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/http.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/log.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/platform/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/platform/builder.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/platform/download.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/platform/experiment.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/torch_helpers.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/utils.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/visualizations/colors.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/src/hafnia/visualizations/image_visualizations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/__init__.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/conftest.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[caltech-101].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[caltech-256].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[cifar100].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[cifar10].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[coco-2017].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[midwest-vehicle-detection].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[mnist].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_check_dataset[tiny-dataset].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[caltech-101].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[caltech-256].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[cifar100].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[cifar10].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[coco-2017].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[midwest-vehicle-detection].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[mnist].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_samples/test_dataset_draw_image_and_target[tiny-dataset].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_blur_anonymization[coco-2017].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_blur_anonymization[tiny-dataset].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_draw_annotations[coco-2017].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_draw_annotations[tiny-dataset].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_mask_region[coco-2017].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/expected_images/test_visualizations/test_mask_region[tiny-dataset].png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/annotations.jsonl +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/annotations.parquet +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/data/182a2c0a3ce312cf.jpg +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/data/4e95c6eb6209880a.jpg +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/data/cf86c7a23edb55ce.jpg +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/coco-2017/dataset_info.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/annotations.jsonl +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/annotations.parquet +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/data/222bbd5721a8a86e.png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/data/3251d85443622e4c.png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/data/3657ababa44af9b6.png +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/data/micro_test_datasets/tiny-dataset/dataset_info.json +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/dataset_recipe/test_dataset_recipe_helpers.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/dataset_recipe/test_dataset_recipes.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/dataset_recipe/test_recipe_transformations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/operations/test_dataset_transformations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/operations/test_table_transformations.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/test_colors.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/test_dataset_helpers.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/test_hafnia_dataset.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/dataset/test_shape_primitives.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/helper_testing.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_builder.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_check_example_scripts.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_cli.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_hafnia_logger.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_samples.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_utils.py +0 -0
- {hafnia-0.2.3 → hafnia-0.2.4}/tests/test_visualizations.py +0 -0
|
@@ -17,7 +17,7 @@ jobs:
|
|
|
17
17
|
outputs:
|
|
18
18
|
package-version: ${{ steps.extract-version.outputs.package_version }}
|
|
19
19
|
steps:
|
|
20
|
-
- uses: actions/checkout@
|
|
20
|
+
- uses: actions/checkout@v5.0.0
|
|
21
21
|
- uses: actions/setup-python@v5.6.0
|
|
22
22
|
with:
|
|
23
23
|
python-version-file: ${{ inputs.python-version-file }}
|
|
@@ -24,14 +24,14 @@ jobs:
|
|
|
24
24
|
build:
|
|
25
25
|
runs-on: ubuntu-latest
|
|
26
26
|
steps:
|
|
27
|
-
- uses: actions/checkout@
|
|
27
|
+
- uses: actions/checkout@v5.0.0
|
|
28
28
|
- uses: actions/setup-python@v5.6.0
|
|
29
29
|
id: python
|
|
30
30
|
with:
|
|
31
31
|
python-version-file: ${{ inputs.python-version-file }}
|
|
32
32
|
|
|
33
33
|
- name: Download package artifact
|
|
34
|
-
uses: actions/download-artifact@
|
|
34
|
+
uses: actions/download-artifact@v5.0.0
|
|
35
35
|
with:
|
|
36
36
|
name: python-package
|
|
37
37
|
path: dist/
|
|
@@ -47,7 +47,7 @@ jobs:
|
|
|
47
47
|
echo "aws_region=${{ secrets.STAGE_AWS_REGION }}" >> $GITHUB_OUTPUT
|
|
48
48
|
fi
|
|
49
49
|
- name: Configure AWS credentials
|
|
50
|
-
uses: aws-actions/configure-aws-credentials@v4.3.
|
|
50
|
+
uses: aws-actions/configure-aws-credentials@v4.3.1
|
|
51
51
|
with:
|
|
52
52
|
role-to-assume: arn:aws:iam::${{ steps.env-vars.outputs.aws_account_id }}:role/${{ secrets.AWS_ROLE_NAME }}
|
|
53
53
|
aws-region: ${{ steps.env-vars.outputs.aws_region }}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import shutil
|
|
3
3
|
import subprocess
|
|
4
|
+
import sys
|
|
4
5
|
import tempfile
|
|
5
6
|
import uuid
|
|
6
7
|
from pathlib import Path
|
|
@@ -91,22 +92,30 @@ def download_dataset_from_access_endpoint(
|
|
|
91
92
|
s3_dataset_files = [f"{s3_uri}/{filename}" for filename in DATASET_FILENAMES_REQUIRED]
|
|
92
93
|
|
|
93
94
|
envs = resource_credentials.aws_credentials()
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
95
|
+
try:
|
|
96
|
+
fast_copy_files_s3(
|
|
97
|
+
src_paths=s3_dataset_files,
|
|
98
|
+
dst_paths=local_dataset_paths,
|
|
99
|
+
append_envs=envs,
|
|
100
|
+
description="Downloading annotations",
|
|
101
|
+
)
|
|
102
|
+
except ValueError as e:
|
|
103
|
+
user_logger.error(f"Failed to download annotations: {e}")
|
|
104
|
+
return
|
|
100
105
|
|
|
101
106
|
if not download_files:
|
|
102
107
|
return
|
|
103
108
|
dataset = HafniaDataset.from_path(path_dataset, check_for_images=False)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
109
|
+
try:
|
|
110
|
+
fast_copy_files_s3(
|
|
111
|
+
src_paths=dataset.samples[ColumnName.REMOTE_PATH].to_list(),
|
|
112
|
+
dst_paths=dataset.samples[ColumnName.FILE_NAME].to_list(),
|
|
113
|
+
append_envs=envs,
|
|
114
|
+
description="Downloading images",
|
|
115
|
+
)
|
|
116
|
+
except ValueError as e:
|
|
117
|
+
user_logger.error(f"Failed to download images: {e}")
|
|
118
|
+
return
|
|
110
119
|
|
|
111
120
|
|
|
112
121
|
def fast_copy_files_s3(
|
|
@@ -117,7 +126,6 @@ def fast_copy_files_s3(
|
|
|
117
126
|
) -> List[str]:
|
|
118
127
|
if len(src_paths) != len(dst_paths):
|
|
119
128
|
raise ValueError("Source and destination paths must have the same length.")
|
|
120
|
-
|
|
121
129
|
cmds = [f"cp {src} {dst}" for src, dst in zip(src_paths, dst_paths)]
|
|
122
130
|
lines = execute_s5cmd_commands(cmds, append_envs=append_envs, description=description)
|
|
123
131
|
return lines
|
|
@@ -134,11 +142,9 @@ def execute_s5cmd_commands(
|
|
|
134
142
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
135
143
|
tmp_file_path = Path(temp_dir, f"{uuid.uuid4().hex}.txt")
|
|
136
144
|
tmp_file_path.write_text("\n".join(commands))
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
str(tmp_file_path),
|
|
141
|
-
]
|
|
145
|
+
s5cmd_bin = (Path(sys.executable).parent / "s5cmd").absolute().as_posix()
|
|
146
|
+
run_cmds = [s5cmd_bin, "run", str(tmp_file_path)]
|
|
147
|
+
sys_logger.debug(run_cmds)
|
|
142
148
|
envs = os.environ.copy()
|
|
143
149
|
envs.update(append_envs)
|
|
144
150
|
|