hafnia 0.4.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

hafnia/platform/s5cmd_utils.py

@@ -0,0 +1,147 @@
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from hafnia.log import sys_logger, user_logger
+from hafnia.utils import progress_bar
+
+
+def find_s5cmd() -> Optional[str]:
+    """Locate the s5cmd executable across different installation methods.
+
+    Searches for s5cmd in:
+    1. System PATH (via shutil.which)
+    2. Python bin directory (Unix-like systems)
+    3. Python executable directory (direct installs)
+
+    Returns:
+        str: Absolute path to s5cmd executable if found, None otherwise.
+    """
+    result = shutil.which("s5cmd")
+    if result:
+        return result
+    python_dir = Path(sys.executable).parent
+    locations = (python_dir / "Scripts" / "s5cmd.exe", python_dir / "bin" / "s5cmd", python_dir / "s5cmd")
+    for loc in locations:
+        if loc.exists():
+            return str(loc)
+    return None
+
+
+def execute_command(args: List[str], append_envs: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess:
+    s5cmd_bin = find_s5cmd()
+    cmds = [s5cmd_bin] + args
+    envs = os.environ.copy()
+    if append_envs:
+        envs.update(append_envs)
+
+    result = subprocess.run(
+        cmds,  # type: ignore[arg-type]
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+        env=envs,
+    )
+    return result
+
+
+def execute_commands(
+    commands: List[str],
+    append_envs: Optional[Dict[str, str]] = None,
+    description: str = "Executing s5cmd commands",
+) -> List[str]:
+    append_envs = append_envs or {}
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        tmp_file_path = Path(temp_dir, f"{uuid.uuid4().hex}.txt")
+        tmp_file_path.write_text("\n".join(commands))
+
+        s5cmd_bin = find_s5cmd()
+        if s5cmd_bin is None:
+            raise ValueError("Can not find s5cmd executable.")
+        run_cmds = [s5cmd_bin, "run", str(tmp_file_path)]
+        sys_logger.debug(run_cmds)
+        envs = os.environ.copy()
+        envs.update(append_envs)
+
+        process = subprocess.Popen(
+            run_cmds,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            env=envs,
+        )
+
+        error_lines = []
+        lines = []
+        for line in progress_bar(process.stdout, total=len(commands), description=description):  # type: ignore[arg-type]
+            if "ERROR" in line or "error" in line:
+                error_lines.append(line.strip())
+            lines.append(line.strip())
+
+        if len(error_lines) > 0:
+            show_n_lines = min(5, len(error_lines))
+            str_error_lines = "\n".join(error_lines[:show_n_lines])
+            user_logger.error(
+                f"Detected {len(error_lines)} errors occurred while executing a total of {len(commands)} "
+                f" commands with s5cmd. The first {show_n_lines} is printed below:\n{str_error_lines}"
+            )
+            raise RuntimeError("Errors occurred during s5cmd execution.")
+    return lines
+
+
+def delete_bucket_content(
+    bucket_prefix: str,
+    remove_bucket: bool = True,
+    append_envs: Optional[Dict[str, str]] = None,
+) -> None:
+    # Remove all files in the bucket
+    returns = execute_command(["rm", f"{bucket_prefix}/*"], append_envs=append_envs)
+
+    if returns.returncode != 0:
+        bucket_is_already_deleted = "no object found" in returns.stderr.strip()
+        if bucket_is_already_deleted:
+            user_logger.info(f"No action was taken. S3 bucket '{bucket_prefix}' is already empty.")
+        else:
+            user_logger.error("Error during s5cmd rm command:")
+            user_logger.error(returns.stdout)
+            raise RuntimeError(f"Failed to delete all files in S3 bucket '{bucket_prefix}'.")
+
+    if remove_bucket:
+        # Remove the bucket itself
+        returns = execute_command(["rb", bucket_prefix], append_envs=append_envs)
+        if returns.returncode != 0:
+            user_logger.error("Error during s5cmd rb command:")
+            user_logger.error(returns.stdout)
+            raise RuntimeError(f"Failed to delete S3 bucket '{bucket_prefix}'.")
+    user_logger.info(f"S3 bucket '{bucket_prefix}' has been deleted.")
+
+
+def list_bucket(bucket_prefix: str, append_envs: Optional[Dict[str, str]] = None) -> List[str]:
+    output = execute_command(["ls", f"{bucket_prefix}/*"], append_envs=append_envs)
+    has_missing_folder = "no object found" in output.stderr.strip()
+    if output.returncode != 0 and not has_missing_folder:
+        user_logger.error("Error during s5cmd ls command:")
+        user_logger.error(output.stderr)
+        raise RuntimeError(f"Failed to list dataset in S3 bucket '{bucket_prefix}'.")
+
+    files_in_s3 = [f"{bucket_prefix}/{line.split(' ')[-1]}" for line in output.stdout.splitlines()]
+    return files_in_s3
+
+
+def fast_copy_files(
+    src_paths: List[str],
+    dst_paths: List[str],
+    append_envs: Optional[Dict[str, str]] = None,
+    description: str = "Copying files",
+) -> List[str]:
+    if len(src_paths) != len(dst_paths):
+        raise ValueError("Source and destination paths must have the same length.")
+    cmds = [f"cp {src} {dst}" for src, dst in zip(src_paths, dst_paths)]
+    lines = execute_commands(cmds, append_envs=append_envs, description=description)
+    return lines

hafnia/utils.py

@@ -65,6 +65,10 @@ def timed(label: str):
     return decorator
 
 
+def get_path_dataset_gallery_images(dataset_name: str) -> Path:
+    return PATH_DATASETS / dataset_name / "gallery_images"
+
+
 def get_path_hafnia_cache() -> Path:
     return Path.home() / "hafnia"
 

{hafnia-0.4.3.dist-info → hafnia-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hafnia
-Version: 0.4.3
+Version: 0.5.0
 Summary: Python SDK for communication with Hafnia platform.
 Author-email: Milestone Systems <hafniaplatform@milestone.dk>
 License-File: LICENSE
@@ -10,7 +10,7 @@ Requires-Dist: click>=8.1.8
 Requires-Dist: emoji>=2.14.1
 Requires-Dist: flatten-dict>=0.4.2
 Requires-Dist: keyring>=25.6.0
-Requires-Dist: mcp==1.16.0
+Requires-Dist: mcp>=1.16.0
 Requires-Dist: mlflow>=3.4.0
 Requires-Dist: more-itertools>=10.7.0
 Requires-Dist: opencv-python-headless>=4.11.0.86
@@ -209,7 +209,7 @@ DatasetInfo(
 ```
 
 You can iterate and access samples in the dataset using the `HafniaDataset` object.
-Each sample contain image and annotations information. 
+Each sample contain image and annotations information.
 
 ```python
 from hafnia.dataset.hafnia_dataset import HafniaDataset, Sample

{hafnia-0.4.3.dist-info → hafnia-0.5.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 hafnia/__init__.py,sha256=0qpjWfVbcfKzLSnfUW6RdclSGkesMQRFS-n_aTJJoSE,179
-hafnia/http.py,sha256=bjXbV_3uKbBdudqMdYtnsMttUAsNRMsetYZ4F2xXlEs,3635
+hafnia/http.py,sha256=PkEuanlUKeERABXttaGAJT6hOZ1_B2CwJodbUV4uZdg,3710
 hafnia/log.py,sha256=sWF8tz78yBtwZ9ddzm19L1MBSBJ3L4G704IGeT1_OEU,784
 hafnia/torch_helpers.py,sha256=Qj8pV5P8tGw6F3W2Rj9Kel7O8hWuUqiKfTdXd3h5UOo,14080
-hafnia/utils.py,sha256=AC4MgF-xkaVrVwKppQ8yZMlTgR3O9Q9ZGeG9sMqd1as,8578
+hafnia/utils.py,sha256=l_awkrb3OttxqSMkPiYcpuP3c_kejkSmiqndSahc1s0,8703
 hafnia/data/__init__.py,sha256=o9QjiGbEcNa6r-qDmwwmxPXf-1UitNl5-WxFNcujqsg,111
 hafnia/data/factory.py,sha256=kHkvOtBUbwaShZBGf1kZzocDJBn_1dHHLrQxnUpJmfY,778
-hafnia/dataset/dataset_details_uploader.py,sha256=I4hh4-hb8Agpv-o6qekx43dxZ_Lfngbqo6RX1D2DNbU,24854
-hafnia/dataset/dataset_helpers.py,sha256=0GbS6PfaiYBulDKRCbd0miN5LHaUIp-XzGt_wZay8xs,5044
-hafnia/dataset/dataset_names.py,sha256=5yt5fcotl-uc9aFM4Fp-Z-oNN0N1p9OY3tJ1ujSr1FQ,6601
-hafnia/dataset/hafnia_dataset.py,sha256=8RTMolUGgPen-F8IA0kDmxCGl4J5t-eendFZNfSFnsQ,29640
-hafnia/dataset/hafnia_dataset_types.py,sha256=aDRdDCBzHlkuY4t1qnCRLP8x_yGF2fCtpIQYmtDIJfA,21193
+hafnia/dataset/dataset_details_uploader.py,sha256=H_zz67bBwbgo4StUwBNmH89WlqydIc-tEQbrRnZDwgg,24161
+hafnia/dataset/dataset_helpers.py,sha256=SwQnFy0T0dGVRuBeT7tG_CHsb9WtW8v8mTg9TnQH2k4,4093
+hafnia/dataset/dataset_names.py,sha256=qYbrsqeKiTsu53w7uWDXdw0Y8cHDEHydoalakChKDcY,7413
+hafnia/dataset/hafnia_dataset.py,sha256=YzQjFAY3m5MJvKyHELWdS1mn9UY4xmGmnSRAS8nQRbY,34037
+hafnia/dataset/hafnia_dataset_types.py,sha256=B0yW7wI6Bf0ILw0HS3LeUu_w0NKJyTZhHvKXT_98q_A,21415
 hafnia/dataset/license_types.py,sha256=b1Jt5e8N89sujIs4T9y39sJEkzpAwCoLDTHDTpkiEOI,2166
 hafnia/dataset/dataset_recipe/dataset_recipe.py,sha256=CkU61k_qaPSu_4Yo-NArr7jHWP417LCmzDKnmzZuDeo,19080
 hafnia/dataset/dataset_recipe/recipe_transforms.py,sha256=j3Oiytt3LI2rCaJid7Y44oT9MXvlZVqvZanngMebIWg,3088
@@ -18,10 +18,11 @@ hafnia/dataset/format_conversions/format_coco.py,sha256=7GjeF016ZBaKxu-VYiqXxuPw
 hafnia/dataset/format_conversions/format_helpers.py,sha256=-lNgn_mrpVM_Xwb3jHH0BlBEPCMKjLqTeYf4PbssbuQ,1144
 hafnia/dataset/format_conversions/format_image_classification_folder.py,sha256=ALVsQfSvBfAkpmUDijznqqd2JBh0Qtnvzq7igxMjMb8,7170
 hafnia/dataset/format_conversions/format_yolo.py,sha256=zvCHo2L_0mPJScMbDtwvZUts9UX2ERKhhYbY31Q6tQA,9912
-hafnia/dataset/format_conversions/torchvision_datasets.py,sha256=EmdTKNxrPT-qE8u7ZzNi9TxdwEBdXRphVliKTWgN0B4,12007
+hafnia/dataset/format_conversions/torchvision_datasets.py,sha256=1H4AMaOYRe6xAU-Pjv7oB7djpis6W6qmzYdHg_hxwKg,12096
+hafnia/dataset/operations/dataset_s3_storage.py,sha256=kIQBK3yc8AqD1kDjWJi8m-ujOvZ5xvgsT93RHUpuByA,8864
 hafnia/dataset/operations/dataset_stats.py,sha256=Ltf-V4_o_IB4UXw9WG9bsVoqeX90yGsjivK0CDggriw,11930
 hafnia/dataset/operations/dataset_transformations.py,sha256=qUNno0rAT1A452uzlR-k1WbatyY9VuMp1QJjkMg9GzE,19495
-hafnia/dataset/operations/table_transformations.py,sha256=l90CLGNZZVhedcEH5r4T3f7OyCprj-2vNKeYXsmWku0,14370
+hafnia/dataset/operations/table_transformations.py,sha256=odcs6e7L-WmkpzkZcF7aRLXIzaXElQX6raFX8FfyZEk,14397
 hafnia/dataset/primitives/__init__.py,sha256=xFLJ3R7gpbuQnNJuFhuu836L3nicwoaY5aHkqk7Bbr8,927
 hafnia/dataset/primitives/bbox.py,sha256=QJJBebltOd9J3idisp3QdX0gCgz6P5xlIlGbth19fG0,6669
 hafnia/dataset/primitives/bitmask.py,sha256=Q7RiNYvMDlcFPkXAWXDJkCIERjnUTCrHu6VeEPX1jEA,7212
@@ -36,9 +37,10 @@ hafnia/experiment/hafnia_logger.py,sha256=BHIOLAds_3JxT0cev_ikUH0XQVIxBJTkcBSx2Q
 hafnia/platform/__init__.py,sha256=L_Q7CNpsJ0HMNPy_rLlLK5RhmuCU7IF4BchxKv6amYc,782
 hafnia/platform/builder.py,sha256=kUEuj5-qtL1uk5v2tUvOCREn5yV-G4Fr6F31haIAb5E,5808
 hafnia/platform/dataset_recipe.py,sha256=ybfSSHVPG0eFUbzg_1McezPSOtMoDZEg7l6rFYndtb4,3857
-hafnia/platform/datasets.py,sha256=RX4XqJ4GRYw4CvrF8-87KWPVBJbg97w8JAUqkWR5F_g,9022
+hafnia/platform/datasets.py,sha256=gfFHltZ70BJ3XeqxGvDMn06B1hmfhS-9bO149qtJruA,13079
 hafnia/platform/download.py,sha256=A6McEvRw5KmEAn3kx3nNnjxQm1Z8ZSAhn_NFgrM66_8,5020
 hafnia/platform/experiment.py,sha256=SrEH0nuwwBXf1Iu4diB1BEPqL-TxW3aQkZWBbM1-tY0,1846
+hafnia/platform/s5cmd_utils.py,sha256=clI_niKjya0E5d8Jmgvfj86jiuxMD9uK83znWKN6tYY,5335
 hafnia/platform/trainer_package.py,sha256=w6JC7o-279ujcwtNTbUaQ9AnPcYRPPbD8EACa6XyUHA,2206
 hafnia/visualizations/colors.py,sha256=003eAJVnBal4abaYIIpsrT7erIOIjTUHHYVJ1Tj1CDc,5226
 hafnia/visualizations/image_visualizations.py,sha256=rB7c-KK-qq0BsSdkaFxCAHOOCTXTUQx0VMEhib7ig0k,7509
@@ -46,15 +48,15 @@ hafnia_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hafnia_cli/__main__.py,sha256=CqD_7RfbfwB6MED3WZ8WxclrFxWcRMtZ5A1Snnst3GM,1761
 hafnia_cli/config.py,sha256=X0dJBYH-7mNAFkHgeZvDbawlQqoyCwoY4C-QhlyYCA0,7695
 hafnia_cli/consts.py,sha256=uCpYX44NCu_Zvte0QwChunxOo-qqhcaJRSYDAIsoJ8A,972
-hafnia_cli/dataset_cmds.py,sha256=fzw8LVn1mR319qorxK7HJ6Bs9lM9GdpMBkA6tAqssIg,1432
+hafnia_cli/dataset_cmds.py,sha256=gxUgTUMzZYEvwUdEDHTuTGFge7pqQPkXPKg20v4GGNU,1948
 hafnia_cli/dataset_recipe_cmds.py,sha256=OYSmpKL0Wxo1ZSxIGfH6w7pEWoI7CjUTmfIELJSZjGQ,2894
 hafnia_cli/experiment_cmds.py,sha256=_KxsMhbjlkIno1PIMXJ0Omw_PSJi8qi9hmtCUqwcj1M,7970
 hafnia_cli/keychain.py,sha256=bNyjjULVQu7kV338wUC65UvbCwmSGOmEjKWPLIQjT0k,2555
-hafnia_cli/profile_cmds.py,sha256=1SJGMQM9tBwvS3prSehw6CgHW9cmrO2EZ7CAsH4f7qU,3477
+hafnia_cli/profile_cmds.py,sha256=yTyOsPsUssLCzFIxURkxbKrFEhYIVDlUC0G2s5Uks-U,3476
 hafnia_cli/runc_cmds.py,sha256=38SGajsizpAydawCWL6gwza9NtLugtHfCdnldllwWJI,5016
 hafnia_cli/trainer_package_cmds.py,sha256=hUBc6gCMV28fcAA0xQdXKL1z-a3aL9lMWcVqjvHO1Uo,2326
-hafnia-0.4.3.dist-info/METADATA,sha256=VO_uuOnjeKRtV9yuyUKtILlZV6Oi8TqO1BwhcIXk5xg,19273
-hafnia-0.4.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hafnia-0.4.3.dist-info/entry_points.txt,sha256=j2jsj1pqajLAiSOnF7sq66A3d1SVeHPKVTVyIFzipSA,52
-hafnia-0.4.3.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
-hafnia-0.4.3.dist-info/RECORD,,
+hafnia-0.5.0.dist-info/METADATA,sha256=VRy-2ThTt7kczs4uPFU_qeKgiuye1JZ2Pv8VoN1H1e4,19272
+hafnia-0.5.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hafnia-0.5.0.dist-info/entry_points.txt,sha256=j2jsj1pqajLAiSOnF7sq66A3d1SVeHPKVTVyIFzipSA,52
+hafnia-0.5.0.dist-info/licenses/LICENSE,sha256=wLZw1B7_mod_CO1H8LXqQgfqlWD6QceJR8--LJYRZGE,1078
+hafnia-0.5.0.dist-info/RECORD,,

{hafnia-0.4.3.dist-info → hafnia-0.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

hafnia_cli/dataset_cmds.py

@@ -50,3 +50,21 @@ def cmd_dataset_download(cfg: Config, dataset_name: str, destination: Optional[c
     except Exception:
         raise click.ClickException(consts.ERROR_GET_RESOURCE)
     return path_dataset
+
+
+@dataset.command("delete")
+@click.argument("dataset_name")
+@click.option(
+    "--interactive/--no-interactive",
+    default=True,
+    help="Whether to ask for confirmation before deleting the dataset.",
+)
+@click.pass_obj
+def cmd_dataset_delete(cfg: Config, dataset_name: str, interactive: bool) -> None:
+    """Delete dataset from Hafnia platform"""
+    from hafnia.platform import datasets
+
+    datasets.delete_dataset_completely_by_name(
+        dataset_name=dataset_name,
+        interactive=interactive,
+    )

hafnia_cli/profile_cmds.py

@@ -94,7 +94,6 @@ def cmd_profile_active(cfg: Config) -> None:
 def profile_show(cfg: Config) -> None:
     masked_key = f"{cfg.api_key[:11]}...{cfg.api_key[-4:]}" if len(cfg.api_key) > 20 else "****"
     console = Console()
-
     table = Table(title=f"{consts.PROFILE_TABLE_HEADER} {cfg.active_profile}", show_header=False)
     table.add_column("Property", style="cyan")
     table.add_column("Value")