h5pack 1.1.0__py3-none-any.whl

h5pack/cli/checksum.py

@@ -0,0 +1,126 @@
+import os
+from time import perf_counter
+from argparse import Namespace
+from ..core.io import (
+    add_extension,
+    get_dir_files
+)
+from ..core.guards import is_file_with_ext
+from ..core.display import (
+    exit_error,
+    exit_warning,
+    print_warning,
+    print_error,
+)
+from ..core.utils import (
+    get_file_checksum,
+    time_to_str
+)
+
+
+def cmd_checksum(args: Namespace) -> None:
+    """Calculates or verifies checksums associated with `.h5` files.
+
+    Args:
+        args (Namespace): User input arguments provided through the console.
+    """
+    # Check input type
+    if is_file_with_ext(file=args.input, ext=".sha256"):  # Verify
+        if args.save:
+            print_warning("--save ignored for .sha256 input")     
+        
+         # Read lines and check they contain only two elements
+        root_dir = os.path.dirname(args.input)
+        print(f"Using root folder '{os.path.abspath(root_dir)}'")
+        print(f"Verifying checksum in '{args.input}' ...")
+
+        start_time = perf_counter()
+
+        with open(args.input, "r") as f:
+            for line_idx, line in enumerate(f):
+                h5_filename, saved_checksum = line.split("\t")
+                h5_file = os.path.join(root_dir, h5_filename)
+                saved_checksum = saved_checksum.rstrip("\n")
+
+                if not is_file_with_ext(h5_file, ext=".h5"):
+                    exit_error(
+                        f"Invalid file '{h5_file}' reference in '{args.input}'"
+                        f" (line {line_idx + 1})"
+                    )
+                
+                checksum = get_file_checksum(h5_file, hash="sha256")
+
+                if saved_checksum == checksum:
+                    print(
+                        f"{h5_filename}\t{saved_checksum} [OK]"
+                    )
+                
+                else:
+                    print_error(
+                        f"{h5_filename} [MISMATCH]"
+                        f"\n  - Saved:      {saved_checksum}"
+                        f"\n  - Calculated: {checksum}"
+                    )
+                
+        end_time = perf_counter()
+        elapsed_time_repr = time_to_str(end_time - start_time, abbrev=False)
+        print(f"Checksum verification completed in {elapsed_time_repr}")
+    
+    else:  # Calculate
+        if is_file_with_ext(args.input, ext=".h5"):
+            all_files = [args.input]
+        
+        elif os.path.isdir(args.input):
+            all_files = get_dir_files(
+                args.input,
+                ext=".h5",
+                recursive=args.recursive
+            )
+        
+        else:
+            exit_error(
+                "Input must be an .h5 file or a folder containing .h5 files"
+            )
+        
+        if len(all_files) == 0:
+            if not args.recursive:
+                exit_warning(
+                    f"0 .h5 files found in in '{args.input}'. Use "
+                    "--recursive/-r if you intended to perform a recursive "
+                    "search"
+                )
+            
+            else:
+                exit_warning(f"0 .h5 files found in '{args.input}'")
+
+        print(f"Calculating checksum for .h5 files in '{args.input}' ...")
+
+        if args.save:
+            checksum_file = add_extension(args.save, ext=".sha256")
+
+            with open(add_extension(args.save, ext=".sha256"), "w") as f:
+                start_time = perf_counter()
+
+                for file in all_files:
+                    checksum = get_file_checksum(file, hash="sha256")
+                    checksum_repr = f"{os.path.basename(file)}\t{checksum}"
+                    f.write(f"{checksum_repr}\n")
+                    print(checksum_repr)
+                
+                end_time = perf_counter()
+                elapsed_time_repr = time_to_str(end_time - start_time)
+                print(
+                    f"Checksum calculation completed in {elapsed_time_repr} "
+                    f"and saved to '{checksum_file}'"
+                )
+        
+        else:
+            start_time = perf_counter()
+
+            for file in all_files:
+                checksum = get_file_checksum(file, hash="sha256")
+                print(f"{os.path.basename(file)}\t{checksum}")
+             
+            end_time = perf_counter()
+            elapsed_time_repr = time_to_str(end_time - start_time)
+            print(f"Checksum calculation completed in {elapsed_time_repr}")

h5pack/cli/info.py

@@ -0,0 +1,83 @@
+import os
+import h5py
+from argparse import Namespace
+from ..core.guards import is_file_with_ext
+from ..core.display import (
+    exit_error,
+    print_warning
+)
+
+
+def cmd_info(args: Namespace) -> None:
+    """Inspects a `.h5` file generated with `h5pack`.
+    
+    Args:
+        args (Namespace): Input user arguments provided through the console.
+    """
+    # Check file exists
+    if not is_file_with_ext(args.input, ext=".h5"):
+        exit_error(f"Invalid input file '{args.input}'")
+
+    # Print info
+    print(f"Input file: '{args.input}'" )
+
+    with h5py.File(args.input, "r") as h5_file:
+        # Check if producer is h5pack, otherwise will not be correctly parsed
+        if not h5_file.attrs.get("producer", "").startswith("h5pack"):
+            exit_error(
+                "This file was not created using h5pack, so it may not be "
+                "formatted as expected and cannot be reliably parsed"
+            )
+        
+        if len(h5_file.attrs) > 0:
+            print("File attribute(s):")
+            key_ljust = max([len(k) for k in h5_file.attrs]) + 6
+
+            for k, v in h5_file.attrs.items():
+                print(f"  - {k}:".ljust(key_ljust) + f"{v}")
+        
+        # Get data group level attributes
+        for data_group_name, data_group_data in h5_file.items():
+            print(f"Data group '{data_group_name}':")
+
+            if len(data_group_data.attrs) > 0:
+                key_ljust = max([len(k) for k in data_group_data.attrs]) + 6
+                
+                print(f"'{data_group_name}' attribute(s):")
+
+                for k, v in data_group_data.items():
+                    print(f"  - {k}:".ljust(key_ljust) + f"{v}") 
+
+            # Get dataset level info
+            for dataset_name, dataset_data in data_group_data.items():
+                if len(dataset_data.attrs) > 0:
+                    key_ljust = max([len(k) for k in dataset_data.attrs]) + 8
+                    print(f"  - '{dataset_name}' attribute(s):")
+                
+                for k, v in dataset_data.attrs.items():
+                    print(f"    - {k}:".ljust(key_ljust) + f"{v}") 
+
+                print(f"  - '{dataset_name}' data attribute(s):")
+                print(f"    - shape: {dataset_data.shape}")
+                print(f"    - dtype: {dataset_data.dtype}")
+
+        # If virtual, check paths are accesible and report broken paths
+        if h5_file.attrs.get("is_virtual") is not None:
+            print("Virtual dataset sources:")
+
+            for data_group_name, data_group_data in h5_file.items():
+                for dataset_name, dataset_data in data_group_data.items():
+                    plist = dataset_data.id.get_create_plist()
+
+                    print(f"  - '{dataset_name}' source(s):")
+
+                    for file_idx in range(plist.get_virtual_count()):
+                        file = plist.get_virtual_filename(file_idx)
+
+                        if not os.path.isfile(file):
+                            print_warning(
+                                f"    - {file_idx}: {file} [NOT FOUND]"
+                            )
+                        
+                        else:
+                            print(f"    - {file_idx}: {file} [OK]")

h5pack/cli/main.py

@@ -0,0 +1,234 @@
+import sys
+import argparse
+from datetime import datetime
+from importlib.metadata import version
+from .checksum import cmd_checksum
+from .info import cmd_info
+from .pack import cmd_pack
+from .unpack import cmd_unpack
+from .virtual import cmd_virtual
+
+
+def get_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="pack/unpack/inspect hierarchical data format datasets",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    subparser = parser.add_subparsers(dest="action")
+
+    # Pack parser
+    pack_parser = subparser.add_parser(
+        "pack",
+        description="pack data into HDF5 dataset files",
+        help="pack data into HDF5 dataset files",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    pack_parser.add_argument(
+        "-c", "--config",
+        type=str,
+        default="h5pack.yaml",
+        help=".yaml configuration file containing dataset specifications"
+    )
+    pack_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        required=True,
+        help="output HDF5 partition file(s) with .h5 extension"
+    ) 
+    pack_partitions_parser = pack_parser.add_mutually_exclusive_group()
+    pack_partitions_parser.add_argument(
+        "-p", "--partitions",
+        type=int,
+        default=1,
+        help="number of partitions to create"
+    )
+    pack_parser.add_argument(
+        "-f", "--files-per-partition",
+        type=int,
+        help="number of files per partition"
+    )
+    pack_parser.add_argument(
+        "-d", "--dataset",
+        type=str,
+        required=True,
+        help="name of the dataset to generate"
+    )
+    pack_parser.add_argument(
+        "--create-virtual",
+        action="store_true",
+        help="create a virtual layout when two or more partitions are created"
+    )
+    pack_parser.add_argument(
+        "--skip-validation",
+        action="store_true",
+        help="skip validating files before generating the partition(s)"
+    )
+    pack_parser.add_argument(
+        "--skip-checksum",
+        action="store_true",
+        help="skip generating the checksum file"
+    )
+    pack_parser.add_argument(
+        "--overwrite",
+        action="store_true",
+        help="allow overwriting existing files"
+    )
+    pack_parser.add_argument(
+        "-w", "--workers",
+        type=int,
+        default=1,
+        help="number of workers (0 means 1 worker per core)"
+    )
+    pack_parser.add_argument(
+        "-u", "--unattended",
+        action="store_true",
+        help="unattended mode (no user prompts)"
+    )
+
+    # Unpack parser
+    unpack_parser = subparser.add_parser(
+        "unpack",
+        description="unpack HDF5 datasets into individual files",
+        help="unpack HDF5 datasets datasets into individual files",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    unpack_parser.add_argument(
+        "input",
+        help="input .h5 file"
+    )
+    unpack_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        help="output folder"
+    )
+
+    # Virtual parser
+    virtual_parser = subparser.add_parser(
+        "virtual",
+        description="create virtual HDF5 datasets",
+        help="create virtual HDF5 datasets",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    virtual_parser.add_argument(
+        "input",
+        type=str,
+        nargs="+",
+        help="input .h5 file(s) or folder(s) containing .h5 file(s)"
+    )
+    virtual_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        required=True,
+        help="output HDF5 file(s) with .h5 extension"
+    )
+    virtual_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="search folders recursively"
+    )
+    virtual_parser.add_argument(
+        "-a", "--attrs",
+        type=str,
+        nargs="+",
+        metavar="KEY VALUE",
+        help="top level attributes to write as a list of 'key' 'value' pairs"
+    )
+    virtual_pattr_parser = virtual_parser.add_mutually_exclusive_group()
+    virtual_pattr_parser.add_argument(
+        "-s", "--select",
+        type=str,
+        metavar="PATTERN",
+        help="select pattern include matching elements from --input"
+    )
+    virtual_pattr_parser.add_argument(
+        "-f", "--filter",
+        type=str,
+        metavar="PATTERN",
+        help="filter pattern to remove matching elements from --input"
+    )
+    virtual_parser.add_argument(
+        "--force-abspath",
+        action="store_true",
+        help="force all embedded paths to be absolute paths"
+    )
+    virtual_parser.add_argument(
+        "-u", "--unattended",
+        action="store_true",
+        help="unattended mode (no user prompts)"
+    )
+
+    # Info parser
+    info_parser = subparser.add_parser(
+        "info",
+        description="inspect HDF5 datasets",
+        help="inspect HDF5 datasets",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    info_parser.add_argument(
+        "input",
+        help="input .h5 file"
+    ) 
+
+    # Checksum parser
+    checksum_parser = subparser.add_parser(
+        "checksum",
+        help="create/verify virtual HDF5 datasets checksum",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False
+    )
+    checksum_parser.add_argument(
+        "input",
+        type=str,
+        help=".sha256 file (to verify) or file or folder (to calculate)"
+    )
+    checksum_parser.add_argument(
+        "--save",
+        type=str,
+        help="save calculated checksum to a .sha256 file"
+    )
+    checksum_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="search folders recursively if input is a folder"
+    )
+
+    return parser
+
+
+def main() -> int:
+    if (
+        len(sys.argv) == 1
+        or (len(sys.argv) == 2 and sys.argv[1] in ("--version"))
+    ):
+        print(
+            f"h5pack version {version('h5pack')} 2024-{datetime.now().year} "
+            "developed by Esteban Gómez (Speech Interaction Technology, Aalto "
+            "University)"
+        )
+        sys.exit(0)
+    
+    parser = get_parser()
+    args = parser.parse_args()
+
+    if args.action == "pack":
+        cmd_pack(args)
+    
+    elif args.action == "virtual":
+        cmd_virtual(args)
+    
+    elif args.action == "checksum":
+        cmd_checksum(args)
+    
+    elif args.action == "info":
+        cmd_info(args)
+    
+    elif args.action == "unpack":
+        cmd_unpack(args)
+    
+    else:
+        raise AssertionError