hdfmap 0.4__py3-none-any.whl

hdfmap/__init__.py

@@ -0,0 +1,76 @@
+"""
+hdfmap
+Map objects within an HDF file and create a dataset namespace.
+
+--- Usage ---
+# HdfMap from NeXus file:
+from hdfmap import create_nexus_map, load_hdf
+hmap = create_nexus_map('file.nxs')
+with load_hdf('file.nxs') as nxs:
+    address = hmap.get_address('energy')
+    energy = nxs[address][()]
+    string = hmap.format_hdf(nxs, "the energy is {energy:.2f} keV")
+    d = hmap.get_data_block(nxs)  # classic data table, d.scannable, d.metadata
+
+# Shortcuts - single file reloading class
+from hdfmap import NexusLoader
+scan = NexusLoader('file.nxs')
+[data1, data2] = scan.get_data(['dataset_name_1', 'dataset_name_2'])
+data = scan.eval('dataset_name_1 * 100 + 2')
+string = scan.format('my data is {dataset_name_1:.2f}')
+
+# Shortcuts - multifile load data
+from hdfmap import hdf_data, hdf_eval, hdf_format, hdf_image
+all_data = hdf_data([f"file{n}.nxs" for n in range(100)], 'dataset_name')
+normalised_data = hdf_eval(filenames, 'total / Transmission / (rc / 300.)')
+descriptions = hdf_eval(filenames, 'Energy: {en:5.3f} keV')
+image = hdf_image(filenames, index=31)
+
+
+By Dr Dan Porter
+Diamond Light Source Ltd
+2024
+"""
+
+from .logging import set_all_logging_level
+from .hdfmap_class import HdfMap
+from .nexus import NexusMap
+from .file_functions import list_files, load_hdf, create_hdf_map, create_nexus_map
+from .file_functions import hdf_data, hdf_image, hdf_eval, hdf_format, nexus_data_block
+from .reloader_class import HdfLoader, NexusLoader
+
+
+__all__ = [
+    HdfMap, NexusMap, load_hdf, create_hdf_map, create_nexus_map,
+    hdf_data, hdf_image, hdf_eval, hdf_format, nexus_data_block, HdfLoader, NexusLoader,
+    set_all_logging_level
+]
+
+__version__ = "0.4.0"
+__date__ = "2024/08/16"
+
+
+def version_info() -> str:
+    return 'hdfmap version %s (%s)' % (__version__, __date__)
+
+
+def module_info() -> str:
+    import sys
+    out = 'Python version %s' % sys.version
+    out += '\n at: %s' % sys.executable
+    out += '\n %s: %s' % (version_info(), __file__)
+    # Modules
+    import numpy
+    out += '\n     numpy version: %s' % numpy.__version__
+    import h5py
+    out += '\n      h5py version: %s' % h5py.__version__
+    # import imageio
+    # out += '\n   imageio version: %s' % imageio.__version__
+    try:
+        import hdf5plugin
+        out += '\n    hdf5plugin: %s' % hdf5plugin.version
+    except ImportError:
+        out += '\n    hdf5plugin: None'
+    import os
+    out += '\nRunning in directory: %s\n' % os.path.abspath('.')
+    return out

hdfmap/eval_functions.py

@@ -0,0 +1,172 @@
+"""
+hdf eval functions
+"""
+
+import os
+import ast
+import builtins
+import datetime
+import re
+import typing
+import numpy as np
+import h5py
+
+from .logging import create_logger
+
+# parameters
+GLOBALS = {'np': np}
+GLOBALS_NAMELIST = dir(builtins) + list(GLOBALS.keys())
+logger = create_logger(__name__)
+# regex patterns
+special_characters = re.compile(r'\W')  # finds all special non-alphanumberic characters
+long_floats = re.compile(r'\d+\.\d{5,}')  # finds floats with long trailing decimals
+
+
+def expression_safe_name(string: str, replace: str = '_') -> str:
+    """
+    Returns an expression safe name
+    :param string: any string
+    :param replace: str replace special characters with this
+    :return: string with special characters replaced
+    """
+    return special_characters.sub('_', string)
+
+
+def round_string_floats(string):
+    """
+    Shorten string by removing long floats
+    :param string: string, e.g. '#810002 scan eta 74.89533603616637 76.49533603616636 0.02 pil3_100k 1 roi2'
+    :return: shorter string, e.g. '#810002 scan eta 74.895 76.495 0.02 pil3_100k 1 roi2'
+    """
+    def subfun(m):
+        return str(round(float(m.group()), 3))
+    return long_floats.sub(subfun, string)
+
+
+def dataset2data(dataset: h5py.Dataset, index: int | slice = (), direct_load=False) -> datetime.datetime | str | np.ndarray:
+    """
+    Read the data from a h5py Dataset and convert to either datetime, str or squeezed numpy array
+    :param dataset: h5py.Dataset containing data
+    :param index: index of array (not used if dataset is string/ bytes type)
+    :param direct_load: loads the data directly without conversion if True
+    :return datetime.datetime: if data is an isoformat string, returns datetime object
+    :return str: if data is another string, returns str with long floats rounded
+    :return np.ndarray: if data is another numeric object
+    """
+    if direct_load:
+        return dataset[index]
+    if np.issubdtype(dataset, np.number):
+        return np.squeeze(dataset[index])  # numeric np.ndarray
+    try:
+        # timestamp -> datetime64 -> datetime
+        timestamp = np.squeeze(dataset[index]).astype(np.datetime64).astype(datetime.datetime)
+        # single datetime obj vs array of datetime obj
+        return timestamp[()] if timestamp.ndim == 0 else timestamp
+    except ValueError:
+        try:
+            string_dataset = dataset.asstr()[()]
+            if dataset.ndim == 0:
+                return round_string_floats(string_dataset)  # bytes or str -> str
+            return string_dataset  # str array
+        except ValueError:
+            return np.squeeze(dataset[index])  # other np.ndarray
+
+
+def check_unsafe_eval(eval_str: str) -> None:
+    """
+    Check str for naughty eval arguments such as sys, os or import
+    This is not foolproof.
+    :param eval_str: str
+    :return: pass or raise error
+    """
+    bad_names = ['import', 'os.', 'sys.', 'open(', 'eval', 'exec']
+    for bad in bad_names:
+        if bad in eval_str:
+            raise Exception('This operation is not allowed as it contains: "%s"' % bad)
+
+
+def find_identifiers(expression: str) -> list[str]:
+    """Returns list of variable names in expression, ommiting builtins and globals"""
+    # varnames = re.findall(r'[a-zA-Z]\w*', expression)
+    return [node.id for node in ast.walk(ast.parse(expression, mode='eval'))
+            if type(node) is ast.Name and node.id not in GLOBALS_NAMELIST]
+
+
+def extra_hdf_data(hdf_file: h5py.File) -> dict:
+    """Extract filename, filepath and other additional data fom hdf file"""
+    filepath = getattr(hdf_file, 'filename', 'unknown')
+    return {
+        'filepath': filepath,
+        'filename': os.path.basename(filepath),
+    }
+
+
+def generate_namespace(hdf_file: h5py.File, hdf_namespace: dict[str, str], identifiers: list[str] | None = None,
+                       default: typing.Any = np.array('--')) -> dict[str, typing.Any]:
+    """
+    Generate namespace dict - create a dictionary linking the name of a dataset to the dataset value
+
+    Adds additional values if not in name_path dict:
+        filename: str, name of hdf_file
+        filepath: str, full path of hdf_file
+        _*name*: str hdf path of *name*
+
+    :param hdf_file: h5py.File object
+    :param hdf_namespace: locations of data in hdf file, dict[identifier]='/hdf/dataset/path'
+    :param identifiers: list of names to load from hdf_file, if None, use generate all items in name_path
+    :param default: any, if varname not in name_path - return default instead
+    :return: dict {'name': value, '_name': '/hdf/path'}
+    """
+    if identifiers is None:
+        identifiers = list(hdf_namespace.keys())
+    # TODO: add ROI commands e.g. nroi[1,2,3,4] -> default_image([1,2,3,4])
+    # TODO: add name@attribute e.g. incident_energy@units -> 'eV'
+    # TODO: add name.label e.g. axes.label -> 'eta [Deg]'
+    namespace = {
+        name: dataset2data(hdf_file[hdf_namespace[name]])
+        for name in identifiers if name in hdf_namespace and hdf_namespace[name] in hdf_file
+    }
+    defaults = {
+        name: default
+        for name in identifiers if (name not in hdf_namespace) or (hdf_namespace[name] not in hdf_file)
+    }
+    hdf_paths = {name: hdf_namespace[name[1:]] for name in identifiers
+                 if name.startswith('_') and name[1:] in hdf_namespace}
+    # add extra params
+    extras = extra_hdf_data(hdf_file)
+    return {**defaults, **extras, **hdf_paths, **namespace}
+
+
+def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str]) -> typing.Any:
+    """
+    Evaluate an expression using the namespace of the hdf file
+    :param hdf_file: h5py.File object
+    :param expression: str expression to be evaluated
+    :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'}
+    :return: eval(expression)
+    """
+    if expression in hdf_file:
+        return dataset2data(hdf_file[expression])
+    check_unsafe_eval(expression)
+    # find identifiers matching names in the namespace
+    identifiers = [name for name in hdf_namespace if name in special_characters.split(expression)]
+    # find other non-builtin identifiers
+    identifiers += [name for name in find_identifiers(expression) if name not in identifiers]
+    namespace = generate_namespace(hdf_file, hdf_namespace, identifiers)
+    logger.info(f"Expression: {expression}\nidentifiers: {identifiers}\n")
+    logger.debug(f"namespace: {namespace}\n")
+    return eval(expression, GLOBALS, namespace)
+
+
+def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str]) -> str:
+    """
+    Evaluate a formatted string expression using the namespace of the hdf file
+    :param hdf_file: h5py.File object
+    :param expression: str expression using {name} format specifiers
+    :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'}
+    :return: eval_hdf(f"expression")
+    """
+    expression = 'f"""' + expression + '"""'  # convert to fstr
+    return eval_hdf(hdf_file, expression, hdf_namespace)
+
+

hdfmap/file_functions.py

@@ -0,0 +1,210 @@
+import os
+import h5py
+import numpy as np
+
+from .logging import create_logger
+from .hdfmap_class import HdfMap
+from .nexus import NexusMap
+
+
+EXTENSIONS = ['.nxs', '.hdf', '.hdf5', '.h5']
+DEFAULT_EXTENSION = EXTENSIONS[0]
+DEFAULT_HDF_PATH = "entry1/scan_command"
+logger = create_logger(__name__)
+
+
+def load_hdf(hdf_filename: str) -> h5py.File:
+    """Load hdf file, return h5py.File object"""
+    return h5py.File(hdf_filename, 'r')
+
+
+def list_files(folder_directory: str, extension=DEFAULT_EXTENSION) -> list[str]:
+    """Return list of files in directory with extension, returning list of full file paths"""
+    try:
+        return sorted(
+            (file.path for file in os.scandir(folder_directory) if file.is_file() and file.name.endswith(extension)),
+            key=lambda x: os.path.getmtime(x)
+        )
+    except FileNotFoundError:
+        return []
+
+
+def as_str_list(string: str | list[str]) -> list[str]:
+    """
+    Helper function to convert str or list of str to list of str
+    :param string: str, byteString, list, array
+    :return: list of str
+    """
+    return list(np.asarray(string, dtype=str).reshape(-1))
+
+
+def create_hdf_map(hdf_filename: str) -> HdfMap:
+    """
+    Create a HdfMap from a hdf file
+    :param hdf_filename: str filename of hdf file
+    :return: HdfMap
+    """
+    with load_hdf(hdf_filename) as hdf:
+        hdf_map = HdfMap(hdf)
+    return hdf_map
+
+
+def create_nexus_map(hdf_filename: str, groups: None | list[str] = None,
+                     default_entry_only: bool = False) -> NexusMap:
+    """
+    Create a HdfMap from a NeXus file, loading default parameters and allowing a reduced, single entry map
+    :param hdf_filename: str filename of hdf file
+    :param groups: list of groups to collect datasets from
+    :param default_entry_only: if True, only the first or default entry will be loaded
+    :return: NexusMap
+    """
+    hdf_map = NexusMap()
+    with load_hdf(hdf_filename) as hdf:
+        hdf_map.populate(hdf, groups=groups, default_entry_only=default_entry_only)
+    if not hdf_map.scannables:
+        print('NXdata not found, getting scannables from most common array size')
+        size = hdf_map.most_common_size()
+        hdf_map.generate_scannables(size)
+    return hdf_map
+
+
+def hdf_data(filenames: str | list[str], name_or_path: str | list[str], hdf_map: HdfMap = None,
+             index=(), default=None, fixed_output=False):
+    """
+    General purpose function to retrieve data from HDF files
+    :param filenames: str or list of str - file paths
+    :param name_or_path: str or list of str - names or paths of HDF datasets
+    :param hdf_map: HdfMap object, or None to generate from first file
+    :param index: dataset index or slice
+    :param default: value to give if dataset doesn't exist in file
+    :param fixed_output: if True, always returns list of list
+    :return if single file, single dataset: single value
+    :return if multi file or multi dataset: list, len(filenames) or len(name_or_path)
+    :return if multi file and multi dataset: list[files: list[names]]
+    """
+    # cast as 1D arrays
+    filenames = as_str_list(filenames)
+    name_or_path = as_str_list(name_or_path)
+    # generate hdf_map
+    if hdf_map is None:
+        hdf_map = create_hdf_map(filenames[0])
+    out = []
+    for filename in filenames:
+        logger.info(f"\nHDF file: {filename}")
+        with load_hdf(filename) as hdf:
+            out.append([hdf_map.get_data(hdf, name, index=index, default=default) for name in name_or_path])
+    if fixed_output:
+        return out
+    if len(filenames) == 1 and len(name_or_path) == 1:
+        return out[0][0]
+    if len(filenames) == 1 and len(name_or_path) > 1:
+        return out[0]
+    if len(name_or_path) == 1:
+        return [val[0] for val in out]
+    return out
+
+
+def hdf_eval(filenames: str | list[str], expression: str, hdf_map: HdfMap = None, fixed_output=False):
+    """
+    Evaluate expression using dataset names
+    :param filenames: str or list of str - file paths
+    :param expression: str expression to evaluate in each file, e.g. "roi2_sum / Transmission"
+    :param hdf_map: HdfMap object, or None to generate from first file
+    :param fixed_output: if True, always returns list len(filenames)
+    :return if single file: single output
+    :return if multi file: list, len(filenames)
+    """
+    # cast as 1D arrays
+    filenames = as_str_list(filenames)
+    # generate hdf_map
+    if hdf_map is None:
+        hdf_map = create_hdf_map(filenames[0])
+    out = []
+    for filename in filenames:
+        logger.info(f"\nHDF file: {filename}")
+        with load_hdf(filename) as hdf:
+            out.append(hdf_map.eval(hdf, expression))
+    if not fixed_output and len(filenames) == 1:
+        return out[0]
+    return out
+
+
+def hdf_format(filenames: str | list[str], expression: str, hdf_map: HdfMap = None, fixed_output=False):
+    """
+    Evaluate string format expression using dataset names
+    :param filenames: str or list of str - file paths
+    :param expression: str expression to evaluate in each file, e.g. "the energy is {en:.2f} keV"
+    :param hdf_map: HdfMap object, or None to generate from first file
+    :param fixed_output: if True, always returns list len(filenames)
+    :return if single file: single output
+    :return if multi file: list, len(filenames)
+    """
+    # cast as 1D arrays
+    filenames = as_str_list(filenames)
+    # generate hdf_map
+    if hdf_map is None:
+        hdf_map = create_hdf_map(filenames[0])
+    out = []
+    for filename in filenames:
+        logger.info(f"\nHDF file: {filename}")
+        with load_hdf(filename) as hdf:
+            out.append(hdf_map.format_hdf(hdf, expression))
+    if not fixed_output and len(filenames) == 1:
+        return out[0]
+    return out
+
+
+def hdf_image(filenames: str | list[str], index: slice = None, hdf_map: HdfMap = None, fixed_output=False):
+    """
+    Evaluate string format expression using dataset names
+    :param filenames: str or list of str - file paths
+    :param index: index or slice of dataset volume, or None to use middle index
+    :param hdf_map: HdfMap object, or None to generate from first file
+    :param fixed_output: if True, always returns list len(filenames)
+    :return if single file: single output - numpy array
+    :return if multi file: list, len(filenames)
+    """
+    # cast as 1D arrays
+    filenames = as_str_list(filenames)
+    # generate hdf_map
+    if hdf_map is None:
+        hdf_map = create_hdf_map(filenames[0])
+    out = []
+    for filename in filenames:
+        logger.info(f"\nHDF file: {filename}")
+        with load_hdf(filename) as hdf:
+            out.append(hdf_map.get_image(hdf, index=index))
+    if not fixed_output and len(filenames) == 1:
+        return out[0]
+    return out
+
+
+def nexus_data_block(filenames: str | list[str], hdf_map: HdfMap = None, fixed_output=False):
+    """
+    Create classic dict like dataloader objects from nexus files
+    E.G.
+        d = nexus_data_block('filename')
+        d.scannable -> array
+        d.metadata.filename -> value
+        d.keys() -> list of items
+
+    :param filenames: str or list of str - file paths
+    :param hdf_map: HdfMap object, or None to generate from first file
+    :param fixed_output: if True, always returns list len(filenames)
+    :return if single file: single output - dict like DataObject
+    :return if multi file: list, len(filenames)
+    """
+    # cast as 1D arrays
+    filenames = as_str_list(filenames)
+    # generate hdf_map
+    if hdf_map is None:
+        hdf_map = create_nexus_map(filenames[0])
+    out = []
+    for filename in filenames:
+        logger.info(f"\nHDF file: {filename}")
+        with load_hdf(filename) as hdf:
+            out.append(hdf_map.get_dataholder(hdf))
+    if not fixed_output and len(filenames) == 1:
+        return out[0]
+    return out
+