pyTEMlib 0.2020.11.0__py3-none-any.whl → 0.2024.8.4__py3-none-any.whl

pyTEMlib/nion_reader.py

@@ -1,293 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: iso-8859-1 -*-
-
-################################################################################
-# Python class for reading Nion Swift files into sidpy Dataset
-# and extracting all metadata
-#
-# Written by Gerd Duscher, UTK 2020
-#
-# Works for python 3
-#
-################################################################################
-from __future__ import division, print_function, absolute_import, unicode_literals
-
-import json
-import struct
-import h5py
-from warnings import warn
-import sys
-import numpy as np
-import os
-
-
-import sidpy
-
-__all__ = ["NionReader", "version"]
-
-version = '0.1beta'
-
-debugLevel = 0  # 0=none, 1-3=basic, 4-5=simple, 6-10 verbose
-
-if sys.version_info.major == 3:
-    unicode = str
-
-# ### utility functions ###
-
-
-def parse_zip(fp):
-    """
-        Parse the zip file headers at fp
-        :param fp: the file pointer from which to parse the zip file
-        :return: A tuple of local files, directory headers, and end of central directory
-        The local files are dictionary where the keys are the local file offset and the
-        values are each a tuple consisting of the name, data position, data length, and crc32.
-        The directory headers are a dictionary where the keys are the names of the files
-        and the values are a tuple consisting of the directory header position, and the
-        associated local file position.
-        The end of central directory is a tuple consisting of the location of the end of
-        central directory header and the location of the first directory header.
-        This method will seek to location 0 of fp and leave fp at end of file.
-
-        This function is copied from  nionswift/nion/swift/model/NDataHandler.py
-
-    """
-    local_files = {}
-    dir_files = {}
-    eocd = None
-    fp.seek(0)
-    while True:
-        pos = fp.tell()
-        signature = struct.unpack('I', fp.read(4))[0]
-        if signature == 0x04034b50:
-            fp.seek(pos + 14)
-            crc32 = struct.unpack('I', fp.read(4))[0]
-            fp.seek(pos + 18)
-            data_len = struct.unpack('I', fp.read(4))[0]
-            fp.seek(pos + 26)
-            name_len = struct.unpack('H', fp.read(2))[0]
-            extra_len = struct.unpack('H', fp.read(2))[0]
-            name_bytes = fp.read(name_len)
-            fp.seek(extra_len, os.SEEK_CUR)
-            data_pos = fp.tell()
-            fp.seek(data_len, os.SEEK_CUR)
-            local_files[pos] = (name_bytes, data_pos, data_len, crc32)
-        elif signature == 0x02014b50:
-            fp.seek(pos + 28)
-            name_len = struct.unpack('H', fp.read(2))[0]
-            extra_len = struct.unpack('H', fp.read(2))[0]
-            comment_len = struct.unpack('H', fp.read(2))[0]
-            fp.seek(pos + 42)
-            pos2 = struct.unpack('I', fp.read(4))[0]
-            name_bytes = fp.read(name_len)
-            fp.seek(pos + 46 + name_len + extra_len + comment_len)
-            dir_files[name_bytes] = (pos, pos2)
-        elif signature == 0x06054b50:
-            fp.seek(pos + 16)
-            pos2 = struct.unpack('I', fp.read(4))[0]
-            eocd = (pos, pos2)
-            break
-        else:
-            raise IOError()
-    return local_files, dir_files, eocd
-
-
-class NionReader(sidpy.Reader):
-
-    def __init__(self, file_path, verbose=False):
-        """
-        file_path: filepath to dm3 file.
-        """
-
-        super().__init__(file_path)
-
-        # initialize variables ##
-        self.verbose = verbose
-        self.__filename = file_path
-
-        path, file_name = os.path.split(self.__filename)
-        self.basename, self.extension = os.path.splitext(file_name)
-        self.data_cube = None
-        self.original_metadata = {}
-        self.dimensions = []
-        if self.extension == '.ndata':
-
-            # - open file for reading
-            try:
-                self.__f = open(self.__filename, "rb")
-            except FileNotFoundError:
-                raise FileNotFoundError('File not found')
-            try:
-                local_files, dir_files, eocd = parse_zip(self.__f)
-            except IOError:
-                raise IOError("File {} does not seem to be of Nion`s .ndata format".format(self.__filename))
-            self.__f.close()
-        elif self.extension == '.h5':
-            try:
-                fp = h5py.File(self.__filename, mode='a')
-                if 'data' not in fp:
-                    raise IOError("File {} does not seem to be of Nion`s .h5 format".format(self.__filename))
-                fp.close()
-            except IOError:
-                raise IOError("File {} does not seem to be of Nion`s .h5 format".format(self.__filename))
-
-    def read(self):
-        if self.extension == '.ndata':
-            try:
-                self.__f = open(self.__filename, "rb")
-            except FileNotFoundError:
-                raise FileNotFoundError('File not found')
-            local_files, dir_files, eocd = parse_zip(self.__f)
-
-            contains_data = b"data.npy" in dir_files
-            contains_metadata = b"metadata.json" in dir_files
-            file_count = contains_data + contains_metadata  # use fact that True is 1, False is 0
-
-            self.__f.seek(local_files[dir_files[b"data.npy"][1]][1])
-
-            self.data_cube = np.load(self.__f)
-
-            json_pos = local_files[dir_files[b"metadata.json"][1]][1]
-            json_len = local_files[dir_files[b"metadata.json"][1]][2]
-            self.__f.seek(json_pos)
-            json_properties = self.__f.read(json_len)
-
-            self.original_metadata = json.loads(json_properties.decode("utf-8"))
-            self.__f.close()
-        elif self.extension == '.h5':
-            # TODO: use lazy load for large datasets
-            self.__f = h5py.File(self.__filename, 'a')
-            if 'data' in self.__f:
-                json_properties = self.__f['data'].attrs.get("properties", "")
-                self.data_cube = self.__f['data'][:]
-                self.original_metadata = json.loads(json_properties)
-
-        self.get_dimensions()
-        ## Need to switch image dimensions in Nion format
-        image_dims = []
-        for dim, axis in enumerate(self.dimensions):
-            print(dim, axis)
-            if axis.dimension_type == sidpy.DimensionTypes.SPATIAL:
-                image_dims.append(dim)
-        print('image_dims', image_dims)
-        print(self.data_cube.shape)
-        if len(image_dims) == 2:
-            self.data_cube = np.swapaxes(self.data_cube, image_dims[0], image_dims[1])
-            temp = self.dimensions[image_dims[0]].copy()
-            self.dimensions[image_dims[0]] = self.dimensions[image_dims[1]].copy()
-            self.dimensions[image_dims[1]] = temp
-
-        dataset = sidpy.Dataset.from_array(self.data_cube)
-
-        for dim, axis in enumerate(self.dimensions):
-            dataset.set_dimension(dim, axis)
-
-        dataset.original_metadata = self.original_metadata
-        if 'dimensional_calibrations' in dataset.original_metadata:
-            for dim in dataset.original_metadata['dimensional_calibrations']:
-                if dim['units'] == '':
-                    dim['units'] = 'pixels'
-
-        dataset.quantity = 'intensity'
-        dataset.units = 'counts'
-        if 'description' in dataset.original_metadata:
-            dataset.title = dataset.original_metadata['description']['title']
-        else:
-            if 'title' in dataset.original_metadata:
-                dataset.title = dataset.original_metadata['title']
-            else:
-                path, file_name = os.path.split(self.__filename)
-                basename, extension = os.path.splitext(file_name)
-                dataset.title = basename
-
-        if 'data_source' in dataset.original_metadata:
-            dataset.source = dataset.original_metadata['data_source']
-        else:
-            dataset.source = 'NionReader'
-
-        self.set_data_type(dataset)
-        dataset.modality = 'STEM data'
-        dataset.h5_dataset = None
-
-        return dataset
-
-    def set_data_type(self, dataset):
-
-        spectral_dim = False
-        for axis in dataset._axes.values():
-            if axis.dimension_type == sidpy.DimensionTypes.SPECTRAL:
-                spectral_dim = True
-
-        if len(dataset.shape) > 3:
-            raise NotImplementedError('Data_type not implemented yet')
-        elif len(dataset.shape) == 3:
-            if spectral_dim:
-                dataset.data_type = 'spectrum_image'
-            else:
-                dataset.data_type = 'IMAGE_STACK'
-                for dim, axis in dataset._axes.items():
-                    if axis.dimension_type != sidpy.DimensionTypes.SPATIAL:
-                        dataset.set_dimension(dim, sidpy.Dimension(axis.values,
-                                                                   name='frame',
-                                                                   units='frame',
-                                                                   quantity='stack',
-                                                                   dimension_type=sidpy.DimensionTypes.TEMPORAL))
-                        break
-
-        elif len(dataset.shape) == 2:
-            if spectral_dim:
-                dataset.data_type = sidpy.DataTypes.SPECTRAL_IMAGE
-            else:
-                dataset.data_type = sidpy.DataTypes.IMAGE
-        elif len(dataset.shape) == 1:
-            if spectral_dim:
-                dataset.data_type = sidpy.DataTypes.SPECTRUM
-            else:
-                dataset.data_type = sidpy.DataTypes.LINE_PLOT
-
-    def get_dimensions(self):
-        dic = self.original_metadata
-
-        reciprocal_name = 'u'
-        spatial_name = 'x'
-
-        if 'dimensional_calibrations' in dic:
-
-            for dim in range(len(dic['dimensional_calibrations'])):
-                dimension_tags = dic['dimensional_calibrations'][dim]
-                units = dimension_tags['units']
-                values = (np.arange(self.data_cube.shape[int(dim)]) - dimension_tags['offset']) * dimension_tags['scale']
-
-                if 'eV' == units:
-                    self.dimensions.append(sidpy.Dimension(values, name='energy_loss', units=units,
-                                                           quantity='energy-loss', dimension_type='spectral'))
-                elif 'eV' in units:
-                    self.dimensions.append(sidpy.Dimension(values, name='energy', units=units,
-                                                           quantity='energy', dimension_type='spectral'))
-                elif '1/' in units or units in ['mrad', 'rad']:
-                    self.dimensions.append(sidpy.Dimension(values, name=reciprocal_name, units=units,
-                                                           quantity='reciprocal distance',dimension_type='reciprocal'))
-                    reciprocal_name = chr(ord(reciprocal_name) + 1)
-                elif 'nm' in units:
-                    self.dimensions.append(sidpy.Dimension(values, name=spatial_name, units=units,
-                                                           quantity='distance', dimension_type='spatial'))
-                    spatial_name = chr(ord(spatial_name) + 1)
-                else:
-                    self.dimensions.append(sidpy.Dimension(values, name='generic', units='generic',
-                                                           quantity='generic', dimension_type='UNKNOWN'))
-
-
-    def get_filename(self):
-        return self.__filename
-
-    filename = property(get_filename)
-
-    def get_raw(self):
-        return self.data
-
-    data = property(get_raw)
-
-    def get_tags(self):
-        return self.original_metadata
-
-    tags = property(get_tags)

pyTEMlib/nsi_reader.py

@@ -1,165 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Reader capable of reading one or all NSID datasets present in a given HDF5 file
-
-Created on Fri May 22 16:29:25 2020
-
-@author: Gerd Duscher, Suhas Somnath
-"""
-from __future__ import division, print_function, absolute_import, unicode_literals
-from warnings import warn
-import sys
-import h5py
-import numpy as np
-
-# pyNSID function
-import sidpy
-
-from pyTEMlib.hdf_utils import get_all_main
-if sys.version_info.major == 3:
-    unicode = str
-
-
-class NSIDReader(sidpy.Reader):
-
-    def __init__(self, h5_object):
-        """
-        h5_object - hdf5 dataset
-            specific Main dataset that needs to be read into a sidpy Dataset.
-            # If no path is specified, read all available NSID Main datasets
-        """
-
-        if not isinstance(h5_object.file, h5py.File):
-            raise TypeError('we can only read h5py datasets')
-
-        super(NSIDReader, self).__init__(file_path=h5_object.file.name)
-
-        self.dset = None
-        self.main_datasets = []
-        if isinstance(h5_object, h5py.Dataset):
-            self.dset = h5_object
-            self.h5_group = self.dset.parent
-
-        elif isinstance(h5_object, h5py.Group):
-            self.h5_group = h5_object
-        else:
-            raise TypeError('we can only read h5py datasets')
-
-        # Find all main datasets is done in read as the file may change between readings
-        # DO NOT close HDF5 file. Dask array will fail if you do so.
-        # TODO: sidpy.Dataset may need the ability to close a HDF5 file
-        # Perhaps this would be done by reading all contents into memory..
-
-    @staticmethod
-    def read_h5py_dataset(dset):
-
-        if not isinstance(dset, h5py.Dataset):
-            raise TypeError('can only read single Dataset, use read_all_in_group or read_all function instead')
-        # create vanilla dask array
-        dataset = sidpy.Dataset.from_array(np.array(dset))
-
-        if 'title' in dset.attrs:
-            dataset.title = dset.attrs['title']
-        else:
-            dataset.title = dset.name
-
-        if 'units' in dset.attrs:
-            dataset.units = dset.attrs['units']
-        else:
-            dataset.units = 'generic'
-
-        if 'quantity' in dset.attrs:
-            dataset.quantity = dset.attrs['quantity']
-        else:
-            dataset.quantity = 'generic'
-
-        if 'data_type' in dset.attrs:
-            dataset.data_type = dset.attrs['data_type']
-        else:
-            dataset.data_type = 'generic'
-
-        if 'modality' in dset.attrs:
-            dataset.modality = dset.attrs['modality']
-        else:
-            dataset.modality = 'generic'
-
-        if 'source' in dset.attrs:
-            dataset.source = dset.attrs['source']
-        else:
-            dataset.source = 'generic'
-
-        dataset.axes = {}
-
-        for dim in range(np.array(dset).ndim):
-            try:
-                label = dset.dims[dim].keys()[-1]
-
-            except ValueError:
-                print('dimension {} not NSID type using generic'.format(dim))
-            name = dset.dims[dim][label].name
-            dim_dict = {'quantity': 'generic', 'units': 'generic', 'dimension_type': 'generic'}
-            h5_dim_dict = dict(dset.parent[name].attrs)
-            if 'quantity' in h5_dim_dict:
-                dim_dict['quantity'] = h5_dim_dict['quantity']
-            else:
-                if 'NAME' in h5_dim_dict:
-                    param = h5_dim_dict['NAME'].decode("utf-8").split('[')
-                    # print(param)
-                    if len(param) == 1:
-                        if param[0] == 'frame':
-                            dim_dict['quantity'] = 'stack'
-                            dim_dict['units'] = 'frame'
-                            dim_dict['dimension_type'] = sidpy.DimensionTypes.TEMPORAL
-                    elif len(param) == 2:
-                        dim_dict['quantity'] = param[0]
-                        dim_dict['units'] = param[1][0:-1]
-
-                        if dim_dict['units'] == 'nm':
-                            dim_dict['dimension_type'] = sidpy.DimensionTypes.SPATIAL
-                        elif dim_dict['units'] == 'eV':
-                            dim_dict['dimension_type'] = sidpy.DimensionTypes.SPECTRAL
-
-            if 'units' in h5_dim_dict:
-                dim_dict['units'] = h5_dim_dict['units']
-            if 'dimension_type' in h5_dim_dict:
-                dim_dict['dimension_type'] = h5_dim_dict['dimension_type']
-
-            dim_dict.update(dict(dset.parent[name].attrs))
-
-            dataset.set_dimension(dim, sidpy.Dimension(np.array(dset.parent[name][()]), name=dset.dims[dim].label,
-                                                       quantity=dim_dict['quantity'], units=dim_dict['units'],
-                                                       dimension_type=dim_dict['dimension_type']))
-
-        if 'metadata' in dset.parent:
-            dataset.metadata = sidpy.base.dict_utils.nest_dict(dict(dset.parent['metadata'].attrs))
-
-        dataset.metadata.update(dict(dset.attrs))
-
-        dataset.original_metadata = {}
-        if 'original_metadata' in dset.parent:
-            dataset.original_metadata = sidpy.base.dict_utils.nest_dict(dict(dset.parent['original_metadata'].attrs))
-
-        # hdf5 information
-        dataset.h5_dataset = dset
-
-        return dataset
-
-    def can_read(self):
-        pass
-
-    def read(self):
-        if not isinstance(self.h5_group, h5py.Group):
-            raise TypeError('This function needs to be initialised with a hdf5 group or dataset first')
-        list_of_main = get_all_main(self.h5_group, verbose=False)
-
-        """
-        Go through each of the identified
-        """
-        list_of_datasets = []
-        for dset in list_of_main:
-            list_of_datasets.append(self.read_h5py_dataset(dset))
-
-        return list_of_datasets
-
-    def read_all_in_group(self, recursive=True):
-        pass