mtmhdf 0.0.2__py3-none-any.whl

mtmhdf/__init__.py

@@ -0,0 +1 @@
+from .reader import Reader

mtmhdf/_hdf4.py

@@ -0,0 +1,53 @@
+from pyhdf.SD import SD, SDC, SDS
+import numpy as np
+
+
+
+
+
+class HDF4:
+    DATATYPES = {
+        4: "char",
+        3: "uchar",
+        20: "int8",
+        21: "uint8",
+        22: "int16",
+        23: "uint16",
+        24: "int32",
+        25: "uint32",
+        5: "float32",
+        6: "float64",
+    }
+    OPENMODES = {"r": SDC.READ, "w": SDC.WRITE}
+
+    @staticmethod
+    def open(file_path:str, mode='r', *args, **kwargs):
+        return SD(file_path, mode=HDF4.OPENMODES[mode], *args, **kwargs)
+
+    @staticmethod
+    def read(fp:SD, dataset_name:str):
+        return fp.select(dataset_name)
+    
+    @staticmethod
+    def keys(fp:SD) -> list[str]:
+        return list(fp.datasets().keys())
+
+    @staticmethod
+    def infos(fp:SD) -> dict:
+        return {name: HDF4.dpinfo(HDF4.read(fp, name)) for name in HDF4.keys(fp)}
+
+    @staticmethod
+    def dpinfo(dp:SDS) -> dict:
+        attrs = dp.attributes()
+        _info_list = dp.info()
+        _info_dict = {
+            "dataset_name": _info_list[0], 
+            "dataset_rank": _info_list[1], 
+            "dataset_dims": _info_list[2], 
+            "dataset_type": HDF4.DATATYPES[_info_list[3]]
+        }
+        _info_dict.update(attrs)
+        return _info_dict
+     
+    
+

mtmhdf/_hdf5.py

@@ -0,0 +1,52 @@
+from netCDF4 import Dataset, Variable
+
+class HDF5(Dataset):
+    @staticmethod
+    def open(file_path:str, mode='r', *args, **kwargs):
+        return Dataset(file_path, mode=HDF5.OPENMODES[mode], *args, **kwargs)
+
+    @staticmethod
+    def read(fp:Dataset, name:str) -> Variable:
+        return HDF5._jump(fp, name)
+
+    @staticmethod
+    def keys(fp:Dataset) -> list[str]:
+        return list(HDF5._walk(fp))
+    
+    @staticmethod
+    def dpinfo(dp: Variable) -> dict:
+        info_dict = dp.__dict__
+        info_dict.update({
+            "dataset_name": dp.group().path + "/" + dp.name,
+            "dataset_dims": dp.shape,
+            "dataset_type": dp.datatype.name
+        })
+        return info_dict
+
+    @staticmethod
+    def infos(fp:Dataset) -> dict:
+        return {name: HDF5.dpinfo(HDF5.read(fp, name)) for name in HDF5.keys(fp)}
+
+    @staticmethod
+    def _walk(fp: Dataset, path=""):
+        if not len(path) or path[-1] != "/":
+            path += "/"
+        current_variables = list(fp.variables.keys())
+        for variable in current_variables:
+            yield path + variable
+        current_groups = list(fp.groups.keys())
+        for group in current_groups:
+            yield from HDF5._walk(fp.groups[group], path + group)
+
+    @staticmethod
+    def _jump(fp: Dataset, path="/"):
+        path_list = path.lstrip("/").split("/")
+        if not len(path_list):
+            return fp
+        subnode_fp = fp.__getitem__(path_list[0])
+        subnode_path = "/" + "/".join(path_list[1:])
+        if len(path_list) == 1:
+            return subnode_fp
+        else:
+            return HDF5._jump(subnode_fp, subnode_path)
+        

mtmhdf/_utils.py

@@ -0,0 +1,15 @@
+import numpy as np
+from numbers import Number
+
+def int2binarystring(num: int, length: int = 0) -> str:
+    return bin(num)[2:].zfill(length)
+
+def bitoffset(data:np.ndarray | Number, bit_start_pos:int, bit_end_pos:int) -> np.ndarray | Number:
+    # 左开右闭区间，即从bit_start_pos开始，到bit_end_pos结束，不包含bit_end_pos
+    return (data >> bit_start_pos) % (2 ** (bit_end_pos - bit_start_pos))
+
+def scale(data:np.ndarray | Number, scale_factor:Number=1, add_offset:Number=0) -> np.ndarray | Number:
+    return data * scale_factor + add_offset
+
+def mask(data:np.ndarray | Number, fill_value:Number) -> np.ma.MaskedArray:
+    return np.ma.masked_values(data, fill_value, rtol=1e-8, atol=1e-9)

mtmhdf/reader.py

@@ -0,0 +1,214 @@
+import os
+import numpy as np
+import numpy.ma as ma
+
+from _hdf4 import HDF4
+from _hdf5 import HDF5
+
+from _utils import int2binarystring, bitoffset, scale, mask
+
+
+class TemplateData:
+    def __init__(self) -> None:
+        raise NotImplementedError("Subclass must implement this method")
+
+    def infos(self) -> dict:
+        raise NotImplementedError("Subclass must implement this method")
+
+    def __getitem__(self, *item) -> np.ma.MaskedArray | np.ndarray:
+        raise NotImplementedError("Subclass must implement this method")
+
+
+class TemplateReader:
+    LinkedDataClass = None
+
+    def __init__(self, data_file:str, *args, **kwargs):
+        raise NotImplementedError("Subclass must implement this method")
+
+    def read(self, name:str, *args, **kwargs):
+        raise NotImplementedError("Subclass must implement this method")
+
+    def __getitem__(self, name:str):
+        return self.rawread(name)
+
+    def readraw(self, name:str):
+        raise NotImplementedError("Subclass must implement this method")
+
+    def readbit(self, name:str, bit_start_pos: int, bit_end_pos: int, *args, **kwargs) -> np.ma.MaskedArray | np.ndarray:
+        # Bit fields within each byte are numbered from the left:
+        # 7, 6, 5, 4, 3, 2, 1, 0.
+        # The left-most bit (bit 7) is the most significant bit.
+        # The right-most bit (bit 0) is the least significant bit.
+        # 左开右闭区间，即从bit_start_pos开始，到bit_end_pos结束，不包含bit_end_pos
+        dp = self.readraw(name)
+        return bitoffset(np.array(dp[:]), bit_start_pos, bit_end_pos)
+
+    def keys(self) -> list[str]:
+        raise NotImplementedError("Subclass must implement this method")
+
+    def infos(self) -> dict:
+        raise NotImplementedError("Subclass must implement this method")
+
+# ===================================================================================================
+
+
+class HDF4Data(TemplateData):
+    def __init__(self, dp:HDF4, mode="manual", isScaleAndOffset: bool = True, isMasked: bool = True, manual_options=None, **kwargs) -> None:
+        self.dp = dp
+        self.mode = mode
+        self.isMasked = isMasked
+        self.isScaleAndOffset = isScaleAndOffset
+
+        default_manual_options = {
+            "attr_scale_factor": "scale_factor",
+            "attr_add_offset": "add_offset",
+            "attr_fill_value": "_FillValue",
+            "attr_decimal": 8,
+        }
+        if manual_options is None:
+            self.manual_options = default_manual_options
+        else:
+            self.manual_options = manual_options.copy()
+            self.manual_options.update(default_manual_options)
+
+    def infos(self):
+        return HDF4.dpinfo(self.dp)
+
+    def manual_transform(self, data: np.ndarray) -> np.ma.MaskedArray|np.ndarray:
+        infos: dict = self.infos()
+        
+        attr_scale_factor = self.manual_options["attr_scale_factor"]
+        attr_add_offset = self.manual_options["attr_add_offset"]
+        attr_fill_value = self.manual_options["attr_fill_value"]
+        attr_decimal = self.manual_options["attr_decimal"]
+
+        scale_factor = round(infos.get(attr_scale_factor, 1), attr_decimal)
+        add_offset = round(infos.get(attr_add_offset, 0), attr_decimal)
+        fill_value = infos.get(attr_fill_value)
+
+        if self.isMasked:
+            data = mask(data, fill_value)
+        if self.isScaleAndOffset:
+            data = scale(data, scale_factor, add_offset)
+        return data  
+
+    def __getitem__(self, *item) -> np.ma.MaskedArray | np.ndarray:
+        data = self.dp.__getitem__(*item)
+        if self.mode == "native":
+            return data
+        elif self.mode == "manual":
+            return self.manual_transform(np.array(data))
+        else:
+            raise ValueError(f"Invalid mode: {self.mode}")
+
+
+class HDF4Reader(TemplateReader):
+    LinkedDataClass = HDF4Data
+
+    def __init__(self, data_file:str, *args, **kwargs):
+        self.fp = HDF4.open(data_file, *args, **kwargs)
+
+    def readraw(self, name:str):
+        return HDF4.read(self.fp, name)
+
+    def read(self, name:str, isScaleAndOffset: bool = True, isMasked: bool = True, **kwargs):
+        dp = HDF4.read(self.fp, name)
+        DataClass = HDF4Reader.LinkedDataClass
+        return DataClass(dp, mode="manual", isScaleAndOffset=isScaleAndOffset, isMasked=isMasked, **kwargs)
+
+    def keys(self) -> list[str]:
+        return HDF4.keys(self.fp)
+
+    def infos(self) -> dict:
+        return HDF4.infos(self.fp)
+
+# ===================================================================================================
+
+
+class HDF5Data(TemplateData):
+    def __init__(self, dp, mode="manual", isScaleAndOffset: bool = False, isMasked: bool = True, manual_options=None, **kwargs) -> None:
+        self.dp = dp
+        self.mode = mode
+        self.isScaleAndOffset = isScaleAndOffset
+        self.isMasked = isMasked
+
+        default_manual_options = {
+            "attr_scale_factor": "scale_factor",
+            "attr_add_offset": "add_offset",
+            "attr_fill_value": "_FillValue",
+            "attr_decimal": 8,
+        }
+        if manual_options is None:
+            self.manual_options = default_manual_options
+        else:
+            self.manual_options = manual_options.copy()
+            self.manual_options.update(default_manual_options)
+            
+        if mode == "manual":
+            self.dp.set_auto_scale(False)
+            self.dp.set_auto_mask(False)
+        elif mode == "native":
+            self.dp.set_auto_scale(isScaleAndOffset)
+            self.dp.set_auto_mask(isMasked)
+        else:
+            raise ValueError(f"Invalid mode: {self.mode}")
+
+    def infos(self):
+        return HDF5.dpinfo(self.dp)
+
+    def manual_transform(self, data: np.ndarray) -> np.ma.MaskedArray|np.ndarray:
+        infos: dict = self.infos()
+        
+        attr_scale_factor = self.manual_options["attr_scale_factor"]
+        attr_add_offset = self.manual_options["attr_add_offset"]
+        attr_fill_value = self.manual_options["attr_fill_value"]
+        attr_decimal = self.manual_options["attr_decimal"]
+        scale_factor = round(infos.get(attr_scale_factor, 1), attr_decimal)
+        add_offset = round(infos.get(attr_add_offset, 0), attr_decimal)
+        fill_value = infos.get(attr_fill_value)
+
+        if self.isMasked:
+            data = mask(data, fill_value)
+        if self.isScaleAndOffset:
+            data = scale(data, scale_factor, add_offset)
+        return data
+
+    def __getitem__(self, *item) -> np.ma.MaskedArray | np.ndarray:
+        data = self.dp.__getitem__(*item)
+        if self.mode == "native":
+            return data
+        elif self.mode == "manual":
+            return self.manual_transform(np.array(data))
+        else:
+            raise ValueError(f"Invalid mode: {self.mode}")
+
+class HDF5Reader(TemplateReader):
+    LinkedDataClass = HDF5Data
+
+    def __init__(self, data_file:str, *args, **kwargs):
+        self.fp = HDF5.open(data_file, *args, **kwargs)
+
+    def readraw(self, name:str):
+        return HDF5.read(self.fp, name)
+
+    def read(self, name:str, isScaleAndOffset: bool = True, isMasked: bool = True, mode="native", **kwargs):
+        dp = HDF5.read(self.fp, name)
+        DataClass = HDF5Reader.LinkedDataClass
+        return DataClass(dp, mode=mode, isScaleAndOffset=isScaleAndOffset, isMasked=isMasked, **kwargs)
+
+    def keys(self) -> list[str]:
+        return HDF5.keys(self.fp)
+
+    def infos(self) -> dict:
+        return HDF5.infos(self.fp)
+
+
+def Reader(data_file: str, *args, **kwargs) -> TemplateReader:
+    _, ext = os.path.splitext(data_file)
+    ext = ext.lower()
+    if ext in ['.nc', '.h5', '.he5', '.hdf5']:
+        return HDF5Reader(data_file, *args, **kwargs)
+    elif ext in ['.hdf', '.hdf4']:
+        return HDF4Reader(data_file, *args, **kwargs)
+    else:
+        raise ValueError(f"Unsupported file extension: {ext}")

mtmhdf-0.0.2.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.4
+Name: mtmhdf
+Version: 0.0.2
+Summary: imutum's packages for HDF
+License: MIT License
+        
+        Copyright (c) 2023 imutum
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Microsoft :: Windows
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyhdf>=0.10.5
+Requires-Dist: netcdf4>=1.6.5
+Requires-Dist: numpy>=1.26.0
+Dynamic: license-file
+
+# imutum_python_hdf
+ 

mtmhdf-0.0.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+mtmhdf/__init__.py,sha256=P4ZUG3Klkmyx1S3RbAxzL8VzWm0P7bQoRJBgbZ3M2eo,26
+mtmhdf/_hdf4.py,sha256=RonrjU2D0v4eG0TcluY6Xb0nKYKY-vcxu6BWEB3tZx4,1235
+mtmhdf/_hdf5.py,sha256=tux71Zey-JyAcvE1r6rTl165xlPVDxmUiWbS_brMQrk,1654
+mtmhdf/_utils.py,sha256=8YeVA2NPu8MoW9pRAw9VTbSwdwvjktgt2M1IQc7VcdQ,711
+mtmhdf/reader.py,sha256=Hi1uN7nh7lRgwuJUr5TesWL1YfDLCGsYDrmEt9PBm7s,7856
+mtmhdf-0.0.2.dist-info/licenses/LICENSE,sha256=-lZJZIaH__2OkAxU2yFoe0cHHe48x8s2LIxCC3KFUqA,1063
+mtmhdf-0.0.2.dist-info/METADATA,sha256=V-9N4BYwexAHmmgDFw1QNbmm-DTAfKgusAlRmVlZrzk,1873
+mtmhdf-0.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mtmhdf-0.0.2.dist-info/top_level.txt,sha256=eBIgfbDXWQslU5lXIbDr0x77IXgUqXKTu6OGzdipLMs,7
+mtmhdf-0.0.2.dist-info/RECORD,,

mtmhdf-0.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mtmhdf-0.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 imutum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mtmhdf-0.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+mtmhdf