h5rdmtoolbox 0.8.1__py3-none-any.whl

h5rdmtoolbox/__init__.py

@@ -0,0 +1,174 @@
+"""h5rdtoolbox repository"""
+import atexit
+import pathlib
+# noinspection PyUnresolvedReferences
+import pint_xarray
+import shutil
+import xarray as xr
+from typing import Union, Callable
+
+from h5rdmtoolbox._cfg import set_config, get_config, get_ureg
+
+pint_xarray.unit_registry = get_ureg()
+
+from . import conventions
+from .conventions.core import Convention
+from . import plotting
+from . import wrapper
+from ._user import UserDir
+from ._version import __version__
+from . import database
+from . import utils
+from .wrapper.core import lower, Lower, File, Group, Dataset
+from . import errors
+
+from .wrapper.accessory import register_special_dataset
+
+name = 'h5rdmtoolbox'
+__this_dir__ = pathlib.Path(__file__).parent
+__author__ = 'Matthias Probst'
+__author_orcid__ = 'https://orcid.org/0000-0001-8729-0482'
+
+logger = utils.create_tbx_logger('h5rdmtoolbox')
+
+logger.setLevel(get_config()['init_logger_level'])
+
+cv_h5py = conventions.Convention('h5py',
+                                 contact=__author_orcid__)
+cv_h5py.register()
+
+cv_h5tbx = conventions.Convention.from_yaml(__this_dir__ / 'data/h5tbx_convention.yaml')
+
+use = conventions.core.use
+
+use(get_config()['default_convention'])
+
+
+class FileDB:
+    """User-friendly interface to database.Folder, database.File or database.Files"""
+
+    def __new__(cls, path, rec=False, **kwargs):
+        if isinstance(path, (list, tuple)):
+            filenames = []
+            for p in [pathlib.Path(_p) for _p in path]:
+                if p.is_file():
+                    filenames.append(p)
+                elif p.is_dir():
+                    if rec:
+                        for f in p.rglob('*.hdf'):
+                            filenames.append(f)
+                    else:
+                        for f in p.glob('*.hdf'):
+                            filenames.append(f)
+            return database.Files(filenames, **kwargs)
+
+        path = pathlib.Path(path)
+        if path.is_dir():
+            return database.Folder(path, rec=rec, **kwargs)
+        return database.File(path, **kwargs)
+
+
+def dump(src: Union[str, File, pathlib.Path]) -> None:
+    """Call h5.dump() on the provided HDF5 file
+
+    Parameters
+    ----------
+    src : str, File, pathlib.Path
+        the HDF5 file or filename to dump
+    """
+    if isinstance(src, File):
+        with File(src.hdf_filename) as h5:
+            return h5.dump()
+    with File(src) as h5:
+        return h5.dump()
+
+
+def dumps(src: Union[str, File, pathlib.Path]):
+    """Call h5.dumps() on the provided HDF5 file"""
+    if isinstance(src, File):
+        with File(src.hdf_filename) as h5:
+            return h5.dumps()
+    with File(src) as h5:
+        return h5.dumps()
+
+
+def register_dataset_decoder(decoder: Callable, decoder_name: str = None, overwrite: bool = False):
+    """A decoder function takes a xarray.DataArray and a dataset as input and returns a xarray.DataArray
+    It is called after the dataset is loaded into memory and before being returned to the user. Be careful:
+    Multiple decoders can be registered and they are called in the order of registration. Hence, your decoder
+    may behave unexpectedly!
+    """
+    from .wrapper import ds_decoder
+    if decoder_name is None:
+        decoder_name = decoder.__name__
+    registered_decorators = ds_decoder.registered_dataset_decoders
+    if decoder_name in registered_decorators or decoder in registered_decorators.values():
+        if not overwrite:
+            raise ValueError(f'decoder "{decoder_name}" already registered. Name and function must be unique.')
+    ds_decoder.registered_dataset_decoders[decoder_name] = decoder
+
+
+atexit_verbose = False
+
+
+@atexit.register
+def clean_temp_data(full: bool = False):
+    """cleaning up the tmp directory"""
+    from ._user import _user_root_dir
+
+    failed_dirs = []
+    failed_dirs_file = UserDir['tmp'] / 'failed.txt'
+    if full:
+        root_tmp = _user_root_dir / 'tmp'
+        if root_tmp.exists():
+            try:
+                shutil.rmtree(root_tmp)
+                root_tmp.mkdir(exist_ok=True, parents=True)
+            except PermissionError as e:
+                print(f'removing tmp folder "{root_tmp}" failed due to "{e}".')
+        return
+
+    for _tmp_session_dir in [UserDir['tmp'], ]:
+        if atexit_verbose:
+            print(f'cleaning up tmp directory "{_tmp_session_dir}"')
+        if _tmp_session_dir.exists():
+            try:
+                if atexit_verbose:
+                    print(f'try deleting tmp in session dir: {_tmp_session_dir}')
+                shutil.rmtree(_tmp_session_dir)
+            except PermissionError as e:
+                if atexit_verbose:
+                    print(f'[!] failed deleting tmp session dir: {_tmp_session_dir}')
+                failed_dirs.append(UserDir['tmp'])
+                if atexit_verbose:
+                    print(f'removing tmp folder "{_tmp_session_dir}" failed due to "{e}". Best is you '
+                          f'manually delete the directory.')
+            finally:
+                lines = []
+                if failed_dirs_file.exists():
+                    with open(failed_dirs_file, 'r') as f:
+                        lines = f.readlines()
+                        for line in lines:
+                            try:
+                                shutil.rmtree(line.strip())
+                            except Exception:
+                                if pathlib.Path(line).exists():
+                                    failed_dirs.append(line)
+
+                if lines or failed_dirs:
+                    with open(failed_dirs_file, 'w') as f:
+                        for fd in failed_dirs:
+                            f.writelines(f'{fd}\n')
+                else:
+                    failed_dirs_file.unlink(missing_ok=True)
+        else:
+            logger.debug(f'No user tmp dir not found: {_tmp_session_dir}')
+
+
+xr.set_options(display_expand_data=False)
+
+__all__ = ('__version__', '__author__', '__author_orcid__', 'UserDir', 'use',
+           'File', 'Group', 'Dataset',
+           'dump', 'dumps', 'cv_h5py', 'lower', 'Lower',
+           'set_config', 'get_config', 'get_ureg',
+           'Convention')

h5rdmtoolbox/_cfg.py

@@ -0,0 +1,102 @@
+"""package configuration. largely based on xarray's configuration concept. no credits taken!"""
+from pint import UnitRegistry
+from typing import Dict, Union
+
+ureg = UnitRegistry(force_ndarray_like=True)
+
+
+def is_valid_logger_level(level: Union[str, int]):
+    """Check if the logger level is valid."""
+    if not isinstance(level, (str, int)):
+        # raise TypeError(f'Invalid type for the logger: {type(logger_value)}')
+        return False
+    if isinstance(level, str):
+        return level.lower() in ('error', 'debug', 'critical', 'warning', 'info', 'fatal', 'warning', 'warn')
+    # 0: NOTSET, 10: DEBUG, 20: INFO, 30: WARNING, 40: ERROR, 50: CRITICAL
+    return level in (0, 10, 20, 30, 40, 50)
+
+
+CONFIG = {'return_xarray': True,
+          'advanced_shape_repr': True,
+          'natural_naming': True,
+          'hdf_compression': 'gzip',
+          'hdf_compression_opts': 5,
+          'xarray_unit_repr_in_plots': 'in',
+          'plotting_name_order': ('plot_name', 'long_name', 'standard_name'),
+          'require_unit': True,  # datasets require units
+          'ureg_format': 'C~',
+          'default_convention': 'h5tbx',
+          'init_logger_level': 'ERROR',
+          'dtime_fmt': '%Y%m%d%H%M%S%f',
+          'expose_user_prop_to_attrs': True,
+          'scale_attribute_name': 'scale',
+          'offset_attribute_name': 'offset',
+          'add_provenance': True,
+          'ignore_standard_attribute_errors': False,
+          'allow_deleting_standard_attributes': False,
+          'parallel_find': True, }
+
+_VALIDATORS = {
+    'return_xarray': lambda x: isinstance(x, bool),
+    'advanced_shape_repr': lambda x: isinstance(x, bool),
+    'natural_naming': lambda x: isinstance(x, bool),
+    'hdf_compression': lambda x: isinstance(x, str),
+    'hdf_compression_opts': lambda x: isinstance(x, int),
+    'xarray_unit_repr_in_plots': lambda x: x in ('/', '()', '(', '[]', '[', '//', 'in'),
+    'plotting_name_order': lambda x: isinstance(x, (tuple, list)) and [xx in ('plot_name', 'long_name', 'standard_name')
+                                                                       for xx in x],
+    'require_unit': lambda x: isinstance(x, bool),
+    'ureg_format': lambda x: isinstance(x, str),
+    'default_convention': lambda x: str(x) in ('h5py', 'h5tbx', 'None'),
+    'init_logger_level': lambda x: is_valid_logger_level(x),
+    'dtime_fmt': lambda x: isinstance(x, str),
+    'expose_user_prop_to_attrs': lambda x: isinstance(x, bool),
+    'scale_attribute_name': lambda x: isinstance(x, str),
+    'offset_attribute_name': lambda x: isinstance(x, str),
+    'add_provenance': lambda x: isinstance(x, bool),
+    'ignore_standard_attribute_errors': lambda x: isinstance(x, bool),
+    'parallel_find': lambda x: isinstance(x, bool),
+}
+
+
+class set_config:
+    """Set the configuration parameters."""
+
+    def __init__(self, **kwargs):
+        self.old = {}
+        for k, v in kwargs.items():
+            if k in _VALIDATORS and not _VALIDATORS[k](v):
+                raise ValueError(f'Config parameter "{k}" has invalid value: "{v}"')
+            if k not in CONFIG:
+                raise KeyError(f'Not a configuration key: "{k}"')
+            self.old[k] = CONFIG[k]
+            if k == 'ureg_format':
+                get_ureg().default_format = str(v)
+        self._update(kwargs)
+
+    def __enter__(self):
+        return
+
+    def __exit__(self, *args, **kwargs):
+        self._update(self.old)
+
+    def _update(self, options_dict: Dict):
+        CONFIG.update(options_dict)
+
+
+# set_config = ConfigSetter()
+
+
+def get_config(key=None):
+    """Return the configuration parameters."""
+    if key is None:
+        return CONFIG
+    return CONFIG[key]
+
+
+def get_ureg() -> UnitRegistry:
+    """get unit registry"""
+    return ureg
+
+
+set_config(ureg_format=CONFIG['ureg_format'])

h5rdmtoolbox/_cli.py

@@ -0,0 +1,211 @@
+"""collection of command line interface functions"""
+
+import argparse
+import h5py
+import webbrowser
+from pprint import pprint
+
+from . import File
+from ._version import __version__
+
+
+def main():
+    """main command line interface function"""
+    parser = argparse.ArgumentParser(description='h5RDMtoolbox command line interface')
+    parser.add_argument('-V', '--version',
+                        action='version', version=f'%(prog)s {__version__}')
+    parser.add_argument('-D', dest='documentation',
+                        action='store_true',
+                        help="Opens documentation in browser")
+    parser.add_argument('-u', '--user-dirs',
+                        action='store_true',
+                        default=False,
+                        help='Print user directories.')
+    parser.add_argument('-d', '--dump',
+                        type=str,
+                        required=False,
+                        default=None,
+                        help='Dump file content.')
+    subparsers = parser.add_subparsers(help='sub-commands help')
+
+    # LAYOUT
+    sp_layout = subparsers.add_parser('layout', help='layout menu')
+    sp_layout.set_defaults(cmd='layout')
+    sp_layout.add_argument('--list-registered', action='store_true',
+                           help='List all registered layouts.')
+    sp_layout.add_argument('-l', '--layout',
+                           type=str,
+                           required=False,
+                           default=None,
+                           help='Registered layout name or filename of a layout.')
+    sp_layout.add_argument('-c', '--check',
+                           type=str,
+                           required=False,
+                           default=None,
+                           help='Filename to run check on.')
+    sp_layout.add_argument('-r', '--register',
+                           type=str,
+                           required=False,
+                           default=None,
+                           help='Register the passed HDF file as a layout.')
+
+    # STANDARD NAME
+    sp_standard_name = subparsers.add_parser('standard_name', help='standard name menu')
+    sp_standard_name.set_defaults(cmd='standard_name')
+    sp_standard_name.add_argument('--list-registered', action='store_true',
+                                  help='List all registered standard name tables.')
+    sp_standard_name.add_argument('-t', '--table',
+                                  type=str,
+                                  required=False,
+                                  default=None,
+                                  help='Registered standard name or filename of a standard table.')
+    sp_standard_name.add_argument('-f', '--file',
+                                  type=str,
+                                  required=False,
+                                  default=None,
+                                  help='Filename to run check on.')
+    # MONGODB
+    sp_mongo = subparsers.add_parser('mongodb', help='mongodb menu')
+    sp_mongo.set_defaults(cmd='mongodb')
+    # sp_mongo.add_argument('list_dbs', nargs='?', default=False, help='List all available databases')
+    # sp_mongo.add_argument('list_collections', nargs='?', default=False, help='List all available collections of the '
+    #                                                                          'selected database')
+    sp_mongo.add_argument('--list-dbs', action='store_true')
+    sp_mongo.add_argument('--list-collections', action='store_true')
+    sp_mongo.add_argument('-i', '--ip',
+                          type=str,
+                          required=False,
+                          default='localhost',
+                          help='IP address. Default is localhost')
+    sp_mongo.add_argument('-p', '--port',
+                          type=int,
+                          required=False,
+                          default=27017,
+                          help='Port. Default is 27017')
+    sp_mongo.add_argument('-d', '--db',
+                          type=str,
+                          required=False,
+                          help='Database')
+    sp_mongo.add_argument('-c', '--collection',
+                          type=str,
+                          required=False,
+                          default=False,
+                          help='Collection')
+    sp_mongo.add_argument('-a', '--add',
+                          type=str,
+                          required=False,
+                          default=False,
+                          help='Add HDF file to database')
+
+    args = parser.parse_args()
+    if args.documentation:
+        webbrowser.open('https://matthiasprobst.github.io/h5RDMtoolbox/')
+        return
+    if args.user_dirs:
+        from ._user import UserDir
+        pprint(UserDir)
+        return
+    if args.dump:
+        with File(args.dump) as h5:
+            h5.sdump()
+    arg_vars = vars(args)
+
+    if 'cmd' in arg_vars:
+        if args.cmd == 'mongodb':
+            from pymongo import MongoClient
+            db = None
+            collection = None
+
+            print(f'\n > Current connection: {args.ip}:{args.port}')
+            client = MongoClient(args.ip, args.port)
+            list_of_databases = client.list_database_names()
+
+            if args.db:
+                if len(list_of_databases) == 0:
+                    print(' > No connection to a client found')
+                    return
+                db = client[args.db]
+
+            if args.collection:
+                # list_of_collections = db.list_collection_names()
+                # if len(list_of_collections) == 0:
+                #     print(f' ! No collections found')
+                #     return
+                collection = db[args.collection]
+                print(f' > Selected collection: {collection}')
+
+            if args.list_dbs:
+                pprint(list_of_databases)
+                return
+
+            if args.list_collections:
+                print(' > Listing collections for ')
+                if db is None:
+                    print(' ! No database selected')
+                    return
+                print(db)
+                list_of_collections = db.list_collection_names()
+                print(' > List of collections:\n--------------------')
+                pprint(list_of_collections)
+                return
+
+            if args.add:
+                if collection is None:
+                    print('No collection selected or found')
+                    return
+                try:
+                    with h5py.File(args.add):
+                        pass
+                except OSError:
+                    print(' ! Unable to open the HDF file. Make sure it is an HDF file and that it can be '
+                          'opened/is not corrupt')
+                    return
+                print(f' > Adding hdf file: {args.add} to collection {collection.name} of database {db.name}')
+                with File(args.add) as h5:
+                    # noinspection PyUnresolvedReferences
+                    from h5rdmtoolbox.database import mongo
+                    h5.mongo.insert(collection=collection, recursive=True, update=True)
+            return
+        if args.cmd == 'standard_name':
+            from .conventions.tbx import StandardNameTable
+            import pathlib
+            if args.list_registered:
+                StandardNameTable.print_registered()
+                return
+            if args.table:
+                snt_filename = pathlib.Path(args.table)
+                if snt_filename.exists():
+                    snt = StandardNameTable.from_yaml(args.table)
+                else:
+                    snt = StandardNameTable.load_registered(args.table)
+                print(f' > Checking file "{args.file}" with standard name table "{snt.versionname}"')
+                with File(args.file) as h5:
+                    snt.check_hdf_group(h5, recursive=True, raise_error=False)
+                return
+        if args.cmd == 'layout':
+            from .conventions import layout
+            import pathlib
+            if args.list_registered:
+                for k in layout.File.Registry().layouts.keys():
+                    print(k)
+                return
+            hdf_filename = None
+            if args.check:
+                hdf_filename = pathlib.Path(args.check)
+            if args.layout:
+                layout_filename = pathlib.Path(args.layout)
+                if layout_filename.exists():
+                    h5lay = layout.File(layout_filename)
+                else:
+                    h5lay = layout.File.load_registered(args.layout, h5repr=None)
+                if hdf_filename is None:
+                    print('No hdf filename. Provide by -f <my_file.hdf>')
+                    return
+                print(f'Checking {hdf_filename} with layout {args.layout}')
+                with h5py.File(hdf_filename) as h5:
+                    h5lay.check(h5)
+                return
+            if args.register:
+                hdf_filename = pathlib.Path(args.register)
+                h5lay = layout.File(hdf_filename)
+                h5lay.register()