dbdreader 0.6.0.dev1__cp313-cp313-win_amd64.whl

dbdreader/decompress.py

@@ -0,0 +1,247 @@
+import os
+from io import BytesIO as ioBytesIO
+from re import search as re_match
+import lz4.block
+
+DECOMPRESSION_ERROR_LIST = ["NO_ERROR",
+                            "ERROR_UNEXPECTED_END_OF_FILE",
+                            "ERROR_FILE_NOT_FOUND",
+                            "ERROR_FAILED_TO_WRITE_BASE_FILE"]
+
+class Decompressor:
+    '''Class to decompress glider files
+
+    Parameters
+    ----------
+    filename : str
+        name of file to decompress
+
+    This class is designed to be used with a context manager
+
+    >>> with Decompressor(filename) as d:
+             data = d.decompress()
+
+    Alternatively, a file can be opened a priori
+
+
+    >>> d = Decompressor()
+    >>> fd = open(filename ,'rb')
+    >>> data = d.decompress(fd)
+    '''
+    SIZEFIELDSIZE = 2
+    ENDIANESS = 'big'
+    COMPRESSION_FACTOR=10
+    CHUNKSIZE = 1024*32
+    def __init__(self, filename=None, fp=None):
+        self.filename = filename
+        self.fp = fp
+    
+        
+    def __enter__(self, *p, **kwds):
+        # Try to open file is self.filename is given, otherwise check wheter self.fp is given.
+        # if not, raise an error.
+        if not self.filename is None:
+            self.fp = open(self.filename,'rb')
+        if self.fp is None:
+            raise ValueError('Supply either a filename or file descriptor to the class constructor.')
+        return self
+
+    def __exit__(self, *p, **kwds):
+        # close the file and leave.
+        self.fp.close()
+        
+    def _decompress_block(self, fp=None):
+        fp = fp or self.fp
+        sb = fp.read(Decompressor.SIZEFIELDSIZE)
+        if sb:
+            size = int.from_bytes(sb, Decompressor.ENDIANESS)
+            b = lz4.block.decompress(fp.read(size), Decompressor.CHUNKSIZE)
+        else:
+            b = None
+        return b
+    
+    def decompressed_blocks(self, n=None, fp=None):
+        ''' Generator that returns decompressed data blocks
+
+        Parameters
+        ----------
+        n : int or None
+            limits the number of blocks read and returned. If None (default) all blocks are returned
+        fp : file descriptor or None
+            file descriptor to use. If None (default), the file descriptor assigned by the constructor is used.
+
+        Yields
+        ------
+        bytes:
+             decompressed data block
+        '''
+        counter = 0
+        if n is None:
+            counter_increment = 0
+            n = 1
+        else:
+            counter_increment = 1
+        while counter < n:
+            block = self._decompress_block(fp)
+            if block is None:
+                break
+            yield block
+            counter+=counter_increment
+
+    def decompress(self, fp=None):
+        ''' Decompresses a an entire file (in memory)
+
+        Parameters
+        ----------
+        fp : file descriptor or None
+            file descriptor to use. If None (default), the file descriptor assigned by the constructor is used.
+
+        Returns
+        -------
+        bytes:
+            decompressed file data as bytes
+        '''
+        fp = fp or self.fp
+        if fp is None:
+            raise ValueError('Supply a file handler or use this class within a context manager')
+        data =b''
+        for block in self.decompressed_blocks(fp=fp):
+            data += block
+        return data
+        
+
+class FileDecompressor:
+    '''Class that provides an easy way to automatically decompress a compressed glider
+       data file and write it into a normal binary data file.
+
+    The factual decompressing is done by decompress method.
+
+    Example
+    -------
+
+    >>> FileDecompressor.decompress("01600000.dcd")
+
+    which would result in the writing of a decompressed file 01600000.dbd.
+    
+    '''
+    def _generate_filename_for_output(self, filename):
+        base, ext = os.path.splitext(filename)
+        if len(ext)!=4:
+            raise ValueError('Unhandled file extension.')
+        if ext.endswith('cg'):
+            s = 'lg'
+        elif ext.endswith('cd'):
+            s = 'bd'
+        elif ext.endswith('cc'):
+            s = 'ac'
+        else:
+            raise ValueError('Unhandled file extension.')
+        return "".join((base, ext[:-2], s))
+                         
+    def decompress(self, filename):
+        ''' Decompresses a file
+
+        Parameters
+        ----------
+        filename : str
+            (compressed) filename
+
+        Returns
+        -------
+        str:
+            uncompressed filename
+
+       '''
+        output_filename = self._generate_filename_for_output(filename)
+        with Decompressor(filename) as d, open(output_filename, 'wb') as fp_out:
+            for block in d.decompressed_blocks():
+                fp_out.write(block)
+        return output_filename
+
+def decompress_file(filename):
+    '''Decompreses a glider data file and writes the normal binary file.'''
+    return FileDecompressor().decompress(filename)
+
+def is_compressed(filename):
+    [basename,ext]=os.path.splitext(filename)
+    # all compressed [demnst]bd files end in [demnst]cd
+    return bool(re_match("[demnst]c[dg]$", ext))
+
+
+class BytesIORW:
+    ''' Helper class implementing a BytesIO buffer that can be written to and read from.
+
+    Note that the methods write() and readline() are implemented only.
+    '''
+    def __init__(self, source):
+        self.bytesIO = ioBytesIO()
+        self.pointer_start = 0
+        self.pointer_end = 0
+        self.source = source
+        
+    def write(self, b):
+        self.pointer_start = self.bytesIO.tell()
+        self.bytesIO.write(b)
+        self.pointer_end = self.bytesIO.tell()
+        self.bytesIO.seek(self.pointer_start)
+        self.is_exhausted = False
+        
+    def readline(self):
+        line = self.bytesIO.readline()
+        pointer = self.bytesIO.tell()
+        if pointer == self.pointer_end:
+            try:
+                self.write(next(self.source))
+            except StopIteration:
+                pass
+            else:
+                line+=self.bytesIO.readline()
+        return line
+        
+
+class CompressedFile:
+    ''' Class to access a compressed file, providing a method
+
+    readline() that returns a decompressed line of data. The compressed
+    file is read block by block, as long as needed.
+
+    The main reason for the class is to be able to read the header of a compressed
+    glider data file.
+    '''
+    
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.decompressor = Decompressor()
+        self.bytesIO = None
+
+    def __enter__(self, *p, **kwds):
+        self.fp = open(self.filename, 'rb')
+        source = self.decompressor.decompressed_blocks(fp=self.fp)
+        self.bytesIO = BytesIORW(source)
+        return self
+
+    
+    def __exit__(self, *p, **kwds):
+        self.fp.close()
+
+    def readline(self):
+        line = self.bytesIO.readline()
+        return line
+    
+    def readlines(self):
+        while True:
+            line = self.readline()
+            if not line:
+                break
+            yield line
+
+    def seek(self, offset):
+        return self.bytesIO.bytesIO.seek(offset)
+
+    def tell(self):
+        return self.bytesIO.bytesIO.tell()
+    
+
+    def close(self):
+        self.fp.close()

dbdreader/scripts.py

@@ -0,0 +1,279 @@
+# Copyright 2006,2007,2014, 2023 Lucas Merckelbach
+#
+# This file is part of dbdreader.
+#
+# dbdreader is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# dbdreader is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+6#
+# You should have received a copy of the GNU General Public License
+# along with dbdreader.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from collections import OrderedDict
+import sys
+import os
+import argparse
+import re
+
+
+
+import dbdreader.decompress 
+
+
+def _makeSortable(filename):
+    [basename,ext]=filename.split('.')
+    [g,y,d,m,s]=basename.split('-')
+    s="%03d"%(int(s))
+    m="%02d"%(int(m))
+    d="%03d"%(int(d))
+    basename="-".join([g,y,d,m,s])
+    filename=".".join([basename,ext])
+    return(filename)
+    
+
+
+
+def _get_short_and_long_filenames(lines, filename):
+    ID=lines[0]
+    ignoreIt=False
+    shortfilename=''
+    longfilename=''
+    if ID=='dbd_label:    DBD(dinkum_binary_data)file':
+        shortfilename=lines[4]
+        longfilename=lines[5]
+        shortfilename=re.sub(r"^.* ","",shortfilename)
+        longfilename=re.sub(r"^.* ","",longfilename)
+        extension=re.sub(r"^.*\.","",filename)
+        shortfilename+="."+extension
+        longfilename+="."+extension
+    elif "the8x3_filename" in ID: # mlg file apparently...
+        longfilename=lines[1]
+        if filename.lower().endswith('mlg'):
+            extension='.mlg'
+        else:
+            extension='.nlg'
+        shortfilename=re.sub(r"^the8x3_filename: +","",ID+extension)
+
+        longfilename=re.sub(r"^full_filename: +","",longfilename)+extension
+    else:
+        sys.stderr.write("Ignoring %s\n"%(filename))
+        ignoreIt=True
+    return ignoreIt, shortfilename, longfilename
+
+def dbdrename():
+    ''' Standalone script to rename dbd files
+
+    Use dbdrename -h for help.
+    '''
+    parser = argparse.ArgumentParser(
+        prog='dbdrename',
+        description='''Program to rename dbd files and friends from numeric format to long format, or vice versa,
+        or convert the long format into a sortable name or the original Webb Research long format,
+        or decompress LZ4 compressed files.''',
+        epilog='')
+
+    parser.add_argument('filenames', nargs="*", help="Filename(s) to process")
+    parser.add_argument('-s', action='store_true', default=True, help='Ensures long format filenames are sortable')
+    parser.add_argument('-n', action='store_true', help='Keeps the original long format filenames (which do not sort correctly).')
+    parser.add_argument('-c', '--convertToSortable', action='store_true', help='Converts files from original long format to a sortable long format')
+    parser.add_argument('-C', '--convertToOriginal', action='store_true', help='Converts files from a sortable long format to the original long format')
+    parser.add_argument('-x', '--decompress', action='store_true', help='Decompresses LZ4 comrpessed files.')
+    parser.add_argument('-X', '--decompressAndRemoveCompressed', action='store_true', help='Decompresses LZ4 comrpessed files, and removes the compressed file')
+    parser.add_argument('-d', '--doNotChangeNameFormat', action='store_true', help='Does not change the filename format. (Only useful in combination with -x or -X)')
+
+
+
+    args = parser.parse_args()
+
+    # override the default value of -s if -d is specified
+    if args.doNotChangeNameFormat:
+        args.s=False
+
+    for i in args.filenames:
+        if not os.path.exists(i):
+            print(f"{i} does not exist. Ignoring.")
+            continue
+        if dbdreader.decompress.is_compressed(i):
+            with dbdreader.decompress.CompressedFile(i) as fd:
+                lines = [fd.readline().decode('ascii').strip() for j in range(7)]
+        else:
+            with open(i,'br') as fd:
+                lines = [fd.readline().decode('ascii').strip() for j in range(7)]
+        ignoreIt, shortfilename, longfilename = _get_short_and_long_filenames(lines, i)
+
+        basename = os.path.basename(i)
+        prefix = i.replace(basename, "")
+
+        if not ignoreIt:
+            command=None
+            if args.convertToSortable or args.convertToOriginal: # input filename must be the longfilename
+                # check if we have a longfilename
+                if i not in [longfilename,_makeSortable(longfilename)]:
+                    print("Chose to ignore processing ",i)
+                else:
+                    if i==_makeSortable(longfilename) and args.convertToOriginal:
+                        # switch to old format
+                        command="mv "+i+" "+longfilename
+                        new_filename = longfilename
+                    elif i==longfilename and args.convertToSortable:
+                        command="mv "+longfilename+" "+_makeSortable(longfilename)
+                        new_filename = _makeSortable(longfilename)
+                    else:
+                        print("ignoring "+i)
+            else: # changing from long to short names or vice versa
+                if not args.n and not args.s: # -d has been requested
+                    new_filename = i
+                    if args.decompress or args.decompressAndRemoveCompressed:
+                        command = ""
+                else: # create the new_filename by translating
+                    if args.s and not args.n:
+                        longfilename=_makeSortable(longfilename)
+                    if basename==shortfilename:
+                        new_filename = os.path.join(prefix, longfilename)
+                        old_filename = os.path.join(prefix, shortfilename)
+                        command = f"mv {old_filename} {new_filename}"
+                    else:
+                        new_filename = os.path.join(prefix, shortfilename)
+                        old_filename = os.path.join(prefix, longfilename)
+                        command = f"mv {old_filename} {new_filename}"
+
+            if command!=None:
+                target = new_filename
+                if command:
+                    R=os.system(command)
+                else:
+                    R=0
+                msg = f"{i} ->"
+                if R!=0:
+                    raise ValueError("Could not execute %s"%(command))
+                if args.decompress or args.decompressAndRemoveCompressed:
+                    try:
+                        target = dbdreader.decompress.decompress_file(new_filename)
+                    except Exception as e:
+                        msg = e
+                    else:
+                        if args.decompressAndRemoveCompressed:
+                            os.unlink(new_filename)
+                            msg = f"{msg} {target}"
+                        else:
+                            msg = f"{msg} {target}/{new_filename}"
+                else:
+                    msg = f"{msg} {target}"
+                print(msg)
+
+
+''' =====================================================================================================
+
+    cac_gen
+
+    ===================================================================================================== '''
+
+
+def _parse_line(line):
+    fields=line.rstrip().split()
+    s=dict(flag=fields[1],
+           global_index=int(fields[2]),
+           index=int(fields[3]),
+           bytesize=int(fields[4]),
+           name=fields[5],
+           unit=fields[6])
+    return s['name'],s
+
+def _read_cac_file(fn):
+    sensors=OrderedDict()
+    fd=open(fn,'r')
+    while True:
+        line=fd.readline()
+        if not line:
+            break
+        n,s=_parse_line(line)
+        sensors[n]=s
+    fd.close()
+    return sensors
+
+def _read_mbdlist(fn):
+    params=[]
+    fd=open(fn,'r')
+    while True:
+        line=fd.readline()
+        if not line:
+            break
+        line=line.rstrip().lstrip()
+        if not line:
+            continue # empty line
+        if line.startswith("#"):
+            continue
+        s=line.lower().split()
+        params.append(s[0])
+    fd.close()
+    return params
+        
+def _generate_cac(s,params):
+    # sort params:
+    params_in_order=[]
+    for k in list(s.keys()):
+        if k in params:
+            params_in_order.append(k)
+    # set all indices to -1
+    for v in s.values():
+        v['index']=-1
+        v['flag']='F'
+    for i,p in enumerate(params_in_order):
+        s[p]['index']=i
+        s[p]['flag']='T'
+    return s
+
+def _write_cac(s,fd):
+    for k,v in s.items():
+        fd.write("s: %s%5d%5d%2d %s %s\n"%(v['flag'],
+                                           v['global_index'],
+                                           v['index'],
+                                           v['bytesize'],
+                                           v['name'],
+                                           v['unit']))
+
+
+def _generate_cac_file(mbdlist_filename,template,output):
+    s=_read_cac_file(template)
+    params=_read_mbdlist(mbdlist_filename)
+    s=_generate_cac(s,params)
+    if output:
+        fd=open(output,'w')
+        _write_cac(s,fd)
+        fd.close()
+    else:
+        _write_cac(s,sys.stdout)
+
+
+def cac_gen():
+    ''' A script to generate cac files
+
+    run cac_gen -h for more help.
+    '''
+    parser = argparse.ArgumentParser(
+        prog='cac_gen',
+        description='''A program to generate a valid cache file from a .dat file and a valid cache file for this
+        glider.
+
+        For example, if you have a cache file for an sbd file, and the sbdlist.dat got modified, then this programs
+        let's you build a new cache file using the old cache file and new sdblist.dat file as input.
+        ''',
+        epilog='')
+
+    parser.add_argument('sdblist',  help="sdblist.dat file, specifying which parameters are present in the binary files")
+    parser.add_argument('cachefile', help="A valid and exisiting cache file for this glider.")
+    parser.add_argument('outputfile', nargs="?", default=None, help="Output filename (writes to stdout if not speficied.)")
+
+    args = parser.parse_args()
+    _generate_cac_file(args.sdblist, args.cachefile, args.outputfile)
+
+
+
+