TSVZ 2.70__py3-none-any.whl → 3.10__py3-none-any.whl

TSVZ.py

@@ -4,59 +4,91 @@ from collections import OrderedDict , deque
 import time
 import atexit
 import threading
+import re
 
 if os.name == 'nt':
     import msvcrt
 elif os.name == 'posix':
     import fcntl
 
-version = '2.70'
+version = '3.10'
 author = 'pan@zopyr.us'
 
+DEFAULT_DELIMITER = '\t'
+DEFAULTS_INDICATOR_KEY = '#_defaults_#'
 
-def pretty_format_table(data):
-	version = 1.0
-	if not data:
-		return ''
-	if type(data) == str:
-		data = data.strip('\n').split('\n')
-		data = [line.split('\t') for line in data]
-	elif isinstance(data, dict):
-		# flatten the 2D dict to a list of lists
-		if isinstance(next(iter(data.values())), dict):
-			tempData = [['key'] + list(next(iter(data.values())).keys())]
-			tempData.extend( [[key] + list(value.values()) for key, value in data.items()])
-			data = tempData
-		else:
-			# it is a dict of lists
-			data = [[key] + list(value) for key, value in data.items()]
-	elif type(data) != list:
-		data = list(data)
-	# format the list into 2d list of list of strings
-	if isinstance(data[0], dict):
-		tempData = [data[0].keys()]
-		tempData.extend([list(item.values()) for item in data])
-		data = tempData
-	data = [[str(item) for item in row] for row in data]
-	num_cols = len(data[0])
-	col_widths = [0] * num_cols
-	# Calculate the maximum width of each column
-	for c in range(num_cols):
-		col_widths[c] = max(len(row[c]) for row in data)
-	# Build the row format string
-	row_format = ' | '.join('{{:<{}}}'.format(width) for width in col_widths)
-	# Print the header
-	header = data[0]
-	outTable = []
-	outTable.append(row_format.format(*header))
-	outTable.append('-+-'.join('-' * width for width in col_widths))
-	for row in data[1:]:
-		# if the row is empty, print an divider
-		if not any(row):
-			outTable.append('-+-'.join('-' * width for width in col_widths))
-		else:
-			outTable.append(row_format.format(*row))
-	return '\n'.join(outTable) + '\n'
+def get_delimiter(delimiter,file_name = ''):
+    if not delimiter:
+        return DEFAULT_DELIMITER
+    elif delimiter == ...:
+        if not file_name:
+            rtn =  '\t'
+        if file_name.endswith('.csv'):
+            rtn =  ','
+        elif file_name.endswith('.nsv'):
+            rtn =  '\0'
+        elif file_name.endswith('.psv'):
+            rtn =  '|'
+        else:
+            rtn =  '\t'
+    elif delimiter == 'comma':
+        rtn =  ','
+    elif delimiter == 'tab':
+        rtn =  '\t'
+    elif delimiter == 'pipe':
+        rtn =  '|'
+    elif delimiter == 'null':
+        rtn =  '\0'
+    else:
+        rtn =  delimiter.encode().decode('unicode_escape')
+    DEFAULT_DELIMITER = rtn
+    return rtn
+
+def pretty_format_table(data, delimiter = DEFAULT_DELIMITER):
+    version = 1.0
+    if not data:
+        return ''
+    if type(data) == str:
+        data = data.strip('\n').split('\n')
+        data = [line.split(delimiter) for line in data]
+    elif isinstance(data, dict):
+        # flatten the 2D dict to a list of lists
+        if isinstance(next(iter(data.values())), dict):
+            tempData = [['key'] + list(next(iter(data.values())).keys())]
+            tempData.extend( [[key] + list(value.values()) for key, value in data.items()])
+            data = tempData
+        else:
+            # it is a dict of lists
+            data = [[key] + list(value) for key, value in data.items()]
+    elif type(data) != list:
+        data = list(data)
+    # format the list into 2d list of list of strings
+    if isinstance(data[0], dict):
+        tempData = [data[0].keys()]
+        tempData.extend([list(item.values()) for item in data])
+        data = tempData
+    data = [[str(item) for item in row] for row in data]
+    num_cols = len(data[0])
+    col_widths = [0] * num_cols
+    # Calculate the maximum width of each column
+    for c in range(num_cols):
+        #col_widths[c] = max(len(row[c]) for row in data)
+        # handle ansii escape sequences
+        col_widths[c] = max(len(re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]','',row[c])) for row in data)
+    # Build the row format string
+    row_format = ' | '.join('{{:<{}}}'.format(width) for width in col_widths)
+    # Print the header
+    header = data[0]
+    outTable = []
+    outTable.append(row_format.format(*header))
+    outTable.append('-+-'.join('-' * width for width in col_widths))
+    for row in data[1:]:
+        # if the row is empty, print an divider
+        if not any(row):
+            outTable.append('-+-'.join('-' * width for width in col_widths))
+        else:
+            outTable.append(row_format.format(*row))
+    return '\n'.join(outTable) + '\n'
 
 def __teePrintOrNot(message,level = 'info',teeLogger = None):
     """
@@ -78,7 +110,7 @@ def __teePrintOrNot(message,level = 'info',teeLogger = None):
     except Exception as e:
         print(message,flush=True)
 
-def processLine(line,taskDic,correctColumnNum,verbose = False,teeLogger = None,strict = True):
+def _processLine(line,taskDic,correctColumnNum,verbose = False,teeLogger = None,strict = True,delimiter = DEFAULT_DELIMITER,defaults = []):
     """
     Process a line of text and update the task dictionary.
 
@@ -89,47 +121,52 @@ def processLine(line,taskDic,correctColumnNum,verbose = False,teeLogger = None,s
     verbose (bool, optional): Whether to print verbose output. Defaults to False.
     teeLogger (object, optional): The tee logger object for printing output. Defaults to None.
     strict (bool, optional): Whether to strictly enforce the correct number of columns. Defaults to True.
+    defaults (list, optional): The default values to use for missing columns. Defaults to [].
 
     Returns:
     tuple: A tuple containing the updated correctColumnNum and the processed lineCache.
 
     """
-    line = line.decode().strip(' ').strip('\x00')
+    line = line.strip(' ').strip('\x00').rstrip('\r\n')
     # we throw away the lines that start with '#'
     if not line :
         if verbose:
             __teePrintOrNot(f"Ignoring empty line: {line}",teeLogger=teeLogger)
         return correctColumnNum , []
-    if line.startswith('#'):
+    if line.startswith('#') and not line.startswith(DEFAULTS_INDICATOR_KEY):
         if verbose:
             __teePrintOrNot(f"Ignoring comment line: {line}",teeLogger=teeLogger)
         return correctColumnNum , []
     # we only interested in the lines that have the correct number of columns
-    lineCache = [segment.strip() for segment in line.split('\t')]
+    lineCache = [segment.rstrip() for segment in line.split(delimiter)]
     if not lineCache:
         return correctColumnNum , []
     if correctColumnNum == -1:
+        if defaults and len(defaults) > 1:
+            correctColumnNum = len(defaults)
+        else:
+            correctColumnNum = len(lineCache)
         if verbose:
             __teePrintOrNot(f"detected correctColumnNum: {len(lineCache)}",teeLogger=teeLogger)
-        correctColumnNum = len(lineCache)
     if not lineCache[0]:
         if verbose:
             __teePrintOrNot(f"Ignoring line with empty key: {line}",teeLogger=teeLogger)
         return correctColumnNum , []
     if len(lineCache) == 1 or not any(lineCache[1:]):
-        if correctColumnNum == 1: taskDic[lineCache[0]] = lineCache
+        if correctColumnNum == 1: 
+            taskDic[lineCache[0]] = lineCache
+        elif lineCache[0] == DEFAULTS_INDICATOR_KEY:
+            if verbose:
+                __teePrintOrNot(f"Empty defaults line found: {line}",teeLogger=teeLogger)
+            defaults = []
         else:
             if verbose:
                 __teePrintOrNot(f"Key {lineCache[0]} found with empty value, deleting such key's representaion",teeLogger=teeLogger)
             if lineCache[0] in taskDic:
                 del taskDic[lineCache[0]]
         return correctColumnNum , []
-    elif len(lineCache) == correctColumnNum:
-        taskDic[lineCache[0]] = lineCache
-        if verbose:
-            __teePrintOrNot(f"Key {lineCache[0]} added",teeLogger=teeLogger)
-    else:
-        if strict:
+    elif len(lineCache) != correctColumnNum:
+        if strict and not any(defaults):
             if verbose:
                 __teePrintOrNot(f"Ignoring line with {len(lineCache)} columns: {line}",teeLogger=teeLogger)
             return correctColumnNum , []
@@ -139,12 +176,26 @@ def processLine(line,taskDic,correctColumnNum,verbose = False,teeLogger = None,s
                 lineCache += ['']*(correctColumnNum-len(lineCache))
             elif len(lineCache) > correctColumnNum:
                 lineCache = lineCache[:correctColumnNum]
-            taskDic[lineCache[0]] = lineCache
             if verbose:
-                __teePrintOrNot(f"Key {lineCache[0]} added after correction",teeLogger=teeLogger)
+                __teePrintOrNot(f"Correcting {lineCache[0]}",teeLogger=teeLogger)
+    # now replace empty values with defaults
+    if defaults and len(defaults) > 1:
+        for i in range(1,len(lineCache)):
+            if not lineCache[i] and i < len(defaults) and defaults[i]:
+                lineCache[i] = defaults[i]
+                if verbose:
+                    __teePrintOrNot(f"Replacing empty value at {i} with default: {defaults[i]}",teeLogger=teeLogger)
+    if lineCache[0] == DEFAULTS_INDICATOR_KEY:
+        if verbose:
+            __teePrintOrNot(f"Defaults line found: {line}",teeLogger=teeLogger)
+        defaults = lineCache
+        return correctColumnNum , []
+    taskDic[lineCache[0]] = lineCache
+    if verbose:
+        __teePrintOrNot(f"Key {lineCache[0]} added",teeLogger=teeLogger)
     return correctColumnNum, lineCache
 
-def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None, strict=False):
+def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None, strict=False,encoding = 'utf8',delimiter = ...,defaults = []):
     """
     Reads the last valid line from a file.
 
@@ -154,13 +205,17 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
         correctColumnNum (int): A column number to pass to processLine function.
         verbose (bool, optional): Whether to print verbose output. Defaults to False.
         teeLogger (optional): Logger to use for tee print. Defaults to None.
+        encoding (str, optional): The encoding of the file. Defaults to None.
         strict (bool, optional): Whether to enforce strict processing. Defaults to False.
+        delimiter (str, optional): The delimiter used in the file. Defaults to None.
+        defaults (list, optional): The default values to use for missing columns. Defaults to [].
 
     Returns:
         list: The last valid line data processed by processLine, or an empty list if none found.
     """
     chunk_size = 1024  # Read in chunks of 1024 bytes
     last_valid_line = []
+    delimiter = get_delimiter(delimiter,file_name=fileName)
     if verbose:
         __teePrintOrNot(f"Reading last line only from {fileName}",teeLogger=teeLogger)
     with open(fileName, 'rb') as file:
@@ -186,13 +241,15 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
             for i in range(len(lines) - 1, -1, -1):
                 if lines[i].strip():  # Skip empty lines
                     # Process the line
-                    correctColumnNum, lineCache = processLine(
-                        lines[i],
-                        taskDic,
-                        correctColumnNum,
+                    correctColumnNum, lineCache = _processLine(
+                        line=lines[i].decode(encoding=encoding),
+                        taskDic=taskDic,
+                        correctColumnNum=correctColumnNum,
                         verbose=verbose,
                         teeLogger=teeLogger,
-                        strict=strict
+                        strict=strict,
+                        delimiter=delimiter,
+                        defaults=defaults,
                     )
                     # If the line is valid, return it
                     if lineCache and any(lineCache):
@@ -204,7 +261,7 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
     # Return empty list if no valid line found
     return last_valid_line
 
-def formatHeader(header,verbose = False,teeLogger = None):
+def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DELIMITER):
     """
     Format the header string.
 
@@ -218,12 +275,12 @@ def formatHeader(header,verbose = False,teeLogger = None):
     """
     if type(header) != str:
         try:
-            header = '\t'.join(header)
+            header = delimiter.join(header)
         except:
             if verbose:
                 __teePrintOrNot('Invalid header, setting header to empty.','error',teeLogger=teeLogger)
             header = ''
-    header = header.strip()
+    header = delimiter.join([segment.rstrip() for segment in header.split(delimiter)])
     # if header:
     #     if not header.endswith('\n'):
     #         header += '\n'
@@ -231,7 +288,7 @@ def formatHeader(header,verbose = False,teeLogger = None):
     #     header = ''
     return header
 
-def lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False):
+def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False,delimiter = DEFAULT_DELIMITER):
     """
     Verify if a line contains the header.
 
@@ -245,26 +302,24 @@ def lineContainHeader(header,line,verbose = False,teeLogger = None,strict = Fals
     Returns:
     bool: True if the header matches the line, False otherwise.
     """
-    escapedHeader = repr(header.strip())
-    escapedLine = repr(line.strip())
+    header = [segment.rstrip() for segment in header.split(delimiter)]
+    line = [segment.rstrip() for segment in line.split(delimiter)]
     if verbose:
-        __teePrintOrNot(f"Header: \n{escapedHeader}",teeLogger=teeLogger)
-        __teePrintOrNot(f"First line: \n{escapedLine}",teeLogger=teeLogger)
-    headerList = header.strip().lower().split('\t')
-    lineList = line.strip().lower().split('\t')
-    if len(headerList) != len(lineList) or any([headerList[i] not in lineList[i] for i in range(len(headerList))]):
-        __teePrintOrNot(f"Header mismatch: \n{escapedLine} \n!= \n{escapedHeader}",teeLogger=teeLogger)
+        __teePrintOrNot(f"Header: \n{header}",teeLogger=teeLogger)
+        __teePrintOrNot(f"First line: \n{line}",teeLogger=teeLogger)
+    if len(header) != len(line) or any([header[i] not in line[i] for i in range(len(header))]):
+        __teePrintOrNot(f"Header mismatch: \n{line} \n!= \n{header}",teeLogger=teeLogger)
         if strict:
             raise Exception("Data format error! Header mismatch")
         return False
     return True
 
-def verifyTSVExistence(fileName,createIfNotExist = True,teeLogger = None,header = '',encoding = 'utf8',strict = True):
+def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,header = '',encoding = 'utf8',strict = True,delimiter = DEFAULT_DELIMITER):
     """
-    Verify the existence of a TSV file.
+    Verify the existence of the tabular file.
 
     Parameters:
-    - fileName (str): The path of the TSV file.
+    - fileName (str): The path of the tabular file.
     - createIfNotExist (bool, optional): Whether to create the file if it doesn't exist. Defaults to True.
     - teeLogger (object, optional): The tee logger object for printing output. Defaults to None.
     - header (str, optional): The header line to verify against. Defaults to ''.
@@ -274,8 +329,14 @@ def verifyTSVExistence(fileName,createIfNotExist = True,teeLogger = None,header
     Returns:
     bool: True if the file exists, False otherwise.
     """
-    if not fileName.endswith('.tsv'):
+    if delimiter and delimiter == '\t' and not fileName.endswith('.tsv'):
         __teePrintOrNot(f'Warning: Filename {fileName} does not end with .tsv','warning',teeLogger=teeLogger)
+    elif delimiter and delimiter == ',' and not fileName.endswith('.csv'):
+        __teePrintOrNot(f'Warning: Filename {fileName} does not end with .csv','warning',teeLogger=teeLogger)
+    elif delimiter and delimiter == '\0' and not fileName.endswith('.nsv'):
+        __teePrintOrNot(f'Warning: Filename {fileName} does not end with .nsv','warning',teeLogger=teeLogger)
+    elif delimiter and delimiter == '|' and not fileName.endswith('.psv'):
+        __teePrintOrNot(f'Warning: Filename {fileName} does not end with .psv','warning',teeLogger=teeLogger)
     if not os.path.isfile(fileName):
         if createIfNotExist:
             with open(fileName, mode ='w',encoding=encoding)as file:
@@ -289,14 +350,15 @@ def verifyTSVExistence(fileName,createIfNotExist = True,teeLogger = None,header
             return False
     return True
 
-def readTSV(fileName,teeLogger = None,header = '',createIfNotExist = False, lastLineOnly = False,verifyHeader = True,verbose = False,taskDic = None,encoding = 'utf8',strict = True):
+def readTSV(fileName,teeLogger = None,header = '',createIfNotExist = False, lastLineOnly = False,verifyHeader = True,verbose = False,taskDic = None,encoding = 'utf8',strict = True,delimiter = '\t',defaults = []):
     """
-    Read a TSV (Tab-Separated Values) file and return the data as a dictionary.
+    Compatibility method, calls readTabularFile. 
+    Read a Tabular (CSV / TSV / NSV) file and return the data as a dictionary.
 
     Parameters:
-    - fileName (str): The path to the TSV file.
+    - fileName (str): The path to the Tabular file.
     - teeLogger (Logger, optional): The logger object to log messages. Defaults to None.
-    - header (str or list, optional): The header of the TSV file. If a string, it should be a tab-separated list of column names. If a list, it should contain the column names. Defaults to ''.
+    - header (str or list, optional): The header of the Tabular file. If a string, it should be a tab-separated list of column names. If a list, it should contain the column names. Defaults to ''.
     - createIfNotExist (bool, optional): Whether to create the file if it doesn't exist. Defaults to False.
     - lastLineOnly (bool, optional): Whether to read only the last valid line of the file. Defaults to False.
     - verifyHeader (bool, optional): Whether to verify the header of the file. Defaults to True.
@@ -304,9 +366,38 @@ def readTSV(fileName,teeLogger = None,header = '',createIfNotExist = False, last
     - taskDic (OrderedDict, optional): The dictionary to store the data. Defaults to an empty OrderedDict.
     - encoding (str, optional): The encoding of the file. Defaults to 'utf8'.
     - strict (bool, optional): Whether to raise an exception if there is a data format error. Defaults to True.
+    - delimiter (str, optional): The delimiter used in the Tabular file. Defaults to '\t'.
+    - defaults (list, optional): The default values to use for missing columns. Defaults to [].
 
     Returns:
-    - OrderedDict: The dictionary containing the data from the TSV file.
+    - OrderedDict: The dictionary containing the data from the Tabular file.
+
+    Raises:
+    - Exception: If the file is not found or there is a data format error.
+
+    """
+    return readTabularFile(fileName,teeLogger = teeLogger,header = header,createIfNotExist = createIfNotExist,lastLineOnly = lastLineOnly,verifyHeader = verifyHeader,verbose = verbose,taskDic = taskDic,encoding = encoding,strict = strict,delimiter = delimiter,defaults=defaults)
+
+def readTabularFile(fileName,teeLogger = None,header = '',createIfNotExist = False, lastLineOnly = False,verifyHeader = True,verbose = False,taskDic = None,encoding = 'utf8',strict = True,delimiter = ...,defaults = []):
+    """
+    Read a Tabular (CSV / TSV / NSV) file and return the data as a dictionary.
+
+    Parameters:
+    - fileName (str): The path to the Tabular file.
+    - teeLogger (Logger, optional): The logger object to log messages. Defaults to None.
+    - header (str or list, optional): The header of the Tabular file. If a string, it should be a tab-separated list of column names. If a list, it should contain the column names. Defaults to ''.
+    - createIfNotExist (bool, optional): Whether to create the file if it doesn't exist. Defaults to False.
+    - lastLineOnly (bool, optional): Whether to read only the last valid line of the file. Defaults to False.
+    - verifyHeader (bool, optional): Whether to verify the header of the file. Defaults to True.
+    - verbose (bool, optional): Whether to print verbose output. Defaults to False.
+    - taskDic (OrderedDict, optional): The dictionary to store the data. Defaults to an empty OrderedDict.
+    - encoding (str, optional): The encoding of the file. Defaults to 'utf8'.
+    - strict (bool, optional): Whether to raise an exception if there is a data format error. Defaults to True.
+    - delimiter (str, optional): The delimiter used in the Tabular file. Defaults to '\t' for TSV, ',' for CSV, '\0' for NSV.
+    - defaults (list, optional): The default values to use for missing columns. Defaults to [].
+
+    Returns:
+    - OrderedDict: The dictionary containing the data from the Tabular file.
 
     Raises:
     - Exception: If the file is not found or there is a data format error.
@@ -314,33 +405,55 @@ def readTSV(fileName,teeLogger = None,header = '',createIfNotExist = False, last
     """
     if taskDic is None:
         taskDic = {}
-    header = formatHeader(header,verbose = verbose,teeLogger = teeLogger)
-    if not verifyTSVExistence(fileName,createIfNotExist = createIfNotExist,teeLogger = teeLogger,header = header,encoding = encoding,strict = strict):
+    delimiter = get_delimiter(delimiter,file_name=fileName)
+    header = _formatHeader(header,verbose = verbose,teeLogger = teeLogger, delimiter = delimiter)
+    if not _verifyFileExistence(fileName,createIfNotExist = createIfNotExist,teeLogger = teeLogger,header = header,encoding = encoding,strict = strict,delimiter=delimiter):
         return taskDic
     with open(fileName, mode ='rb')as file:
         correctColumnNum = -1
-        if header.strip():
+        if header.rstrip():
             if verifyHeader:
-                line = file.readline().decode().strip()
-                if lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
-                    correctColumnNum = len(header.strip().split('\t'))
+                line = file.readline().decode(encoding=encoding)
+                if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+                    correctColumnNum = len(header.split(delimiter))
                     if verbose:
                         __teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
         if lastLineOnly:
-            lineCache = read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=verbose, teeLogger=teeLogger, strict=strict)
+            lineCache = read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=verbose, teeLogger=teeLogger, strict=strict, delimiter=delimiter, defaults=defaults)
             if lineCache:
                 taskDic[lineCache[0]] = lineCache
             return lineCache
         for line in file:
-            correctColumnNum, lineCache = processLine(line,taskDic,correctColumnNum,verbose = verbose,teeLogger = teeLogger,strict = strict)
+            correctColumnNum, lineCache = _processLine(line.decode(encoding=encoding),taskDic,correctColumnNum,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter=delimiter,defaults = defaults)
     return taskDic
 
-def appendTSV(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExist = False,verifyHeader = True,verbose = False,encoding = 'utf8', strict = True):
+def appendTSV(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExist = False,verifyHeader = True,verbose = False,encoding = 'utf8', strict = True, delimiter = '\t'):
+    """
+    Compatibility method, calls appendTabularFile.
+    Append a line of data to a Tabular file.
+    Parameters:
+    - fileName (str): The path of the Tabular file.
+    - lineToAppend (str or list): The line of data to append. If it is a string, it will be split by delimiter to form a list.
+    - teeLogger (optional): A logger object for logging messages.
+    - header (str, optional): The header line to verify against. If provided, the function will check if the existing header matches the provided header.
+    - createIfNotExist (bool, optional): If True, the file will be created if it does not exist. If False and the file does not exist, an exception will be raised.
+    - verifyHeader (bool, optional): If True, the function will verify if the existing header matches the provided header. If False, the header will not be verified.
+    - verbose (bool, optional): If True, additional information will be printed during the execution.
+    - encoding (str, optional): The encoding of the file.
+    - strict (bool, optional): If True, the function will raise an exception if there is a data format error. If False, the function will ignore the error and continue.
+    - delimiter (str, optional): The delimiter used in the Tabular file. Defaults to '\t' for TSV, ',' for CSV, '\0' for NSV.
+    Raises:
+    - Exception: If the file does not exist and createIfNotExist is False.
+    - Exception: If the existing header does not match the provided header.
+    """
+    return appendTabularFile(fileName,lineToAppend,teeLogger = teeLogger,header = header,createIfNotExist = createIfNotExist,verifyHeader = verifyHeader,verbose = verbose,encoding = encoding, strict = strict, delimiter = delimiter)
+
+def appendTabularFile(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExist = False,verifyHeader = True,verbose = False,encoding = 'utf8', strict = True, delimiter = ...):
     """
-    Append a line of data to a TSV file.
+    Append a line of data to a Tabular file.
     Parameters:
-    - fileName (str): The path of the TSV file.
-    - lineToAppend (str or list): The line of data to append. If it is a string, it will be split by tabs ('\t') to form a list.
+    - fileName (str): The path of the Tabular file.
+    - lineToAppend (str or list): The line of data to append. If it is a string, it will be split by delimiter to form a list.
     - teeLogger (optional): A logger object for logging messages.
     - header (str, optional): The header line to verify against. If provided, the function will check if the existing header matches the provided header.
     - createIfNotExist (bool, optional): If True, the file will be created if it does not exist. If False and the file does not exist, an exception will be raised.
@@ -348,15 +461,17 @@ def appendTSV(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExis
     - verbose (bool, optional): If True, additional information will be printed during the execution.
     - encoding (str, optional): The encoding of the file.
     - strict (bool, optional): If True, the function will raise an exception if there is a data format error. If False, the function will ignore the error and continue.
+    - delimiter (str, optional): The delimiter used in the Tabular file. Defaults to '\t' for TSV, ',' for CSV, '\0' for NSV.
     Raises:
     - Exception: If the file does not exist and createIfNotExist is False.
     - Exception: If the existing header does not match the provided header.
     """
-    header = formatHeader(header,verbose = verbose,teeLogger = teeLogger)
-    if not verifyTSVExistence(fileName,createIfNotExist = createIfNotExist,teeLogger = teeLogger,header = header,encoding = encoding,strict = strict):
+    delimiter = get_delimiter(delimiter,file_name=fileName)
+    header = _formatHeader(header,verbose = verbose,teeLogger = teeLogger,delimiter=delimiter)
+    if not _verifyFileExistence(fileName,createIfNotExist = createIfNotExist,teeLogger = teeLogger,header = header,encoding = encoding,strict = strict,delimiter=delimiter):
         return
     if type(lineToAppend) == str:
-        lineToAppend = lineToAppend.strip().split('\t')
+        lineToAppend = lineToAppend.split(delimiter)
     else:
         for i in range(len(lineToAppend)):
             if type(lineToAppend[i]) != str:
@@ -367,11 +482,11 @@ def appendTSV(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExis
     
     with open(fileName, mode ='r+b')as file:
         correctColumnNum = len(lineToAppend)
-        if header.strip():
+        if header.rstrip():
             if verifyHeader:
-                line = file.readline().decode().strip()
-                if lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
-                    correctColumnNum = len(header.strip().split('\t'))
+                line = file.readline().decode(encoding=encoding)
+                if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+                    correctColumnNum = len(header.split(delimiter))
                     if verbose:
                         __teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
         # truncate / fill the lineToAppend to the correct number of columns
@@ -383,15 +498,16 @@ def appendTSV(fileName,lineToAppend,teeLogger = None,header = '',createIfNotExis
         file.seek(-1, os.SEEK_END)
         if file.read(1) != b'\n':
             file.write(b'\n')
-        file.write('\t'.join(lineToAppend).encode() + b'\n')
+        file.write(get_delimiter(delimiter).join(lineToAppend).encode(encoding=encoding) + b'\n')
         if verbose:
             __teePrintOrNot(f"Appended {lineToAppend} to {fileName}",teeLogger=teeLogger)
 
-def clearTSV(fileName,teeLogger = None,header = '',verifyHeader = False,verbose = False,encoding = 'utf8',strict = False):
+def clearTSV(fileName,teeLogger = None,header = '',verifyHeader = False,verbose = False,encoding = 'utf8',strict = False,delimiter = '\t'):
     """
-    Clear the contents of a TSV file. Will create if not exist.
+    Compatibility method, calls clearTabularFile.
+    Clear the contents of a Tabular file. Will create if not exist.
     Parameters:
-    - fileName (str): The path of the TSV file.
+    - fileName (str): The path of the Tabular file.
     - teeLogger (optional): A logger object for logging messages.
     - header (str, optional): The header line to verify against. If provided, the function will check if the existing header matches the provided header.
     - verifyHeader (bool, optional): If True, the function will verify if the existing header matches the provided header. If False, the header will not be verified.
@@ -399,14 +515,29 @@ def clearTSV(fileName,teeLogger = None,header = '',verifyHeader = False,verbose
     - encoding (str, optional): The encoding of the file.
     - strict (bool, optional): If True, the function will raise an exception if there is a data format error. If False, the function will ignore the error and continue.
     """
-    header = formatHeader(header,verbose = verbose,teeLogger = teeLogger)
-    if not verifyTSVExistence(fileName,createIfNotExist = True,teeLogger = teeLogger,header = header,encoding = encoding,strict = False):
+    return clearTabularFile(fileName,teeLogger = teeLogger,header = header,verifyHeader = verifyHeader,verbose = verbose,encoding = encoding,strict = strict,delimiter = delimiter)
+
+def clearTabularFile(fileName,teeLogger = None,header = '',verifyHeader = False,verbose = False,encoding = 'utf8',strict = False,delimiter = ...):
+    """
+    Clear the contents of a Tabular file. Will create if not exist.
+    Parameters:
+    - fileName (str): The path of the Tabular file.
+    - teeLogger (optional): A logger object for logging messages.
+    - header (str, optional): The header line to verify against. If provided, the function will check if the existing header matches the provided header.
+    - verifyHeader (bool, optional): If True, the function will verify if the existing header matches the provided header. If False, the header will not be verified.
+    - verbose (bool, optional): If True, additional information will be printed during the execution.
+    - encoding (str, optional): The encoding of the file.
+    - strict (bool, optional): If True, the function will raise an exception if there is a data format error. If False, the function will ignore the error and continue.
+    """
+    delimiter = get_delimiter(delimiter,file_name=fileName)
+    header = _formatHeader(header,verbose = verbose,teeLogger = teeLogger,delimiter=delimiter)
+    if not _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = teeLogger,header = header,encoding = encoding,strict = False,delimiter=delimiter):
         raise Exception("Something catastrophic happened! File still not found after creation")
     else:
         with open(fileName, mode ='r+',encoding=encoding)as file:
-            if header.strip() and verifyHeader:
-                line = file.readline().strip()
-                if not lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+            if header.rstrip() and verifyHeader:
+                line = file.readline()
+                if not _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
                     __teePrintOrNot(f'Warning: Header mismatch in {fileName}. Keeping original header in file...','warning',teeLogger)
                 file.truncate()
             else:
@@ -442,14 +573,17 @@ class TSVZed(OrderedDict):
         except Exception as e:
             print(message,flush=True)
 
-    def __init__ (self,fileName,teeLogger = None,header = '',createIfNotExist = True,verifyHeader = True,rewrite_on_load = True,rewrite_on_exit = False,rewrite_interval = 0, append_check_delay = 0.01,monitor_external_changes = True,verbose = False,encoding = None):
+    def __init__ (self,fileName,teeLogger = None,header = '',createIfNotExist = True,verifyHeader = True,rewrite_on_load = True,rewrite_on_exit = False,rewrite_interval = 0, append_check_delay = 0.01,monitor_external_changes = True,verbose = False,encoding = 'utf8',delimiter = ...,defualts = [],strict = False):
         super().__init__()
         self.version = version
+        self.strict = strict
         self.externalFileUpdateTime = getFileUpdateTimeNs(fileName)
         self.lastUpdateTime = self.externalFileUpdateTime
         self._fileName = fileName
         self.teeLogger = teeLogger
-        self.header = formatHeader(header,verbose = verbose,teeLogger = self.teeLogger)
+        self.delimiter = get_delimiter(delimiter,file_name=fileName)
+        self.defaults = defualts
+        self.header = _formatHeader(header,verbose = verbose,teeLogger = self.teeLogger,delimiter=self.delimiter)
         self.correctColumnNum = -1
         self.createIfNotExist = createIfNotExist
         self.verifyHeader = verifyHeader
@@ -477,6 +611,27 @@ class TSVZed(OrderedDict):
         self.load()
         atexit.register(self.stopAppendThread)
 
+    def setDefaults(self,defaults):
+        if not defaults:
+            defaults = []
+            return
+        if isinstance(defaults,str):
+            defaults = defaults.split(self.delimiter)
+        elif not isinstance(defaults,list):
+            try:
+                defaults = list(defaults)
+            except:
+                if self.verbose:
+                    self.__teePrintOrNot('Invalid defaults, setting defaults to empty.','error')
+                defaults = []
+                return
+        if not any(defaults):
+            defaults = []
+            return
+        if defaults[0] != DEFAULTS_INDICATOR_KEY:
+            defaults = [DEFAULTS_INDICATOR_KEY]+defaults
+        self.defaults = defaults
+
     def load(self):
         self.reload()
         if self.rewrite_on_load:
@@ -490,10 +645,10 @@ class TSVZed(OrderedDict):
         if self.verbose:
             self.__teePrintOrNot(f"Loading {self._fileName}")
         super().clear()
-        readTSV(self._fileName, teeLogger = self.teeLogger, header = self.header, createIfNotExist = self.createIfNotExist, verifyHeader = self.verifyHeader, verbose = self.verbose, taskDic = self,encoding = self.encoding if self.encoding else None)
+        readTabularFile(self._fileName, teeLogger = self.teeLogger, header = self.header, createIfNotExist = self.createIfNotExist, verifyHeader = self.verifyHeader, verbose = self.verbose, taskDic = self,encoding = self.encoding if self.encoding else None, strict = self.strict, delimiter = self.delimiter, defaults=self.defaults)
         if self.verbose:
             self.__teePrintOrNot(f"Loaded {len(self)} records from {self._fileName}")
-        self.correctColumnNum = len(self.header.split('\t')) if (self.header and self.verifyHeader) else (len(self[next(iter(self))]) if self else -1)
+        self.correctColumnNum = len(self.header.split(self.delimiter)) if (self.header and self.verifyHeader) else (len(self[next(iter(self))]) if self else -1)
         if self.verbose:
             self.__teePrintOrNot(f"correctColumnNum: {self.correctColumnNum}")
         #super().update(loadedData)
@@ -505,30 +660,55 @@ class TSVZed(OrderedDict):
         return self
     
     def __setitem__(self,key,value):
-        key = str(key).strip()
+        key = str(key).rstrip()
         if not key:
             self.__teePrintOrNot('Key cannot be empty','error')
             return
         if type(value) == str:
-            value = value.strip().split('\t')
+            value = value.split(self.delimiter)
         # sanitize the value
-        value = [(str(segment).strip() if type(segment) != str else segment.strip()) if segment else '' for segment in value]
-        #value = list(map(lambda segment: str(segment).strip(), value))
+        value = [(str(segment).rstrip() if type(segment) != str else segment.rstrip()) if segment else '' for segment in value]
+        # escape the delimiter and newline characters
+        value = [segment.replace(self.delimiter,'<sep>').replace('\n','\\n') for segment in value]
         # the first field in value should be the key
         # add it if it is not there
         if not value or value[0] != key:
             value = [key]+value
         # verify the value has the correct number of columns
         if self.correctColumnNum != 1 and len(value) == 1:
-            # this means we want to clear / deelte the key
+            # this means we want to clear / delete the key
             self.__delitem__(key)
         elif self.correctColumnNum > 0:
-            assert len(value) == self.correctColumnNum, f"Data format error! Expected {self.correctColumnNum} columns, but got {len(value) } columns"
+            if len(value) != self.correctColumnNum:
+                if self.strict:
+                    self.__teePrintOrNot(f"Value {value} does not have the correct number of columns: {self.correctColumnNum}. Refuse adding key...",'error')
+                    return
+                elif self.verbose:
+                    self.__teePrintOrNot(f"Value {value} does not have the correct number of columns: {self.correctColumnNum}, correcting...",'warning')
+                if len(value) < self.correctColumnNum:
+                    value += ['']*(self.correctColumnNum-len(value))
+                elif len(value) > self.correctColumnNum:
+                    value = value[:self.correctColumnNum]
         else:
             self.correctColumnNum = len(value)
+        if self.defaults and len(self.defaults) > 1:
+            for i in range(1,len(value)):
+                if not value[i] and i < len(self.defaults) and self.defaults[i]:
+                    value[i] = self.defaults[i]
+                    if self.verbose:
+                        self.__teePrintOrNot(f"    Replacing empty value at {i} with default: {self.defaults[i]}")
+        if key == DEFAULTS_INDICATOR_KEY:
+            self.defaults = value
+            if self.verbose:
+                self.__teePrintOrNot(f"Defaults set to {value}")
+            if not self.memoryOnly:
+                self.appendQueue.append(self.delimiter.join(value))
+                self.lastUpdateTime = get_time_ns()
+                if self.verbose:
+                    self.__teePrintOrNot(f"Appending Defaults {key} to the appendQueue")
+            return
         if self.verbose:
             self.__teePrintOrNot(f"Setting {key} to {value}")
-
         if key in self:
             if self[key] == value:
                 if self.verbose:
@@ -537,13 +717,17 @@ class TSVZed(OrderedDict):
             self.dirty = True
         # update the dictionary, 
         super().__setitem__(key,value)
-        if self.verbose:
-            self.__teePrintOrNot(f"Key {key} updated")
         if self.memoryOnly:
+            if self.verbose:
+                self.__teePrintOrNot(f"Key {key} updated in memory only")
+            return
+        elif key.startswith('#'):
+            if self.verbose:
+                self.__teePrintOrNot(f"Key {key} updated in memory only as it starts with #")
             return
         if self.verbose:
             self.__teePrintOrNot(f"Appending {key} to the appendQueue")
-        self.appendQueue.append('\t'.join(value))
+        self.appendQueue.append(self.delimiter.join(value))
         self.lastUpdateTime = get_time_ns()
         # if not self.appendThread.is_alive():
         #     self.commitAppendToFile()
@@ -552,25 +736,38 @@ class TSVZed(OrderedDict):
 
     
     def __delitem__(self,key):
-        key = str(key).strip()
+        key = str(key).rstrip()
+        if key == DEFAULTS_INDICATOR_KEY:
+            self.defaults = []
+            if self.verbose:
+                self.__teePrintOrNot(f"Defaults cleared")
+            if not self.memoryOnly:
+                self.__appendEmptyLine(key)
+                if self.verbose:
+                    self.__teePrintOrNot(f"Appending empty default line {key}")
+            return
         # delete the key from the dictionary and update the file
         if key not in self:
             if self.verbose:
                 self.__teePrintOrNot(f"Key {key} not found")
             return
         super().__delitem__(key)
-        if self.memoryOnly:
+        if self.memoryOnly or key.startswith('#'):
+            if self.verbose:
+                self.__teePrintOrNot(f"Key {key} deleted in memory")
             return
         self.__appendEmptyLine(key)
+        if self.verbose:
+            self.__teePrintOrNot(f"Appending empty line {key}")
         self.lastUpdateTime = get_time_ns()
         
     def __appendEmptyLine(self,key):
         self.dirty = True
         if self.correctColumnNum > 0:
-            emptyLine = key+'\t'*(self.correctColumnNum-1)
+            emptyLine = key+self.delimiter*(self.correctColumnNum-1)
         elif len(self[key]) > 1:
             self.correctColumnNum = len(self[key])
-            emptyLine = key+'\t'*(self.correctColumnNum-1)
+            emptyLine = key+self.delimiter*(self.correctColumnNum-1)
         else:
             emptyLine = key
         if self.verbose:
@@ -745,7 +942,7 @@ memoryOnly:{self.memoryOnly}
             if self.header:
                 file.write(self.header+'\n')
             for key in self:
-                file.write('\t'.join(self[key])+'\n')
+                file.write(self.delimiter.join(self[key])+'\n')
             self.release_file_obj(file)
             if self.verbose:
                 self.__teePrintOrNot(f"{len(self)} records written to {self._fileName}")
@@ -761,27 +958,32 @@ memoryOnly:{self.memoryOnly}
         return self
     
     def mapToFile(self):
+        mec = self.monitor_external_changes
+        self.monitor_external_changes = False
         try:
             if (not self.monitor_external_changes) and self.externalFileUpdateTime < getFileUpdateTimeNs(self._fileName):
                 self.__teePrintOrNot(f"Warning: Overwriting external changes in {self._fileName}",'warning')
-            file = self.get_file_obj('r+')
+            file = self.get_file_obj('r+b')
             overWrite = False
-            line = file.readline()
-            aftPos = file.tell()
-            if self.header and not lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = False):
-                file.seek(0)
-                file.write(self.header+'\n')
-                # if the header is not the same length as the line, we need to overwrite the file
-                if aftPos != file.tell():
-                    overWrite = True
-                if self.verbose:
-                    self.__teePrintOrNot(f"Header {self.header} written to {self._fileName}")
+            if self.header:
+                line = file.readline().decode(self.encoding)
+                aftPos = file.tell()
+                if not _lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = self.strict):
+                    file.seek(0)
+                    file.write(f'{self.header}\n'.encode(encoding=self.encoding))
+                    # if the header is not the same length as the line, we need to overwrite the file
+                    if aftPos != file.tell():
+                        overWrite = True
+                    if self.verbose:
+                        self.__teePrintOrNot(f"Header {self.header} written to {self._fileName}")
             for value in self.values():
-                strToWrite = '\t'.join(value)+'\n'
+                if value[0].startswith('#'):
+                    continue
+                strToWrite = self.delimiter.join(value)
                 if overWrite:
                     if self.verbose:
                         self.__teePrintOrNot(f"Overwriting {value} to {self._fileName}")
-                    file.write(strToWrite)
+                    file.write(strToWrite.encode(encoding=self.encoding)+b'\n')
                     continue
                 pos = file.tell()
                 line = file.readline()
@@ -789,15 +991,17 @@ memoryOnly:{self.memoryOnly}
                 if not line or pos == aftPos:
                     if self.verbose:
                         self.__teePrintOrNot(f"End of file reached. Appending {value} to {self._fileName}")
-                    file.write(strToWrite)
+                    file.write(strToWrite.encode(encoding=self.encoding))
                     overWrite = True
                     continue
+                strToWrite = strToWrite.encode(encoding=self.encoding).ljust(len(line)-1)+b'\n'
                 if line != strToWrite:
                     if self.verbose:
-                        self.__teePrintOrNot(f"Overwriting {value} to {self._fileName}")
+                        self.__teePrintOrNot(f"Modifing {value} to {self._fileName}")
                     file.seek(pos)
                     # fill the string with space to write to the correct length
-                    file.write(strToWrite.rstrip('\n').ljust(len(line)-1)+'\n')
+                    #file.write(strToWrite.rstrip('\n').ljust(len(line)-1)+'\n')
+                    file.write(strToWrite)
                     if aftPos != file.tell():
                         overWrite = True
             file.truncate()
@@ -813,6 +1017,8 @@ memoryOnly:{self.memoryOnly}
             import traceback
             self.__teePrintOrNot(traceback.format_exc(),'error')
             self.deSynced = True
+        self.externalFileUpdateTime = getFileUpdateTimeNs(self._fileName)
+        self.monitor_external_changes = mec
         return self
     
     def checkExternalChanges(self):
@@ -831,9 +1037,10 @@ memoryOnly:{self.memoryOnly}
 
     def _appendWorker(self):
         while not self.shutdownEvent.is_set():
-            self.checkExternalChanges()
-            self.rewrite()
-            self.commitAppendToFile()
+            if not self.memoryOnly:
+                self.checkExternalChanges()
+                self.rewrite()
+                self.commitAppendToFile()
             time.sleep(self.append_check_delay)
             # self.appendEvent.wait()
             # self.appendEvent.clear()
@@ -883,15 +1090,19 @@ memoryOnly:{self.memoryOnly}
     def get_file_obj(self,modes = 'a'):
         self.writeLock.acquire()
         try:
-            if not self.encoding:
-                self.encoding = 'utf8'
-            file = open(self._fileName, mode=modes, encoding=self.encoding)
+            if 'b' not in modes:
+                if not self.encoding:
+                    self.encoding = 'utf8'
+                file = open(self._fileName, mode=modes, encoding=self.encoding)
+            else:
+                file = open(self._fileName, mode=modes)
             # Lock the file after opening
             if os.name == 'posix':
                 fcntl.lockf(file, fcntl.LOCK_EX)
             elif os.name == 'nt':
                 # For Windows, locking the entire file, avoiding locking an empty file
-                lock_length = max(1, os.path.getsize(self._fileName))
+                #lock_length = max(1, os.path.getsize(self._fileName))
+                lock_length = 2147483647
                 msvcrt.locking(file.fileno(), msvcrt.LK_LOCK, lock_length)
             if self.verbose:
                 self.__teePrintOrNot(f"File {self._fileName} locked with mode {modes}")
@@ -910,13 +1121,18 @@ memoryOnly:{self.memoryOnly}
         try:
             file.flush()  # Ensure the file is flushed before unlocking
             os.fsync(file.fileno())  # Ensure the file is synced to disk before unlocking
-            if os.name == 'posix':
-                fcntl.lockf(file, fcntl.LOCK_UN)
-            elif os.name == 'nt':
-                # Unlocking the entire file; for Windows, ensure not unlocking an empty file
-                unlock_length = max(1, os.path.getsize(file.name))
-                msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, unlock_length)
-            file.close()  # Ensure file is closed after unlocking
+            if not file.closed:
+                if os.name == 'posix':
+                    fcntl.lockf(file, fcntl.LOCK_UN)
+                elif os.name == 'nt':
+                    # Unlocking the entire file; for Windows, ensure not unlocking an empty file
+                    #unlock_length = max(1, os.path.getsize(os.path.realpath(file.name)))
+                    unlock_length = 2147483647
+                    try:
+                        msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, unlock_length)
+                    except:
+                        pass
+                file.close()  # Ensure file is closed after unlocking
             if self.verbose:
                 self.__teePrintOrNot(f"File {file.name} unlocked / released")
         except Exception as e:
@@ -925,26 +1141,47 @@ memoryOnly:{self.memoryOnly}
             except Exception as e:
                 self.__teePrintOrNot(f"Failed to release writeLock for {file.name}: {e}",'error')
             self.__teePrintOrNot(f"Failed to release file {file.name}: {e}",'error')
-        try:
-            self.writeLock.release()  # Ensure the thread lock is always released
-        except Exception as e:
-            self.__teePrintOrNot(f"Failed to release writeLock for {file.name}: {e}",'error')
-        self.externalFileUpdateTime = getFileUpdateTimeNs(self._fileName)
+            import traceback
+            self.__teePrintOrNot(traceback.format_exc(),'error')
+        # release the write lock if not already released
+        if self.writeLock.locked():
+            try:
+                self.writeLock.release()  # Ensure the thread lock is always released
+            except Exception as e:
+                self.__teePrintOrNot(f"Failed to release writeLock for {file.name}: {e}",'error')
+            self.externalFileUpdateTime = getFileUpdateTimeNs(self._fileName)
 
 
 def __main__():
     import argparse
-    parser = argparse.ArgumentParser(description='TSVZed: A TSV file manager')
-    parser.add_argument('filename', type=str, help='The TSV file to read')
+    parser = argparse.ArgumentParser(description='TSVZed: A TSV / CSV / NSV file manager')
+    parser.add_argument('filename', type=str, help='The file to read')
     parser.add_argument('operation', type=str,nargs='?', choices=['read','append','delete','clear'], help='The operation to perform. Default: read', default='read')
-    parser.add_argument('line', type=str, nargs='*', help='The line to append to the TSV file. it follows as : {key} {value1} {value2} ... if a key without value be inserted, the value will get deleted.')
-    parser.add_argument('-c', '--header', type=str, help='Perform checks with this header of the TSV file. seperate using \\t')
-    parser.add_argument('-f', '--force', action='store_true', help='Force the operation. Ignore checks for column numbers / headers')
+    parser.add_argument('line', type=str, nargs='*', help='The line to append to the Tabular file. it follows as : {key} {value1} {value2} ... if a key without value be inserted, the value will get deleted.')
+    parser.add_argument('-d', '--delimiter', type=str, help='The delimiter of the Tabular file. Default: Infer from last part of filename, or tab if cannot determine. Note: accept unicode escaped char, raw char, or string "comma,tab,null" will refer to their characters. ', default=...)
+    parser.add_argument('-c', '--header', type=str, help='Perform checks with this header of the Tabular file. seperate using --delimiter.')
+    parser.add_argument('--defaults', type=str, help='Default values to fill in the missing columns. seperate using --delimiter. Ex. if -d = comma, --defaults="key,value1,value2..." Note: Please specify the key. But it will not be used as a key need to be unique in data.')
+    strictMode = parser.add_mutually_exclusive_group()
+    strictMode.add_argument('-s', '--strict', dest = 'strict',action='store_true', help='Strict mode. Do not parse values that seems malformed, check for column numbers / headers')
+    strictMode.add_argument('-f', '--force', dest = 'strict',action='store_false', help='Force the operation. Ignore checks for column numbers / headers')
     parser.add_argument('-v', '--verbose', action='store_true', help='Print verbose output')
     parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {version} by {author}')
     args = parser.parse_args()
-
-    header = args.header.replace('\\t','\t') if args.header else ''
+    args.delimiter = get_delimiter(delimiter=args.delimiter,file_name=args.filename)
+    if args.header and args.header.endswith('\\'):
+        args.header += '\\'
+    try:
+        header = args.header.encode().decode('unicode_escape') if args.header else ''
+    except Exception as e:
+        print(f"Failed to decode header: {args.header}")
+        header = ''
+    defaults = []
+    if args.defaults:
+        try:
+            defaults = args.defaults.encode().decode('unicode_escape').split(args.delimiter)
+        except Exception as e:
+            print(f"Failed to decode defaults: {args.defaults}")
+            defaults = []
 
     if args.operation == 'read':
         # check if the file exist
@@ -952,14 +1189,14 @@ def __main__():
             print(f"File not found: {args.filename}")
             return
         # read the file
-        data = readTSV(args.filename, verifyHeader = False, verbose=args.verbose,strict= not args.force)
-        print(pretty_format_table(data.values()))
+        data = readTabularFile(args.filename, verifyHeader = False, verbose=args.verbose,strict= args.strict, delimiter=args.delimiter, defaults=defaults)
+        print(pretty_format_table(data.values(),delimiter=args.delimiter))
     elif args.operation == 'append':
-        appendTSV(args.filename, args.line,createIfNotExist = True, header=header, verbose=args.verbose, strict= not args.force)
+        appendTabularFile(args.filename, args.line,createIfNotExist = True, header=header, verbose=args.verbose, strict= args.strict, delimiter=args.delimiter)
     elif args.operation == 'delete':
-        appendTSV(args.filename, args.line[:1],createIfNotExist = True, header=header, verbose=args.verbose, strict= not args.force)
+        appendTabularFile(args.filename, args.line[:1],createIfNotExist = True, header=header, verbose=args.verbose, strict= args.strict, delimiter=args.delimiter)
     elif args.operation == 'clear':
-        clearTSV(args.filename, header=header, verbose=args.verbose, verifyHeader=not args.force)
+        clearTabularFile(args.filename, header=header, verbose=args.verbose, verifyHeader=args.strict, delimiter=args.delimiter)
     else:
         print("Invalid operation")
         return