TSVZ 3.36__tar.gz → 3.38__tar.gz

{tsvz-3.36 → tsvz-3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 3.36
+Version: 3.38
 Summary: An simple in memory wrapper around a TSV file to function as a database
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan
@@ -11,6 +11,9 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (G
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
+Requires-Dist: argparse
+Provides-Extra: completion
+Requires-Dist: argcomplete; extra == "completion"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -18,6 +21,8 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 

{tsvz-3.36 → tsvz-3.38}/TSVZ.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 3.36
+Version: 3.38
 Summary: An simple in memory wrapper around a TSV file to function as a database
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan
@@ -11,6 +11,9 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (G
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
+Requires-Dist: argparse
+Provides-Extra: completion
+Requires-Dist: argcomplete; extra == "completion"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -18,6 +21,8 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 

{tsvz-3.36 → tsvz-3.38}/TSVZ.egg-info/SOURCES.txt

@@ -5,4 +5,6 @@ TSVZ.egg-info/PKG-INFO
 TSVZ.egg-info/SOURCES.txt
 TSVZ.egg-info/dependency_links.txt
 TSVZ.egg-info/entry_points.txt
-TSVZ.egg-info/top_level.txt
+TSVZ.egg-info/requires.txt
+TSVZ.egg-info/top_level.txt
+tests/test_regression.py

tsvz-3.38/TSVZ.egg-info/requires.txt

@@ -0,0 +1,4 @@
+argparse
+
+[completion]
+argcomplete

{tsvz-3.36 → tsvz-3.38}/TSVZ.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python3
+#! /usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
 # /// script
 # requires-python = ">=3.6"
 # dependencies = [
@@ -9,7 +10,6 @@ import functools
 import io
 import os
 import re
-from tabnanny import verbose
 import threading
 import time
 import sys
@@ -26,23 +26,42 @@ if os.name == 'nt':
 elif os.name == 'posix':
 	import fcntl
 
-version = '3.36'
+version = '3.38'
 __version__ = version
 author = 'pan@zopyr.us'
-COMMIT_DATE = '2026-02-02'
+COMMIT_DATE = '2026-06-04'
 
 DEFAULT_DELIMITER = '\t'
 DEFAULTS_INDICATOR_KEY = '#_defaults_#'
 
-COMPRESSED_FILE_EXTENSIONS = ['gz','gzip','bz2','bzip2','xz','lzma']
+COMPRESSED_FILE_EXTENSIONS = ['gz','gzip','bz2','bzip2','xz','lzma','zst','zstd']
 
-def get_delimiter(delimiter,file_name = ''):
+def _isCompressedFile(fileName):
+	return fileName.rpartition('.')[2].lower() in COMPRESSED_FILE_EXTENSIONS
+
+def get_delimiter(delimiter = ...,file_name = ''):
+	"""
+	Resolve the delimiter to use for an operation.
+
+	When the delimiter is not provided (the ``...`` sentinel or a falsy value),
+	it is determined automatically: inferred from the file name when one is
+	available, otherwise falling back to the last-determined default. Explicit
+	delimiters (including the aliases 'comma', 'tab', 'pipe', 'null' and escape
+	sequences) are normalized. The resolved value is remembered as the new
+	module-level default.
+
+	Parameters:
+	- delimiter: The delimiter or alias, or ``...``/``None`` to auto-determine.
+	- file_name (str, optional): The file name to infer the delimiter from.
+
+	Returns:
+	str: The resolved delimiter.
+	"""
 	global DEFAULT_DELIMITER
-	if not delimiter:
-		return DEFAULT_DELIMITER
-	elif delimiter == ...:
+	if delimiter is ...:
 		if not file_name:
-			rtn =  '\t'
+			# Nothing to infer from; reuse the last-determined default.
+			return DEFAULT_DELIMITER
 		elif file_name.endswith('.csv'):
 			rtn =  ','
 		elif file_name.endswith('.nsv'):
@@ -51,6 +70,8 @@ def get_delimiter(delimiter,file_name = ''):
 			rtn =  '|'
 		else:
 			rtn =  '\t'
+	elif not delimiter:
+		return DEFAULT_DELIMITER
 	elif delimiter == 'comma':
 		rtn =  ','
 	elif delimiter == 'tab':
@@ -75,34 +96,43 @@ def eprint(*args, **kwargs):
 		print(*args, **kwargs)
 
 def openFileAsCompressed(fileName,mode = 'rb',encoding = 'utf8',teeLogger = None,compressLevel = 1):
+	lowerFileName = fileName.lower()
 	if 'b' not in mode:
 		mode += 't'
 	kwargs = {}
 	if 'r' not in mode:
-		if fileName.endswith('.xz'):
+		if lowerFileName.endswith('.xz'):
 			kwargs['preset'] = compressLevel
+		elif lowerFileName.endswith('.zst') or lowerFileName.endswith('.zstd'):
+			kwargs['level'] = compressLevel
 		else:
 			kwargs['compresslevel'] = compressLevel
 	if 'b' not in mode:
 		kwargs['encoding'] = encoding
-	if fileName.endswith('.xz') or fileName.endswith('.lzma'):
+	if lowerFileName.endswith('.xz') or lowerFileName.endswith('.lzma'):
 		try:
 			import lzma
 			return lzma.open(fileName, mode, **kwargs)
 		except Exception:
 			__teePrintOrNot(f"Failed to open {fileName} with lzma, trying bin",teeLogger=teeLogger)
-	elif fileName.endswith('.gz') or fileName.endswith('.gzip'):
+	elif lowerFileName.endswith('.gz') or lowerFileName.endswith('.gzip'):
 		try:
 			import gzip
 			return gzip.open(fileName, mode, **kwargs)
 		except Exception:
 			__teePrintOrNot(f"Failed to open {fileName} with gzip, trying bin",teeLogger=teeLogger)
-	elif fileName.endswith('.bz2') or fileName.endswith('.bzip2'):
+	elif lowerFileName.endswith('.bz2') or lowerFileName.endswith('.bzip2'):
 		try:
 			import bz2
 			return bz2.open(fileName, mode, **kwargs)
 		except Exception:
 			__teePrintOrNot(f"Failed to open {fileName} with bz2, trying bin",teeLogger=teeLogger)
+	elif lowerFileName.endswith('.zst') or lowerFileName.endswith('.zstd'):
+		try:
+			from compression import zstd
+			return zstd.open(fileName, mode, **kwargs)
+		except Exception:
+			__teePrintOrNot(f"Failed to open {fileName} with zstd, trying bin",teeLogger=teeLogger)
 	if 't' in mode:
 		mode = mode.replace('t','')
 		return open(fileName, mode, encoding=encoding)
@@ -328,7 +358,7 @@ def format_bytes(size, use_1024_bytes=None, to_int=False, to_str=False,str_forma
 			power = 2**10
 			n = 0
 			power_labels = {0 : '', 1: 'Ki', 2: 'Mi', 3: 'Gi', 4: 'Ti', 5: 'Pi'}
-			while size > power:
+			while size >= power:
 				size /= power
 				n += 1
 			return f"{size:{str_format}}{' '}{power_labels[n]}"
@@ -336,7 +366,7 @@ def format_bytes(size, use_1024_bytes=None, to_int=False, to_str=False,str_forma
 			power = 10**3
 			n = 0
 			power_labels = {0 : '', 1: 'K', 2: 'M', 3: 'G', 4: 'T', 5: 'P'}
-			while size > power:
+			while size >= power:
 				size /= power
 				n += 1
 			return f"{size:{str_format}}{' '}{power_labels[n]}"
@@ -403,7 +433,7 @@ def __teePrintOrNot(message,level = 'info',teeLogger = None):
 	except Exception:
 		print(message,flush=True)
 
-def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT_DELIMITER,defaults = ...,
+def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = ...,defaults = ...,
 				 storeOffset = False, offset = -1):
 	"""
 	Process a line of text and update the task dictionary.
@@ -423,6 +453,7 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 	"""
 	if defaults is ...:
 		defaults = []
+	delimiter = get_delimiter(delimiter)
 	line = line.strip('\x00').rstrip('\r\n')
 	if not line or (line.startswith('#') and not line.startswith(DEFAULTS_INDICATOR_KEY)):
 		# if verbose:
@@ -446,7 +477,7 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 			# if verbose:
 			# 	__teePrintOrNot(f"Empty defaults line found: {line}",teeLogger=teeLogger)
 			defaults.clear()
-			defaults[0] = DEFAULTS_INDICATOR_KEY
+			defaults[:] = [DEFAULTS_INDICATOR_KEY]
 		else:
 			# if verbose:
 			# 	__teePrintOrNot(f"Key {lineCache[0]} found with empty value, deleting such key's representaion",teeLogger=teeLogger)
@@ -481,6 +512,44 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 	# 	__teePrintOrNot(f"Key {lineCache[0]} added",teeLogger=teeLogger)
 	return correctColumnNum, lineCache
 
+def _read_last_valid_line_forward(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None,
+								  strict=False, encoding='utf8', delimiter=...,
+								  defaults=None, storeOffset=False):
+	"""
+	Forward, single-pass variant of read_last_valid_line for compressed files.
+
+	Backward chunked seeking forces a full re-decompress on every seek (and is not
+	reliably supported for some codecs e.g. zstd), so for compressed files we decompress
+	once, streaming forward, and keep only the last valid line / offset. A scratch dict is
+	used so memory stays flat instead of accumulating the whole file into taskDic.
+	"""
+	if defaults is None:
+		defaults = []
+	delimiter = get_delimiter(delimiter, file_name=fileName)
+	last_valid = -1 if storeOffset else []
+	scratch = {}
+	with openFileAsCompressed(fileName, 'rb', encoding=encoding, teeLogger=teeLogger) as file:
+		while True:
+			offset = file.tell()
+			line = file.readline()
+			if not line:
+				break
+			if not line.strip():
+				continue
+			correctColumnNum, lineCache = _processLine(
+				line=line.decode(encoding=encoding, errors='replace'),
+				taskDic=scratch,
+				correctColumnNum=correctColumnNum,
+				strict=strict,
+				delimiter=delimiter,
+				defaults=defaults,
+				storeOffset=storeOffset,
+				offset=offset,
+			)
+			if lineCache:
+				last_valid = offset if storeOffset else lineCache
+	return last_valid
+
 def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None, strict=False,
 						 encoding = 'utf8',delimiter = ...,defaults = ...,storeOffset = False	):
 	"""
@@ -508,6 +577,12 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
 	delimiter = get_delimiter(delimiter,file_name=fileName)
 	if verbose:
 		__teePrintOrNot(f"Reading last line only from {fileName}",teeLogger=teeLogger)
+	if _isCompressedFile(fileName):
+		# Backward seeking through a codec is slow/unsupported; decompress once, forward.
+		return _read_last_valid_line_forward(
+			fileName, taskDic, correctColumnNum, verbose=verbose, teeLogger=teeLogger,
+			strict=strict, encoding=encoding, delimiter=delimiter, defaults=defaults,
+			storeOffset=storeOffset)
 	with openFileAsCompressed(fileName, 'rb',encoding=encoding, teeLogger=teeLogger) as file:
 		file.seek(0, os.SEEK_END)
 		file_size = file.tell()
@@ -558,7 +633,8 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
 	return last_valid_line
 
 @functools.lru_cache(maxsize=None)
-def _get_sanitization_re(delimiter = DEFAULT_DELIMITER):
+def _get_sanitization_re(delimiter = ...):
+	delimiter = get_delimiter(delimiter)
 	return re.compile(r"(</sep/>|</LF/>|<sep>|<LF>|\n|" + re.escape(delimiter) + r")")
 
 _sanitize_replacements = {
@@ -568,9 +644,10 @@ _sanitize_replacements = {
 }
 _inverse_sanitize_replacements = {v: k for k, v in _sanitize_replacements.items()}
 
-def _sanitize(data,delimiter = DEFAULT_DELIMITER):
+def _sanitize(data,delimiter = ...):
 	if not data:
 		return data
+	delimiter = get_delimiter(delimiter)
 	def repl(m):
 		tok = m.group(0)
 		if tok == delimiter:
@@ -584,9 +661,10 @@ def _sanitize(data,delimiter = DEFAULT_DELIMITER):
 	else:
 		return [pattern.sub(repl,str(segment)) if segment else '' for segment in data]
 
-def _unsanitize(data,delimiter = DEFAULT_DELIMITER):
+def _unsanitize(data,delimiter = ...):
 	if not data:
 		return data
+	delimiter = get_delimiter(delimiter)
 	def repl(m):
 		tok = m.group(0)
 		if tok == "<sep>":
@@ -598,7 +676,7 @@ def _unsanitize(data,delimiter = DEFAULT_DELIMITER):
 	else:
 		return [pattern.sub(repl,str(segment).rstrip()) if segment else '' for segment in data]
 
-def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DELIMITER):
+def _formatHeader(header,verbose = False,teeLogger = None,delimiter = ...):
 	"""
 	Format the header string.
 
@@ -611,7 +689,7 @@ def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DE
 		list: The formatted header list of string.
 	"""
 	if isinstance(header,str):
-		header = header.split(delimiter)
+		header = header.split(get_delimiter(delimiter))
 	else:
 		try:
 			header = [str(s) for s in header]
@@ -621,7 +699,7 @@ def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DE
 			header = []
 	return [s.rstrip() for s in header]
 
-def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False,delimiter = DEFAULT_DELIMITER):
+def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False,delimiter = ...):
 	"""
 	Verify if a line contains the header.
 
@@ -635,6 +713,7 @@ def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = Fal
 	Returns:
 	bool: True if the header matches the line, False otherwise.
 	"""
+	delimiter = get_delimiter(delimiter)
 	line = _formatHeader(line,verbose=verbose,teeLogger=teeLogger,delimiter=delimiter)
 	if verbose:
 		__teePrintOrNot(f"Header: \n{header}",teeLogger=teeLogger)
@@ -646,7 +725,7 @@ def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = Fal
 		return False
 	return True
 
-def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,header = [],encoding = 'utf8',strict = True,delimiter = DEFAULT_DELIMITER):
+def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,header = [],encoding = 'utf8',strict = True,delimiter = ...):
 	"""
 	Verify the existence of the tabular file.
 
@@ -661,6 +740,7 @@ def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,heade
 	Returns:
 	bool: True if the file exists, False otherwise.
 	"""
+	delimiter = get_delimiter(delimiter, file_name=fileName)
 	remainingFileName, _ ,extenstionName = fileName.rpartition('.')
 	if extenstionName in COMPRESSED_FILE_EXTENSIONS:
 		remainingFileName, _ ,extenstionName = remainingFileName.rpartition('.')
@@ -677,7 +757,6 @@ def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,heade
 			try:
 				with openFileAsCompressed(fileName, mode ='wb',encoding=encoding,teeLogger=teeLogger)as file:
 					header = delimiter.join(_sanitize(_formatHeader(header,
-													 verbose=verbose,
 													 teeLogger=teeLogger,
 													 delimiter=delimiter,
 													 ),delimiter=delimiter))
@@ -770,7 +849,7 @@ def readTabularFile(fileName,teeLogger = None,header = '',createIfNotExist = Fal
 	with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 		if any(header) and verifyHeader:
 				line = file.readline().decode(encoding=encoding,errors='replace')
-				if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict) and correctColumnNum == -1:
+				if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter) and correctColumnNum == -1:
 					correctColumnNum = len(header)
 					if verbose:
 						__teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
@@ -872,7 +951,7 @@ def appendLinesTabularFile(fileName,linesToAppend,teeLogger = None,header = '',c
 	if any(header) and verifyHeader:
 		with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 			line = file.readline().decode(encoding=encoding,errors='replace')
-			if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+			if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter):
 				correctColumnNum = len(header)
 				if verbose:
 					__teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
@@ -926,7 +1005,7 @@ def clearTabularFile(fileName,teeLogger = None,header = '',verifyHeader = False,
 		with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 			if any(header) and verifyHeader:
 				line = file.readline().decode(encoding=encoding,errors='replace')
-				if not _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+				if not _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter):
 					__teePrintOrNot(f'Warning: Header mismatch in {fileName}. Keeping original header in file...','warning',teeLogger)
 				header = _formatHeader(line,verbose = verbose,teeLogger = teeLogger,delimiter=delimiter)
 		with openFileAsCompressed(fileName, mode ='wb',encoding=encoding,teeLogger=teeLogger)as file:
@@ -1019,10 +1098,10 @@ def scrubTabularFile(fileName,teeLogger = None,header = '',createIfNotExist = Fa
 		appendLinesTabularFile(fileName,file,teeLogger = teeLogger,header = header,createIfNotExist = createIfNotExist,verifyHeader = verifyHeader,verbose = verbose,encoding = encoding,strict = strict,delimiter = delimiter)
 	return file
 
-def getListView(tsvzDic,header = [],delimiter = DEFAULT_DELIMITER):
+def getListView(tsvzDic,header = [],delimiter = ...):
 	if header:
 		if isinstance(header,str):
-			header = header.split(delimiter)
+			header = header.split(get_delimiter(delimiter))
 		elif not isinstance(header,list):
 			try:
 				header = list(header)
@@ -1066,7 +1145,7 @@ class TSVZed(OrderedDict):
 	rewrite_on_exit : bool, default=False
 		If True, rewrites the entire file when closing/exiting.
 	rewrite_interval : float, default=0
-		Minimum time interval (in seconds) between full file rewrites. 0 means no limit.
+		Minimum time interval (in seconds) between full file rewrites. 0 means do not rewrite.
 	append_check_delay : float, default=0.01
 		Time delay (in seconds) between checks of the append queue by the worker thread.
 	monitor_external_changes : bool, default=True
@@ -1595,7 +1674,7 @@ memoryOnly:{self.memoryOnly}
 			if self.header:
 				line = file.readline().decode(self.encoding,errors='replace')
 				aftPos = file.tell()
-				if not _lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = self.strict):
+				if not _lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = self.strict,delimiter = self.delimiter):
 					header = self.delimiter.join(_sanitize(self.header,delimiter=self.delimiter))
 					file.seek(0)
 					file.write(f'{header}\n'.encode(encoding=self.encoding,errors='replace'))
@@ -1726,6 +1805,7 @@ memoryOnly:{self.memoryOnly}
 	
 	def get_file_obj(self,modes = 'ab'):
 		self.writeLock.acquire()
+		file = None
 		try:
 			if not self.encoding:
 				self.encoding = 'utf8'
@@ -1752,6 +1832,8 @@ memoryOnly:{self.memoryOnly}
 		# if write lock is already released, return
 		if not self.writeLock.locked():
 			return
+		if file is None:
+			return
 		try:
 			file.flush()  # Ensure the file is flushed before unlocking
 			os.fsync(file.fileno())  # Ensure the file is synced to disk before unlocking
@@ -2024,13 +2106,18 @@ class TSVZedLite(MutableMapping):
 					)
 		if self.verbose:
 			eprint(f"Read at position {pos}: {segments}")
-		if key is not ... and segments[0] != key:
+		if key is not ...:
+			if not segments:
+				eprint(f"Error: No segments found at position {pos}")
+			elif segments[0] != key:
 				eprint(f"Warning: Key mismatch at position {pos}: expected {key}, got {segments[0]}")
-				if self.strict:
-					eprint("Error: Key mismatch and strict mode enabled. Raising KeyError.")
-					raise KeyError(key)
-				else :
-					eprint("Continuing despite key mismatch due to non-strict mode. Expect errors!")
+			else:
+				return segments
+			if self.strict:
+				eprint("Error: Key mismatch and strict mode enabled. Raising KeyError.")
+				raise KeyError(key)
+			else :
+				eprint("Continuing despite key mismatch due to non-strict mode. Expect errors!")
 		return segments
 
 	# Implement basic __getitem__, __setitem__, __delitem__, __iter__, and __len__. needed for MutableMapping
@@ -2083,6 +2170,7 @@ class TSVZedLite(MutableMapping):
 			self.defaults = value
 			if self.verbose:
 				eprint(f"Defaults set to {value}")
+			return
 		elif key.startswith('#'):
 			if self.verbose:
 				eprint(f"Key {key} updated in memory (data in index) as it starts with #")
@@ -2261,6 +2349,11 @@ def __main__():
 	strictMode.add_argument('-f', '--force', dest = 'strict',action='store_false', help='Force the operation. Ignore checks for column numbers / headers')
 	parser.add_argument('-v', '--verbose', action='store_true', help='Print verbose output')
 	parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {version} @ {COMMIT_DATE} by {author}')
+	try:
+		import argcomplete
+		argcomplete.autocomplete(parser,always_complete_options='long')
+	except ImportError:
+		pass
 	args = parser.parse_args()
 	args.delimiter = get_delimiter(delimiter=args.delimiter,file_name=args.filename)
 	if args.header and args.header.endswith('\\'):

{tsvz-3.36 → tsvz-3.38}/setup.py

@@ -15,6 +15,14 @@ setup(
 			'tsvz = TSVZ:__main__',
         ],
     },
+    install_requires=[
+        'argparse',
+    ],
+    extras_require={
+        'completion': [
+            'argcomplete',
+        ],
+    },
     classifiers=[
         'Programming Language :: Python :: 3',
         'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',

tsvz-3.38/tests/test_regression.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+import TSVZ
+
+
+class TestTSVZRegressions(unittest.TestCase):
+    def test_append_and_read_preserves_embedded_tab_and_newline(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            path = os.path.join(temp_dir, "embedded.tsv")
+            header = ["key", "payload"]
+            row = ["row1", "hello\tworld\nline2"]
+
+            TSVZ.appendTabularFile(
+                path,
+                row,
+                header=header,
+                createIfNotExist=True,
+                verifyHeader=True,
+                strict=True,
+                delimiter="\t",
+            )
+
+            data = TSVZ.readTabularFile(
+                path,
+                header=header,
+                verifyHeader=True,
+                strict=True,
+                delimiter="\t",
+            )
+
+            self.assertIn("row1", data)
+            self.assertEqual(data["row1"], row)
+
+    def test_read_uses_defaults_row_for_missing_values(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            path = os.path.join(temp_dir, "defaults.tsv")
+            with open(path, "w", encoding="utf8") as handle:
+                handle.write("key\tcpu\tmem\n")
+                handle.write(f"{TSVZ.DEFAULTS_INDICATOR_KEY}\t4\t8G\n")
+                handle.write("job1\t2\t16G\n")
+                handle.write("job2\t\t32G\n")
+
+            data = TSVZ.readTabularFile(
+                path,
+                header=["key", "cpu", "mem"],
+                verifyHeader=True,
+                strict=True,
+                delimiter="\t",
+            )
+
+            self.assertEqual(data["job1"], ["job1", "2", "16G"])
+            self.assertEqual(data["job2"], ["job2", "4", "32G"])
+
+    def test_last_line_only_ignores_delete_marker_line(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            path = os.path.join(temp_dir, "lastline.tsv")
+            with open(path, "w", encoding="utf8") as handle:
+                handle.write("key\tvalue\n")
+                handle.write("alive\t42\n")
+                handle.write("alive\n")
+
+            last_valid = TSVZ.readTabularFile(
+                path,
+                header=["key", "value"],
+                verifyHeader=True,
+                lastLineOnly=True,
+                strict=False,
+                delimiter="\t",
+            )
+
+            self.assertEqual(last_valid, ["alive", "42"])
+
+    def test_append_and_read_gzip_file(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            path = os.path.join(temp_dir, "compressed.tsv.gz")
+            header = ["key", "value"]
+
+            TSVZ.appendTabularFile(
+                path,
+                ["gk", "gv"],
+                header=header,
+                createIfNotExist=True,
+                verifyHeader=True,
+                strict=True,
+                delimiter="\t",
+            )
+
+            data = TSVZ.readTabularFile(
+                path,
+                header=header,
+                verifyHeader=True,
+                strict=True,
+                delimiter="\t",
+            )
+
+            self.assertEqual(data["gk"], ["gk", "gv"])
+
+
+if __name__ == "__main__":
+    unittest.main()