TSVZ 3.37__tar.gz → 3.38__tar.gz

{tsvz-3.37 → tsvz-3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 3.37
+Version: 3.38
 Summary: An simple in memory wrapper around a TSV file to function as a database
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan

{tsvz-3.37 → tsvz-3.38}/TSVZ.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 3.37
+Version: 3.38
 Summary: An simple in memory wrapper around a TSV file to function as a database
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan

{tsvz-3.37 → tsvz-3.38}/TSVZ.py

@@ -10,7 +10,6 @@ import functools
 import io
 import os
 import re
-from tabnanny import verbose
 import threading
 import time
 import sys
@@ -27,23 +26,42 @@ if os.name == 'nt':
 elif os.name == 'posix':
 	import fcntl
 
-version = '3.37'
+version = '3.38'
 __version__ = version
 author = 'pan@zopyr.us'
-COMMIT_DATE = '2026-04-21'
+COMMIT_DATE = '2026-06-04'
 
 DEFAULT_DELIMITER = '\t'
 DEFAULTS_INDICATOR_KEY = '#_defaults_#'
 
 COMPRESSED_FILE_EXTENSIONS = ['gz','gzip','bz2','bzip2','xz','lzma','zst','zstd']
 
-def get_delimiter(delimiter,file_name = ''):
+def _isCompressedFile(fileName):
+	return fileName.rpartition('.')[2].lower() in COMPRESSED_FILE_EXTENSIONS
+
+def get_delimiter(delimiter = ...,file_name = ''):
+	"""
+	Resolve the delimiter to use for an operation.
+
+	When the delimiter is not provided (the ``...`` sentinel or a falsy value),
+	it is determined automatically: inferred from the file name when one is
+	available, otherwise falling back to the last-determined default. Explicit
+	delimiters (including the aliases 'comma', 'tab', 'pipe', 'null' and escape
+	sequences) are normalized. The resolved value is remembered as the new
+	module-level default.
+
+	Parameters:
+	- delimiter: The delimiter or alias, or ``...``/``None`` to auto-determine.
+	- file_name (str, optional): The file name to infer the delimiter from.
+
+	Returns:
+	str: The resolved delimiter.
+	"""
 	global DEFAULT_DELIMITER
-	if not delimiter:
-		return DEFAULT_DELIMITER
-	elif delimiter == ...:
+	if delimiter is ...:
 		if not file_name:
-			rtn =  '\t'
+			# Nothing to infer from; reuse the last-determined default.
+			return DEFAULT_DELIMITER
 		elif file_name.endswith('.csv'):
 			rtn =  ','
 		elif file_name.endswith('.nsv'):
@@ -52,6 +70,8 @@ def get_delimiter(delimiter,file_name = ''):
 			rtn =  '|'
 		else:
 			rtn =  '\t'
+	elif not delimiter:
+		return DEFAULT_DELIMITER
 	elif delimiter == 'comma':
 		rtn =  ','
 	elif delimiter == 'tab':
@@ -338,7 +358,7 @@ def format_bytes(size, use_1024_bytes=None, to_int=False, to_str=False,str_forma
 			power = 2**10
 			n = 0
 			power_labels = {0 : '', 1: 'Ki', 2: 'Mi', 3: 'Gi', 4: 'Ti', 5: 'Pi'}
-			while size > power:
+			while size >= power:
 				size /= power
 				n += 1
 			return f"{size:{str_format}}{' '}{power_labels[n]}"
@@ -346,7 +366,7 @@ def format_bytes(size, use_1024_bytes=None, to_int=False, to_str=False,str_forma
 			power = 10**3
 			n = 0
 			power_labels = {0 : '', 1: 'K', 2: 'M', 3: 'G', 4: 'T', 5: 'P'}
-			while size > power:
+			while size >= power:
 				size /= power
 				n += 1
 			return f"{size:{str_format}}{' '}{power_labels[n]}"
@@ -413,7 +433,7 @@ def __teePrintOrNot(message,level = 'info',teeLogger = None):
 	except Exception:
 		print(message,flush=True)
 
-def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT_DELIMITER,defaults = ...,
+def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = ...,defaults = ...,
 				 storeOffset = False, offset = -1):
 	"""
 	Process a line of text and update the task dictionary.
@@ -433,6 +453,7 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 	"""
 	if defaults is ...:
 		defaults = []
+	delimiter = get_delimiter(delimiter)
 	line = line.strip('\x00').rstrip('\r\n')
 	if not line or (line.startswith('#') and not line.startswith(DEFAULTS_INDICATOR_KEY)):
 		# if verbose:
@@ -456,7 +477,7 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 			# if verbose:
 			# 	__teePrintOrNot(f"Empty defaults line found: {line}",teeLogger=teeLogger)
 			defaults.clear()
-			defaults[0] = DEFAULTS_INDICATOR_KEY
+			defaults[:] = [DEFAULTS_INDICATOR_KEY]
 		else:
 			# if verbose:
 			# 	__teePrintOrNot(f"Key {lineCache[0]} found with empty value, deleting such key's representaion",teeLogger=teeLogger)
@@ -491,6 +512,44 @@ def _processLine(line,taskDic,correctColumnNum,strict = True,delimiter = DEFAULT
 	# 	__teePrintOrNot(f"Key {lineCache[0]} added",teeLogger=teeLogger)
 	return correctColumnNum, lineCache
 
+def _read_last_valid_line_forward(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None,
+								  strict=False, encoding='utf8', delimiter=...,
+								  defaults=None, storeOffset=False):
+	"""
+	Forward, single-pass variant of read_last_valid_line for compressed files.
+
+	Backward chunked seeking forces a full re-decompress on every seek (and is not
+	reliably supported for some codecs e.g. zstd), so for compressed files we decompress
+	once, streaming forward, and keep only the last valid line / offset. A scratch dict is
+	used so memory stays flat instead of accumulating the whole file into taskDic.
+	"""
+	if defaults is None:
+		defaults = []
+	delimiter = get_delimiter(delimiter, file_name=fileName)
+	last_valid = -1 if storeOffset else []
+	scratch = {}
+	with openFileAsCompressed(fileName, 'rb', encoding=encoding, teeLogger=teeLogger) as file:
+		while True:
+			offset = file.tell()
+			line = file.readline()
+			if not line:
+				break
+			if not line.strip():
+				continue
+			correctColumnNum, lineCache = _processLine(
+				line=line.decode(encoding=encoding, errors='replace'),
+				taskDic=scratch,
+				correctColumnNum=correctColumnNum,
+				strict=strict,
+				delimiter=delimiter,
+				defaults=defaults,
+				storeOffset=storeOffset,
+				offset=offset,
+			)
+			if lineCache:
+				last_valid = offset if storeOffset else lineCache
+	return last_valid
+
 def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, teeLogger=None, strict=False,
 						 encoding = 'utf8',delimiter = ...,defaults = ...,storeOffset = False	):
 	"""
@@ -518,6 +577,12 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
 	delimiter = get_delimiter(delimiter,file_name=fileName)
 	if verbose:
 		__teePrintOrNot(f"Reading last line only from {fileName}",teeLogger=teeLogger)
+	if _isCompressedFile(fileName):
+		# Backward seeking through a codec is slow/unsupported; decompress once, forward.
+		return _read_last_valid_line_forward(
+			fileName, taskDic, correctColumnNum, verbose=verbose, teeLogger=teeLogger,
+			strict=strict, encoding=encoding, delimiter=delimiter, defaults=defaults,
+			storeOffset=storeOffset)
 	with openFileAsCompressed(fileName, 'rb',encoding=encoding, teeLogger=teeLogger) as file:
 		file.seek(0, os.SEEK_END)
 		file_size = file.tell()
@@ -568,7 +633,8 @@ def read_last_valid_line(fileName, taskDic, correctColumnNum, verbose=False, tee
 	return last_valid_line
 
 @functools.lru_cache(maxsize=None)
-def _get_sanitization_re(delimiter = DEFAULT_DELIMITER):
+def _get_sanitization_re(delimiter = ...):
+	delimiter = get_delimiter(delimiter)
 	return re.compile(r"(</sep/>|</LF/>|<sep>|<LF>|\n|" + re.escape(delimiter) + r")")
 
 _sanitize_replacements = {
@@ -578,9 +644,10 @@ _sanitize_replacements = {
 }
 _inverse_sanitize_replacements = {v: k for k, v in _sanitize_replacements.items()}
 
-def _sanitize(data,delimiter = DEFAULT_DELIMITER):
+def _sanitize(data,delimiter = ...):
 	if not data:
 		return data
+	delimiter = get_delimiter(delimiter)
 	def repl(m):
 		tok = m.group(0)
 		if tok == delimiter:
@@ -594,9 +661,10 @@ def _sanitize(data,delimiter = DEFAULT_DELIMITER):
 	else:
 		return [pattern.sub(repl,str(segment)) if segment else '' for segment in data]
 
-def _unsanitize(data,delimiter = DEFAULT_DELIMITER):
+def _unsanitize(data,delimiter = ...):
 	if not data:
 		return data
+	delimiter = get_delimiter(delimiter)
 	def repl(m):
 		tok = m.group(0)
 		if tok == "<sep>":
@@ -608,7 +676,7 @@ def _unsanitize(data,delimiter = DEFAULT_DELIMITER):
 	else:
 		return [pattern.sub(repl,str(segment).rstrip()) if segment else '' for segment in data]
 
-def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DELIMITER):
+def _formatHeader(header,verbose = False,teeLogger = None,delimiter = ...):
 	"""
 	Format the header string.
 
@@ -621,7 +689,7 @@ def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DE
 		list: The formatted header list of string.
 	"""
 	if isinstance(header,str):
-		header = header.split(delimiter)
+		header = header.split(get_delimiter(delimiter))
 	else:
 		try:
 			header = [str(s) for s in header]
@@ -631,7 +699,7 @@ def _formatHeader(header,verbose = False,teeLogger = None,delimiter = DEFAULT_DE
 			header = []
 	return [s.rstrip() for s in header]
 
-def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False,delimiter = DEFAULT_DELIMITER):
+def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = False,delimiter = ...):
 	"""
 	Verify if a line contains the header.
 
@@ -645,6 +713,7 @@ def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = Fal
 	Returns:
 	bool: True if the header matches the line, False otherwise.
 	"""
+	delimiter = get_delimiter(delimiter)
 	line = _formatHeader(line,verbose=verbose,teeLogger=teeLogger,delimiter=delimiter)
 	if verbose:
 		__teePrintOrNot(f"Header: \n{header}",teeLogger=teeLogger)
@@ -656,7 +725,7 @@ def _lineContainHeader(header,line,verbose = False,teeLogger = None,strict = Fal
 		return False
 	return True
 
-def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,header = [],encoding = 'utf8',strict = True,delimiter = DEFAULT_DELIMITER):
+def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,header = [],encoding = 'utf8',strict = True,delimiter = ...):
 	"""
 	Verify the existence of the tabular file.
 
@@ -671,6 +740,7 @@ def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,heade
 	Returns:
 	bool: True if the file exists, False otherwise.
 	"""
+	delimiter = get_delimiter(delimiter, file_name=fileName)
 	remainingFileName, _ ,extenstionName = fileName.rpartition('.')
 	if extenstionName in COMPRESSED_FILE_EXTENSIONS:
 		remainingFileName, _ ,extenstionName = remainingFileName.rpartition('.')
@@ -687,7 +757,6 @@ def _verifyFileExistence(fileName,createIfNotExist = True,teeLogger = None,heade
 			try:
 				with openFileAsCompressed(fileName, mode ='wb',encoding=encoding,teeLogger=teeLogger)as file:
 					header = delimiter.join(_sanitize(_formatHeader(header,
-													 verbose=verbose,
 													 teeLogger=teeLogger,
 													 delimiter=delimiter,
 													 ),delimiter=delimiter))
@@ -780,7 +849,7 @@ def readTabularFile(fileName,teeLogger = None,header = '',createIfNotExist = Fal
 	with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 		if any(header) and verifyHeader:
 				line = file.readline().decode(encoding=encoding,errors='replace')
-				if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict) and correctColumnNum == -1:
+				if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter) and correctColumnNum == -1:
 					correctColumnNum = len(header)
 					if verbose:
 						__teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
@@ -882,7 +951,7 @@ def appendLinesTabularFile(fileName,linesToAppend,teeLogger = None,header = '',c
 	if any(header) and verifyHeader:
 		with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 			line = file.readline().decode(encoding=encoding,errors='replace')
-			if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+			if _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter):
 				correctColumnNum = len(header)
 				if verbose:
 					__teePrintOrNot(f"correctColumnNum: {correctColumnNum}",teeLogger=teeLogger)
@@ -936,7 +1005,7 @@ def clearTabularFile(fileName,teeLogger = None,header = '',verifyHeader = False,
 		with openFileAsCompressed(fileName, mode ='rb',encoding=encoding,teeLogger=teeLogger)as file:
 			if any(header) and verifyHeader:
 				line = file.readline().decode(encoding=encoding,errors='replace')
-				if not _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict):
+				if not _lineContainHeader(header,line,verbose = verbose,teeLogger = teeLogger,strict = strict,delimiter = delimiter):
 					__teePrintOrNot(f'Warning: Header mismatch in {fileName}. Keeping original header in file...','warning',teeLogger)
 				header = _formatHeader(line,verbose = verbose,teeLogger = teeLogger,delimiter=delimiter)
 		with openFileAsCompressed(fileName, mode ='wb',encoding=encoding,teeLogger=teeLogger)as file:
@@ -1029,10 +1098,10 @@ def scrubTabularFile(fileName,teeLogger = None,header = '',createIfNotExist = Fa
 		appendLinesTabularFile(fileName,file,teeLogger = teeLogger,header = header,createIfNotExist = createIfNotExist,verifyHeader = verifyHeader,verbose = verbose,encoding = encoding,strict = strict,delimiter = delimiter)
 	return file
 
-def getListView(tsvzDic,header = [],delimiter = DEFAULT_DELIMITER):
+def getListView(tsvzDic,header = [],delimiter = ...):
 	if header:
 		if isinstance(header,str):
-			header = header.split(delimiter)
+			header = header.split(get_delimiter(delimiter))
 		elif not isinstance(header,list):
 			try:
 				header = list(header)
@@ -1076,7 +1145,7 @@ class TSVZed(OrderedDict):
 	rewrite_on_exit : bool, default=False
 		If True, rewrites the entire file when closing/exiting.
 	rewrite_interval : float, default=0
-		Minimum time interval (in seconds) between full file rewrites. 0 means no limit.
+		Minimum time interval (in seconds) between full file rewrites. 0 means do not rewrite.
 	append_check_delay : float, default=0.01
 		Time delay (in seconds) between checks of the append queue by the worker thread.
 	monitor_external_changes : bool, default=True
@@ -1605,7 +1674,7 @@ memoryOnly:{self.memoryOnly}
 			if self.header:
 				line = file.readline().decode(self.encoding,errors='replace')
 				aftPos = file.tell()
-				if not _lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = self.strict):
+				if not _lineContainHeader(self.header,line,verbose = self.verbose,teeLogger = self.teeLogger,strict = self.strict,delimiter = self.delimiter):
 					header = self.delimiter.join(_sanitize(self.header,delimiter=self.delimiter))
 					file.seek(0)
 					file.write(f'{header}\n'.encode(encoding=self.encoding,errors='replace'))
@@ -1736,6 +1805,7 @@ memoryOnly:{self.memoryOnly}
 	
 	def get_file_obj(self,modes = 'ab'):
 		self.writeLock.acquire()
+		file = None
 		try:
 			if not self.encoding:
 				self.encoding = 'utf8'
@@ -1762,6 +1832,8 @@ memoryOnly:{self.memoryOnly}
 		# if write lock is already released, return
 		if not self.writeLock.locked():
 			return
+		if file is None:
+			return
 		try:
 			file.flush()  # Ensure the file is flushed before unlocking
 			os.fsync(file.fileno())  # Ensure the file is synced to disk before unlocking
@@ -2034,13 +2106,18 @@ class TSVZedLite(MutableMapping):
 					)
 		if self.verbose:
 			eprint(f"Read at position {pos}: {segments}")
-		if key is not ... and segments[0] != key:
+		if key is not ...:
+			if not segments:
+				eprint(f"Error: No segments found at position {pos}")
+			elif segments[0] != key:
 				eprint(f"Warning: Key mismatch at position {pos}: expected {key}, got {segments[0]}")
-				if self.strict:
-					eprint("Error: Key mismatch and strict mode enabled. Raising KeyError.")
-					raise KeyError(key)
-				else :
-					eprint("Continuing despite key mismatch due to non-strict mode. Expect errors!")
+			else:
+				return segments
+			if self.strict:
+				eprint("Error: Key mismatch and strict mode enabled. Raising KeyError.")
+				raise KeyError(key)
+			else :
+				eprint("Continuing despite key mismatch due to non-strict mode. Expect errors!")
 		return segments
 
 	# Implement basic __getitem__, __setitem__, __delitem__, __iter__, and __len__. needed for MutableMapping
@@ -2093,6 +2170,7 @@ class TSVZedLite(MutableMapping):
 			self.defaults = value
 			if self.verbose:
 				eprint(f"Defaults set to {value}")
+			return
 		elif key.startswith('#'):
 			if self.verbose:
 				eprint(f"Key {key} updated in memory (data in index) as it starts with #")