TSVZ 4.4__tar.gz → 4.6__tar.gz

{tsvz-4.4 → tsvz-4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 4.4
+Version: 4.6
 Summary: Key-value store kept in a TSV / CSV / NSV / PSV file, with tsvz-spec-v1 support
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan
@@ -34,7 +34,7 @@ key-value store in a delimiter-separated text file**. The first column of each
 row is the key; the file behaves like an ordered dictionary that is persisted to
 disk as you change it.
 
-**Version 4.4**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
+**Version 4.6**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
 next to your code and `import TSVZ`, or install it.
 
 ```bash

{tsvz-4.4 → tsvz-4.6}/README.md

@@ -7,7 +7,7 @@ key-value store in a delimiter-separated text file**. The first column of each
 row is the key; the file behaves like an ordered dictionary that is persisted to
 disk as you change it.
 
-**Version 4.4**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
+**Version 4.6**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
 next to your code and `import TSVZ`, or install it.
 
 ```bash

{tsvz-4.4 → tsvz-4.6}/TSVZ.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TSVZ
-Version: 4.4
+Version: 4.6
 Summary: Key-value store kept in a TSV / CSV / NSV / PSV file, with tsvz-spec-v1 support
 Home-page: https://github.com/yufei-pan/TSVZ
 Author: Yufei Pan
@@ -34,7 +34,7 @@ key-value store in a delimiter-separated text file**. The first column of each
 row is the key; the file behaves like an ordered dictionary that is persisted to
 disk as you change it.
 
-**Version 4.4**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
+**Version 4.6**, Python **3.6+**, one self-contained module (`TSVZ.py`): drop it
 next to your code and `import TSVZ`, or install it.
 
 ```bash

{tsvz-4.4 → tsvz-4.6}/TSVZ.py

@@ -40,7 +40,7 @@ if os.name == 'nt':
 elif os.name == 'posix':
 	import fcntl
 
-version = '4.4'
+version = '4.6'
 __version__ = version
 author = 'pan@zopyr.us'
 COMMIT_DATE = '2026-10-12'
@@ -179,13 +179,18 @@ def get_terminal_size():
 	try:
 		import os
 		_tsize = os.get_terminal_size()
+		if not (_tsize[0] and _tsize[1]):
+			raise OSError('the terminal has no size set')
 	except Exception:
 		try:
 			import fcntl
 			import struct
 			import termios
 			packed = fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0))
-			_tsize = struct.unpack('HHHH', packed)[:2]
+			rows, columns = struct.unpack('HHHH', packed)[:2]
+			if not (rows and columns):
+				raise OSError('the terminal has no size set')
+			_tsize = (columns, rows)
 		except Exception:
 			import shutil
 			_tsize = shutil.get_terminal_size(fallback=(240, 50))
@@ -1654,6 +1659,34 @@ def _specFormatRecord(cells, delimiter, marker=False):
 	return delimiter.join([first] + [_specEncodeField(cell, delimiter) for cell in cells[1:]])
 
 
+def _specTombstones(keys, delimiter, strip):
+	"""``(cells, marker)`` records that delete ``keys`` (spec §9.2) from a store whose stripping is ``strip``.
+
+	While stripping is on a reader strips a tombstone's key too (spec §9.1): the
+	tombstone of a key whose encoded form ends in a space or tab, stored while
+	stripping was off, would delete the stripped key instead. A batch holding
+	one is written with stripping off, and stripping is turned back on after it
+	(spec §9.6).
+	"""
+	rows = [([key], False) for key in keys]
+	if strip and any(key.endswith((' ', '\t')) and _specEncodeField(key, delimiter, isKey=True).endswith((' ', '\t'))
+					 for key in keys):
+		rows.insert(0, (['#_strip_trailing_whites_#', 'false'], True))
+		rows.append((['#_strip_trailing_whites_#', 'true'], True))
+	return rows
+
+
+def _specFormatRecords(rows, delimiter):
+	"""``rows`` (``[(cells, marker), ...]``) as record lines, each with its '\\n'."""
+	return ''.join(_specFormatRecord(cells, delimiter, marker=marker) + '\n' for cells, marker in rows)
+
+
+def _specTombstoneLines(key, delimiter, strip):
+	"""The tombstone of ``key`` (spec §9.6) as lines joined by '\\n', without a last '\\n'."""
+	return '\n'.join(_specFormatRecord(cells, delimiter, marker=marker)
+					 for cells, marker in _specTombstones([key], delimiter, strip))
+
+
 class _SpecState(object):
 	"""Reader state of spec §12, carried across records and parts."""
 
@@ -2007,9 +2040,10 @@ def _gzipUnpad(region):
 	try:
 		inflater.decompress(stripped[start:])
 		if not inflater.eof:  # its last bytes may be zero bytes of its own
-			# Fed only zeros, compressed data can end only inside a final stored block (65535 bytes, after a
-			# header of at most 5); anywhere else it fails or never ends. So 65544 zeros give the whole answer.
-			zeros = min(zeros, 65544)
+			# Fed only zeros, compressed data in a Huffman block ends within a few hundred bytes, fails or
+			# never ends. In a stored block it can end only if the block is final, after the rest of its
+			# NLEN (both bytes zero only when LEN is 65535) and of its data: at most 2 + 65535 zeros.
+			zeros = min(zeros, 2 + 65535)
 			fed = 0
 			while fed < zeros:  # in pieces, with bounded output
 				piece = b'\0' * min(1 << 16, zeros - fed)
@@ -2603,16 +2637,23 @@ def _specBackupPart(path, reporter):
 			_lockFile(f)
 			try:
 				first = backup = '{}.damaged-{}'.format(path, time.strftime('%Y%m%dT%H%M%S'))
+				# Readable by whoever could read the part, writable by its owner only, never setuid,
+				# setgid, sticky or executable (the umask still applies on top).
+				st = os.fstat(f.fileno())
+				mode = st.st_mode & 0o644
 				suffix = 0
 				while True:
 					try:
-						copy = open(backup, 'xb')  # never overwrite an earlier backup
+						# never overwrite an earlier backup; O_BINARY: no newline translation on Windows
+						flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)
+						copy = os.fdopen(os.open(backup, flags, mode), 'wb')
 						break
 					except FileExistsError:
 						suffix += 1
 						backup = '{}.{}'.format(first, suffix)
 				try:
 					with copy:
+						_matchOwner(copy.fileno(), st)
 						shutil.copyfileobj(f, copy)
 						copy.flush()
 						os.fsync(copy.fileno())
@@ -2671,6 +2712,22 @@ def _specSnapshot(fileName):
 		return parts, None
 
 
+def _matchOwner(fd, st):
+	"""Give a new copy the owner and group of the file ``st`` describes. Where that is not allowed,
+	a copy left in another group loses its group bits, so no one reads it who could not read the file."""
+	if not hasattr(os, 'fchown'):
+		return
+	for uid in (st.st_uid, -1):
+		try:
+			os.fchown(fd, uid, st.st_gid)
+			return
+		except OSError:
+			pass
+	created = os.fstat(fd)
+	if created.st_gid != st.st_gid:
+		os.fchmod(fd, created.st_mode & 0o7707)  # what the umask left, without the group bits
+
+
 def _specRewriteInPlace(path, content, expect=None, memberSize=None):
 	"""Replace a part's bytes in place (same inode, spec §18.7 / §19.8).
 
@@ -2690,18 +2747,73 @@ def _specRewriteInPlace(path, content, expect=None, memberSize=None):
 				st = os.fstat(f.fileno())
 				if expect is not None and _specStamp(st) != expect:
 					return None
-				if st.st_size == len(raw) and f.read() == raw:
+				fd = f.fileno()
+				# The whole part, for a failed rewrite to put back whenever it fails. Read unbuffered, as
+				# the file is written below: a read-ahead in f would leave it a stale position to close at.
+				old = _readFrom0(fd, st.st_size)
+				if old == raw:
 					return False
-				f.seek(0)
-				f.write(raw)
-				f.truncate()
-				f.flush()
-				os.fsync(f.fileno())
+				try:
+					_reserveSpace(fd, len(raw))  # spec §19.8: a full disk refuses the rewrite before any byte changes
+					_writeFrom0(fd, raw)
+				except BaseException as e:
+					# Put the old bytes back rather than leave a half-rewritten part (or one a failed
+					# reservation lengthened).
+					try:
+						_writeFrom0(fd, old)
+						os.fsync(fd)
+					except OSError as undo:
+						damaged = '{}: the rewrite failed ({!r}) and the old bytes could not be put back ({}): the part is damaged'.format(
+							path, e, undo)
+						if isinstance(e, OSError):
+							raise OSError(e.errno, damaged)
+						_warn('TSVZ warning: ' + damaged)  # an interrupt stays an interrupt
+					raise
+				os.fsync(fd)
 				return True
 			finally:
+				try:
+					_seekRawStart(f)  # msvcrt unlocks from the raw position: the range it locked
+				except OSError:
+					pass
 				_unlockFile(f)
 
 
+def _reserveSpace(fd, length):
+	"""Reserve ``length`` bytes from the start of the file before it is rewritten in place (spec
+	§19.8). Where the platform or the file system cannot reserve space, nothing is reserved."""
+	fallocate = getattr(os, 'posix_fallocate', None)
+	if fallocate is None or length <= 0:
+		return
+	try:
+		fallocate(fd, 0, length)
+	except OSError as e:
+		if e.errno not in (errno.EOPNOTSUPP, getattr(errno, 'ENOTSUP', errno.EOPNOTSUPP), errno.ENOSYS, errno.EINVAL):
+			raise
+
+
+def _readFrom0(fd, length):
+	"""Up to ``length`` bytes from the start of the file (unbuffered)."""
+	os.lseek(fd, 0, os.SEEK_SET)
+	chunks = []
+	while length > 0:
+		chunk = os.read(fd, length)
+		if not chunk:
+			break
+		chunks.append(chunk)
+		length -= len(chunk)
+	return b''.join(chunks)
+
+
+def _writeFrom0(fd, data):
+	"""Write ``data`` over the file from offset 0 and cut the file there (unbuffered)."""
+	os.lseek(fd, 0, os.SEEK_SET)
+	view = memoryview(data)
+	while view:
+		view = view[os.write(fd, view):]
+	os.ftruncate(fd, len(data))
+
+
 def _specPreamble(state, delimiter, compensate=False, version=True):
 	"""Marker lines that re-create ``state`` (spec §19.2.3a): version, non-default markers, defaults last.
 
@@ -2904,7 +3016,7 @@ def _specClearTabularFile(fileName, teeLogger=None, header='', verifyHeader=Fals
 				attempt.note('refused', None, 'clear skipped: a part of the store could not be read; nothing was written')
 				return False
 			if len(load.parts) > 1:
-				tombstones = ''.join(_specFormatRecord([key], delimiter) + '\n' for key in load.data)
+				tombstones = _specFormatRecords(_specTombstones(load.data, delimiter, load.state.strip), delimiter)
 				if tombstones:
 					_specAppendPayload(load.active, tombstones.encode('utf-8'), attempt, memberSize=load.state.memberSize)
 				attempt.note('multipart', None, 'cleared a multi-part store by appending tombstones; older parts are not compacted')
@@ -3557,8 +3669,8 @@ class TSVZed(OrderedDict):
 		# the mapping by the caller, so self[key] must not be accessed.
 		self.dirty = True
 		if self._spec:
-			# Spec §9.2: a tombstone is the lone key, never padded.
-			self.appendQueue.append(_specFormatRecord([key], self.delimiter))
+			# Spec §9.2: a tombstone is the lone key, never padded. One queue item, so it is committed whole.
+			self.appendQueue.append(_specTombstoneLines(key, self.delimiter, self._specState.strip))
 			return self
 		if self.correctColumnNum > 1:
 			emptyLine = [key]+['']*(self.correctColumnNum-1)
@@ -3780,7 +3892,15 @@ memoryOnly:{self.memoryOnly}
 		return self.close()
 
 	def popitem(self, last=True):
-		key, value = super().popitem(last)
+		if sys.version_info >= (3, 11):
+			key, value = OrderedDict.popitem(self, last)
+		else:
+			# Before Python 3.11 OrderedDict.popitem goes through self[key] and del self[key], which strip the key.
+			if not OrderedDict.__len__(self):
+				raise KeyError('dictionary is empty')
+			key = next(OrderedDict.__reversed__(self)) if last else next(OrderedDict.__iter__(self))
+			value = OrderedDict.__getitem__(self, key)
+			OrderedDict.__delitem__(self, key)
 		if not self.memoryOnly:
 			self.__appendEmptyLine(key)
 		self.lastUpdateTime = get_time_ns()
@@ -4485,7 +4605,7 @@ class TSVZedLite(MutableMapping):
 		self._specWrite(_specFormatRecord(value, self.delimiter, marker=True))
 
 	def _specWrite(self, line):
-		"""Append one record to the active part; return its offset, or None for a compressed part."""
+		"""Append one record (or lines written together) to the active part; return its offset, or None for a compressed part."""
 		data = (line + '\n').encode('utf-8', errors='replace')
 		if self.fileObj is None:
 			reporter = _Reporter(self._activePath)
@@ -4658,7 +4778,7 @@ class TSVZedLite(MutableMapping):
 
 	def __mapDeleteToFile(self,key):
 		if self._spec:
-			self._specWrite(_specFormatRecord([key], self.delimiter))
+			self._specWrite(_specTombstoneLines(key, self.delimiter, self._specState.strip))
 			return
 		# Persist a deletion by appending a tombstone (a lone key with no
 		# values); on read such a row deletes the key (see _processLine). The
@@ -4802,7 +4922,7 @@ class TSVZedLite(MutableMapping):
 					eprint(f"Key {key} not found")
 				return
 			self.indexes.pop(key, None)
-			self._specWrite(_specFormatRecord([key], self.delimiter))
+			self.__mapDeleteToFile(key)
 			return
 		if key == DEFAULTS_INDICATOR_KEY:
 			self.__mapDeleteToFile(key)
@@ -5379,11 +5499,14 @@ class _ServeStore(object):
 	def _payload(self, rows, log):
 		"""Bytes for ``rows`` (``[(cells, marker), ...]``), formatted as the direct writers format them."""
 		if self.spec:
-			return ''.join(_specFormatRecord(cells, self.delimiter, marker=marker) + '\n'
-						   for cells, marker in rows).encode('utf-8')
+			return _specFormatRecords(rows, self.delimiter).encode('utf-8')
 		return _legacyFormatPayload(self.path, [cells for cells, marker in rows], log, [''], True, self.verbose,
 									'utf8', self.strict, self.delimiter)[0]
 
+	def _tombstone(self, key):
+		"""Rows that delete ``key``: written with stripping off when it would strip ``key`` (spec §9.6)."""
+		return _specTombstones([key], self.delimiter, self.state.strip) if self.spec else [([key], False)]
+
 	def _durable(self, sync):
 		return sync or (self.spec and self.state.writeAck == 'disk')
 
@@ -5421,7 +5544,7 @@ class _ServeStore(object):
 			if row is None:
 				return None
 			row = list(row)
-			queued = self._submit([([key], False)], sync, log)
+			queued = self._submit(self._tombstone(key), sync, log)
 		self._ack(queued, written=True)  # the row is only popped once its tombstone is written
 		return row
 
@@ -5435,7 +5558,7 @@ class _ServeStore(object):
 				return None
 			key = next(reversed(self.data)) if last else next(iter(self.data))
 			row = list(self.data[key])
-			queued = self._submit([([key], False)], sync, log)
+			queued = self._submit(self._tombstone(key), sync, log)
 		self._ack(queued, written=True)
 		return row
 
@@ -5620,21 +5743,32 @@ def _serveDispatch(store, line, bulk, log):
 	if operation in _SERVE_NEED_STORE and store.missing(log):
 		return 1, [], 'no such store'  # spec §20.2.2
 	unreadable = (1, _SERVE_UNREADABLE)
+	# Each read takes its status under the store lock it read under: another request's reload may
+	# change ``unreadable`` once the lock is free.
 	if operation == 'read':
-		rows = store.read(log)
-		status = unreadable if store.unreadable else (0, '')
+		with store.lock:
+			rows = store.read(log)
+			status = unreadable if store.unreadable else (0, '')
 		return status[0], [_specFormatRecord(row, '\t') for row in rows], status[1]
 	if operation == 'get':
-		rows, missing = store.get(args, log)
-		status = unreadable if store.unreadable else (3 if missing else 0, '')
+		with store.lock:
+			rows, missing = store.get(args, log)
+			status = unreadable if store.unreadable else (3 if missing else 0, '')
 		return status[0], [_specFormatRecord(row, '\t') for row in rows], status[1]
 	if operation == 'has':
-		found = store.has(args[0], log)
-		return (1, [], _SERVE_UNREADABLE) if store.unreadable else (0 if found else 3, [], '')
+		with store.lock:
+			found = store.has(args[0], log)
+			return (1, [], _SERVE_UNREADABLE) if store.unreadable else (0 if found else 3, [], '')
 	if operation == 'len':
-		return 0, [str(store.length(log))], ''
+		with store.lock:
+			count = store.length(log)
+			status = unreadable if store.unreadable else (0, '')  # spec §4.10, §20.6: a stopped read failed
+		return status[0], [str(count)], status[1]
 	if operation == 'keys':
-		return 0, [_specEncodeField(key, '\t', isKey=True) for key in store.keys(log)], ''
+		with store.lock:
+			keys = store.keys(log)
+			status = unreadable if store.unreadable else (0, '')
+		return status[0], [_specEncodeField(key, '\t', isKey=True) for key in keys], status[1]
 	if operation in ('set', 'append', 'delete'):
 		keysOnly = operation == 'delete'
 		if isBulk: