multiCMD 1.50__tar.gz → 1.51__tar.gz

{multicmd-1.50 → multicmd-1.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: multiCMD
-Version: 1.50
+Version: 1.51
 Summary: Run commands simultaneously
 Home-page: https://github.com/yufei-pan/multiCMD
 Author: Yufei Pan
@@ -209,8 +209,10 @@ print(ex.get_results())
 `run_command(s)` returns right away and queues the commands. One dispatcher thread
 starts each queued command only after taking a permit from the semaphore, so the
 executor holds at most one thread per permit plus the dispatcher, however many
-commands are queued. A queued command's `task.thread` is `None` until it starts,
-and `stop()` skips commands that have not started yet.
+commands are queued. A command's `task.thread` is `None` until it starts, and goes
+back to `None` once it has finished and the executor has joined its thread, so a
+large batch does not keep a `Thread` object per finished command. `stop()` skips
+commands that have not started yet.
 
 ### Range expansion from Python
 

{multicmd-1.50 → multicmd-1.51}/README.md

@@ -183,8 +183,10 @@ print(ex.get_results())
 `run_command(s)` returns right away and queues the commands. One dispatcher thread
 starts each queued command only after taking a permit from the semaphore, so the
 executor holds at most one thread per permit plus the dispatcher, however many
-commands are queued. A queued command's `task.thread` is `None` until it starts,
-and `stop()` skips commands that have not started yet.
+commands are queued. A command's `task.thread` is `None` until it starts, and goes
+back to `None` once it has finished and the executor has joined its thread, so a
+large batch does not keep a `Thread` object per finished command. `stop()` skips
+commands that have not started yet.
 
 ### Range expansion from Python
 

{multicmd-1.50 → multicmd-1.51}/multiCMD.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: multiCMD
-Version: 1.50
+Version: 1.51
 Summary: Run commands simultaneously
 Home-page: https://github.com/yufei-pan/multiCMD
 Author: Yufei Pan
@@ -209,8 +209,10 @@ print(ex.get_results())
 `run_command(s)` returns right away and queues the commands. One dispatcher thread
 starts each queued command only after taking a permit from the semaphore, so the
 executor holds at most one thread per permit plus the dispatcher, however many
-commands are queued. A queued command's `task.thread` is `None` until it starts,
-and `stop()` skips commands that have not started yet.
+commands are queued. A command's `task.thread` is `None` until it starts, and goes
+back to `None` once it has finished and the executor has joined its thread, so a
+large batch does not keep a `Thread` object per finished command. `stop()` skips
+commands that have not started yet.
 
 ### Range expansion from Python
 

{multicmd-1.50 → multicmd-1.51}/multiCMD.py

@@ -23,9 +23,9 @@ import time
 from pprint import pformat
 
 #%% global vars
-version = '1.50'
+version = '1.51'
 __version__ = version
-COMMIT_DATE = '2026-10-09'
+COMMIT_DATE = '2026-10-10'
 # fire-and-forget threads (run_commands with wait_for_return=False) that join_threads() waits on
 __running_threads = set()
 __running_threads_lock = threading.Lock()
@@ -96,8 +96,9 @@ class AsyncExecutor:
 
 		Commands are queued and a dispatcher thread starts each one only after taking a permit
 		from the semaphore, so the executor holds at most one thread per permit it uses plus the
-		dispatcher, however many commands are queued. A queued command's task.thread stays None
-		until it starts.
+		dispatcher, however many commands are queued. A command's task.thread is None until it
+		starts, and goes back to None once it has finished and the executor has joined its thread,
+		so finished tasks do not keep their Thread objects.
 		@params:
 			max_threads: The maximum number of threads to use ( int ) ( Note: if passing semaphore, this likely will be ignored )
 			semaphore: The semaphore to use for threading ( threading.Semaphore )
@@ -158,23 +159,31 @@ class AsyncExecutor:
 		'''
 		while True:
 			try:
-				thread = self._finished.popleft()
+				thread, task = self._finished.popleft()
 			except IndexError:
 				return
 			thread.join()
 			with self._lock:
 				self._threads.pop(thread, None)
+				# a finished Thread object costs ~2 KB, and the task outlives it in self.tasks
+				if task is not None and task.thread is thread:
+					task.thread = None
 
-	def _start_thread(self, target, args):
+	def _start_thread(self, target, args, task=None):
 		thread = threading.Thread(target=target, args=args, daemon=True)
 		# registered before it starts, so it is tracked before it can report itself finished
 		with self._lock:
 			self._threads[thread] = None
+			if task is not None:
+				# set before it starts, so _reap() never sees the task without its thread
+				task.thread = thread
 			try:
 				thread.start()
 			except BaseException:
 				# never started: it would never report itself finished
 				del self._threads[thread]
+				if task is not None:
+					task.thread = None
 				raise
 		return thread
 
@@ -199,7 +208,7 @@ class AsyncExecutor:
 					continue
 				self._reap()
 				try:
-					task.thread = self._start_thread(_run_executor_task, (task, _HeldPermit(sem), timeout, quiet, dry_run, self._finished))
+					self._start_thread(_run_executor_task, (task, _HeldPermit(sem), timeout, quiet, dry_run, self._finished), task)
 				except BaseException:
 					sem.release()
 					raise
@@ -209,7 +218,7 @@ class AsyncExecutor:
 				self._dispatching = False
 			raise
 		finally:
-			self._finished.append(threading.current_thread())
+			self._finished.append((threading.current_thread(), None))
 
 	def run_commands(self, commands, timeout=...,max_threads=...,quiet=...,dry_run=...,parse = ...,sem = ...):
 		'''
@@ -696,12 +705,12 @@ def __run_command(task,sem, timeout=60, quiet=False,dry_run=False,with_stdErr=Fa
 
 def _run_executor_task(task, permit, timeout, quiet, dry_run, finished):
 	'''
-	Run an AsyncExecutor command, then report this thread finished so the executor joins it ( internal )
+	Run an AsyncExecutor command, then report this thread finished so the executor joins it and drops it from the task ( internal )
 	'''
 	try:
 		__run_command(task, permit, timeout, quiet, dry_run, identity=...)
 	finally:
-		finished.append(threading.current_thread())
+		finished.append((threading.current_thread(), task))
 
 def __run_tracked_command(task, sem, timeout, quiet, dry_run, identity):
 	'''

{multicmd-1.50 → multicmd-1.51}/tests/test_async_executor.py

@@ -69,7 +69,10 @@ def _stack_mappings(size):
 	return count
 
 
-def test_executor_queues_commands_instead_of_parking_threads():
+def test_executor_queues_commands_instead_of_parking_threads(monkeypatch):
+	ran = []
+	run_task = multiCMD._run_executor_task
+	monkeypatch.setattr(multiCMD, '_run_executor_task', lambda task, *args: (ran.append(task), run_task(task, *args)))
 	gate = threading.Semaphore(0)
 	baseline = threading.active_count()
 	ex = AsyncExecutor(semaphore=gate, quiet=True, dry_run=True)
@@ -83,7 +86,8 @@ def test_executor_queues_commands_instead_of_parking_threads():
 		gate.release()
 	ex.wait(timeout=-1)
 	assert ex.runningThreads == []
-	assert all(task.thread is not None and not task.is_alive() for task in tasks)
+	assert sorted(map(id, ran)) == sorted(map(id, tasks))
+	assert all(task.thread is None and not task.is_alive() for task in tasks)
 
 
 @pytest.mark.slow
@@ -121,9 +125,15 @@ def test_executor_releases_finished_thread_stacks():
 
 def test_executor_threads_stay_out_of_global_set():
 	ex = AsyncExecutor(max_threads=2, quiet=True)
-	tasks = ex.run_commands([['true'], ['true']])
+	tasks = ex.run_commands([['sleep', '0.3'], ['sleep', '0.3']])
+	# finished tasks drop their thread, so take the threads while the commands run
+	deadline = time.monotonic() + 5
+	while not all(task.thread for task in tasks) and time.monotonic() < deadline:
+		time.sleep(0.01)
+	threads = {task.thread for task in tasks}
+	assert None not in threads
+	assert not threads & _global_threads()
 	ex.wait(timeout=-1)
-	assert not {task.thread for task in tasks} & _global_threads()
 
 
 def test_finished_fire_and_forget_threads_leave_global_set():
@@ -134,6 +144,15 @@ def test_finished_fire_and_forget_threads_leave_global_set():
 	assert not {task.thread for task in tasks} & _global_threads()
 
 
+def test_finished_executor_tasks_release_their_threads():
+	ex = AsyncExecutor(max_threads=4, quiet=True)
+	tasks = ex.run_commands([['true']] * 20)
+	ex.wait(timeout=-1)
+	assert [task.returncode for task in tasks] == [0] * 20
+	# a finished Thread object costs ~2 KB, kept for as long as the executor keeps the task
+	assert all(task.thread is None and not task.is_alive() for task in tasks)
+
+
 def test_stop_skips_queued_commands():
 	ex = AsyncExecutor(semaphore=threading.Semaphore(0), quiet=True)
 	tasks = ex.run_commands([['sleep', '5']] * 3)