firewatcher 1.58__tar.gz → 1.59__tar.gz

{firewatcher-1.58/firewatcher.egg-info → firewatcher-1.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firewatcher
-Version: 1.58
+Version: 1.59
 Summary: Watch logs for incident patterns and persist surrounding context off-box
 Home-page: https://github.com/yufei-pan/firewatcher
 Author: Yufei Pan
@@ -59,7 +59,7 @@ On a match, `firewatcher` writes:
 - a per-incident capture under `{output-folder}/{YYYY-MM}/…`
 - a line in `{output-folder}/journal.log`
 
-Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears.
+Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears. Each captured line is flushed as it is written, and the capture file is fsynced at most once a second and when the capture closes, so a host crash loses at most about the last second of it.
 
 ## systemd service
 
@@ -99,7 +99,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--filter_only` | Filter existing logs only (`cat` instead of `tail -F`, or `journalctl` without `--follow`) |
 | `--compress-after-months` | Tar.xz monthly dirs after N months (default: 3) |
 | `--delete-after-months` | Delete `YYYY-MM` dirs after N months (default: 0 = never) |
-| `--capture_line_count_max` | Max lines before/after a match (default: 10000) |
+| `--capture_line_count_max` | Max lines before/after a match (default: 10000). Also caps how many recent lines are held in memory |
 | `--config PATH` | INI config file. Default: `/etc/firewatcher/firewatcher.conf` when that file exists. An explicit path must exist |
 | `--llm` / `--no-llm` | Enable or disable LLM analysis of each finished capture |
 | `--llm-base-url URL` | OpenAI-compatible API root, such as `http://127.0.0.1:11434/v1` |

{firewatcher-1.58 → firewatcher-1.59}/README.md

@@ -34,7 +34,7 @@ On a match, `firewatcher` writes:
 - a per-incident capture under `{output-folder}/{YYYY-MM}/…`
 - a line in `{output-folder}/journal.log`
 
-Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears.
+Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears. Each captured line is flushed as it is written, and the capture file is fsynced at most once a second and when the capture closes, so a host crash loses at most about the last second of it.
 
 ## systemd service
 
@@ -74,7 +74,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--filter_only` | Filter existing logs only (`cat` instead of `tail -F`, or `journalctl` without `--follow`) |
 | `--compress-after-months` | Tar.xz monthly dirs after N months (default: 3) |
 | `--delete-after-months` | Delete `YYYY-MM` dirs after N months (default: 0 = never) |
-| `--capture_line_count_max` | Max lines before/after a match (default: 10000) |
+| `--capture_line_count_max` | Max lines before/after a match (default: 10000). Also caps how many recent lines are held in memory |
 | `--config PATH` | INI config file. Default: `/etc/firewatcher/firewatcher.conf` when that file exists. An explicit path must exist |
 | `--llm` / `--no-llm` | Enable or disable LLM analysis of each finished capture |
 | `--llm-base-url URL` | OpenAI-compatible API root, such as `http://127.0.0.1:11434/v1` |

{firewatcher-1.58 → firewatcher-1.59/firewatcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firewatcher
-Version: 1.58
+Version: 1.59
 Summary: Watch logs for incident patterns and persist surrounding context off-box
 Home-page: https://github.com/yufei-pan/firewatcher
 Author: Yufei Pan
@@ -59,7 +59,7 @@ On a match, `firewatcher` writes:
 - a per-incident capture under `{output-folder}/{YYYY-MM}/…`
 - a line in `{output-folder}/journal.log`
 
-Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears.
+Live capture is written as lines arrive, so a copy can already be on another filesystem if the host then disappears. Each captured line is flushed as it is written, and the capture file is fsynced at most once a second and when the capture closes, so a host crash loses at most about the last second of it.
 
 ## systemd service
 
@@ -99,7 +99,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--filter_only` | Filter existing logs only (`cat` instead of `tail -F`, or `journalctl` without `--follow`) |
 | `--compress-after-months` | Tar.xz monthly dirs after N months (default: 3) |
 | `--delete-after-months` | Delete `YYYY-MM` dirs after N months (default: 0 = never) |
-| `--capture_line_count_max` | Max lines before/after a match (default: 10000) |
+| `--capture_line_count_max` | Max lines before/after a match (default: 10000). Also caps how many recent lines are held in memory |
 | `--config PATH` | INI config file. Default: `/etc/firewatcher/firewatcher.conf` when that file exists. An explicit path must exist |
 | `--llm` / `--no-llm` | Enable or disable LLM analysis of each finished capture |
 | `--llm-base-url URL` | OpenAI-compatible API root, such as `http://127.0.0.1:11434/v1` |

{firewatcher-1.58 → firewatcher-1.59}/firewatcher.py

@@ -16,16 +16,12 @@ import datetime
 import os
 import unicodedata
 import configparser
-import json
 import queue
-import secrets
 import socket
 import syslog
 import threading
-import urllib.error
-import urllib.request
 
-version = '1.58'
+version = '1.59'
 __version__ = version
 
 DEFAULT_OUTPUT_FOLDER = '/var/log/captured_messages/'
@@ -35,10 +31,13 @@ SYSTEMD_RUNTIME_DIR = '/run/systemd/system'
 SYSTEMD_UNIT_DIR = '/etc/systemd/system'
 SLUG_MAX_LEN = 80
 _OWN_SYSLOG_IDENT_RE = re.compile(r'(?:^|\s)firewatcher\[\d+\]:')
+_OWN_FILE_NEEDLE = '/' + os.path.basename(__file__)
+_GLOB_CHARS = frozenset('*?[')
 
 DEFAULT_CONFIG_PATH = '/etc/firewatcher/firewatcher.conf'
 SELF_MARKER = 'firewatcher-self'
 ANALYSIS_QUEUE_MAX = 4
+CAPTURE_SYNC_INTERVAL = 1.0
 OVERFLOW_NOTIFY_TIMEOUT = 5
 PROGRAMMATIC_TITLE_MAX = 120
 NTFY_TITLE_MAX = 200
@@ -168,41 +167,60 @@ def load_patterns(pattern_files):
 
 def _is_own_log_line(line):
 	"""True for this process's own syslog/journal lines, not incidental matching text."""
-	if SELF_MARKER in line:
-		return True
-	if _OWN_SYSLOG_IDENT_RE.search(line):
-		return True
-	base = os.path.basename(__file__)
-	if base and ('/' + base) in line:
-		return True
-	return False
+	return SELF_MARKER in line or _OWN_FILE_NEEDLE in line or _OWN_SYSLOG_IDENT_RE.search(line) is not None
 
-def match_patterns(line, patterns):
-	"""Check if the line matches any of the patterns based on the matching mode."""
-	if _is_own_log_line(line):
-		return False
-	if 'regex' in patterns:
-		if any(regex.search(line) for regex in patterns['regex']):
-			return True
-	if 'fnmatch' in patterns:
-		if any(fnmatch.fnmatch(line, pattern) for pattern in patterns['fnmatch']):
-			return True
-	return False
-	
-def get_matched_pattern(line, patterns):
-	'''Get the first matched pattern for the line'''
-	if 'regex' in patterns:
-		for regex in patterns['regex']:
-			# if regex.search(line):
-			# 	return regex.pattern
+class PatternMatcher(object):
+	"""load_patterns() output compiled once for the per-line hot path.
+
+	A `*text*` fnmatch pattern without wildcards inside is a plain substring test,
+	which is far cheaper than fnmatch's backtracking regex. Patterns are checked in
+	sorted order (regex, then substrings, then wildcard globs) so the label does not
+	depend on set order.
+	"""
+
+	def __init__(self, patterns):
+		self.regexes = sorted(patterns.get('regex', ()), key=lambda regex: (regex.pattern, regex.flags))
+		self.substrings = []
+		self.globs = []
+		for pattern in sorted(patterns.get('fnmatch', ())):
+			inner = pattern[1:-1]
+			if len(pattern) > 2 and pattern[0] == '*' and pattern[-1] == '*' and not _GLOB_CHARS.intersection(inner):
+				self.substrings.append((inner, pattern))
+			else:
+				# fnmatch.fnmatch is fnmatchcase on POSIX; this is its compiled form.
+				self.globs.append((re.compile(fnmatch.translate(pattern)).match, pattern))
+
+	def first_match(self, line):
+		"""Label of the first matching pattern: the regex match text or the fnmatch pattern."""
+		for regex in self.regexes:
 			matched = regex.search(line)
 			if matched:
 				return matched[0]
-	if 'fnmatch' in patterns:
-		for pattern in patterns['fnmatch']:
-			if fnmatch.fnmatch(line, pattern):
+		for needle, pattern in self.substrings:
+			if needle in line:
 				return pattern
-	return None
+		for match, pattern in self.globs:
+			if match(line):
+				return pattern
+		return None
+
+	def find(self, line):
+		"""Like first_match(), but this process's own log lines never match."""
+		label = self.first_match(line)
+		if label is None or _is_own_log_line(line):
+			return None
+		return label
+
+def _as_matcher(patterns):
+	return patterns if isinstance(patterns, PatternMatcher) else PatternMatcher(patterns)
+
+def match_patterns(line, patterns):
+	"""Check if the line matches any of the patterns based on the matching mode."""
+	return _as_matcher(patterns).find(line) is not None
+
+def get_matched_pattern(line, patterns):
+	'''Get the first matched pattern for the line'''
+	return _as_matcher(patterns).first_match(line)
 
 class ConfigError(Exception):
 	"""Raised when the config file or merged settings are unusable."""
@@ -440,6 +458,7 @@ def _ini_text(value):
 
 def _generate_ntfy_url():
 	"""A hostname plus 72 random bits in ntfy's allowed URL-safe alphabet."""
+	import secrets
 	prefix = 'firewatcher-'
 	suffix = secrets.token_urlsafe(9)
 	hostname = re.sub(r'[^A-Za-z0-9_-]+', '-', _hostname()).strip('-_') or 'localhost'
@@ -617,6 +636,10 @@ def _notify_body(text, path):
 
 
 def _urlopen_bytes(url, data, headers, timeout):
+	# Imported on first use: urllib.request pulls in ssl and http.client (about 9 MB RSS),
+	# and only LLM analysis and ntfy pushes need them.
+	import urllib.error
+	import urllib.request
 	request = urllib.request.Request(url, data=data, headers=headers, method='POST')
 	try:
 		with urllib.request.urlopen(request, timeout=timeout) as response:
@@ -631,6 +654,7 @@ def _urlopen_bytes(url, data, headers, timeout):
 
 
 def _parse_llm_object(content):
+	import json
 	text = str(content).strip()
 	if text.startswith('```'):
 		rows = text.splitlines()
@@ -690,6 +714,7 @@ def _capture_excerpt(path, max_bytes):
 
 
 def call_llm(settings, incident):
+	import json
 	excerpt = _capture_excerpt(incident['path'], settings.llm_max_bytes)
 	user = (
 		'Host: %s\nMatched pattern: %s\nMatched line: %s\nCapture path: %s\n\nExcerpt:\n%s'
@@ -857,30 +882,81 @@ def _start_dispatcher(settings):
 		return None
 	return IncidentDispatcher(settings)
 
-def _end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=None, dispatcher=None):
-	if not log_file_name:
+class CaptureFile(object):
+	"""Append handle kept open for one capture.
+
+	commit() flushes to the OS after each captured line, so losing this process
+	loses nothing. fsync runs at most once per sync_interval and on close, which
+	bounds what a host crash (or an NFS client cache) can lose to about that long.
+	"""
+
+	def __init__(self, path, sync_interval):
+		self.path = path
+		self.sync_interval = sync_interval
+		self.handle = open(path, 'a')
+		self.last_sync = None
+		self.dirty = False
+
+	def write(self, text):
+		self.handle.write(text)
+
+	def commit(self):
+		self.handle.flush()
+		self.dirty = True
+		now = time.time()
+		if self.last_sync is None or now - self.last_sync >= self.sync_interval:
+			self.sync(now)
+
+	def sync_due(self):
+		"""When committed data must be fsynced, or None when nothing is pending."""
+		return self.last_sync + self.sync_interval if self.dirty else None
+
+	def sync(self, now=None):
+		self.handle.flush()
+		try:
+			os.fsync(self.handle.fileno())
+		except OSError:
+			# Some filesystems cannot fsync; the data has still been flushed.
+			pass
+		self.dirty = False
+		self.last_sync = time.time() if now is None else now
+
+	def close(self):
+		try:
+			self.sync()
+		finally:
+			self.handle.close()
+
+def _end_capture(capture, captured_line_counter, capture_line_count_max, incident=None, dispatcher=None):
+	if not capture:
 		return
-	with open(log_file_name, "a") as log_file:
+	try:
 		if captured_line_counter > capture_line_count_max:
-			log_file.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Truncated to {capture_line_count_max} lines.{bcolors.ENDC}\n")
-		log_file.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
-		log_file.write(f"{bcolors.warning}End of capture at {datetime.datetime.now().isoformat()}{bcolors.ENDC}\n")
-		log_file.write(f"{bcolors.warning}Captured {captured_line_counter} lines after first match.{bcolors.ENDC}\n")
-		log_file.write(bcolors.warning+'-'*80 + bcolors.ENDC +'\n')
+			capture.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Truncated to {capture_line_count_max} lines.{bcolors.ENDC}\n")
+		capture.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
+		capture.write(f"{bcolors.warning}End of capture at {datetime.datetime.now().isoformat()}{bcolors.ENDC}\n")
+		capture.write(f"{bcolors.warning}Captured {captured_line_counter} lines after first match.{bcolors.ENDC}\n")
+		capture.write(bcolors.warning+'-'*80 + bcolors.ENDC +'\n')
 		print(f"  End of capture at {datetime.datetime.now().isoformat()}")
-		log_file.write('-'*80 +'\n')
+		capture.write('-'*80 +'\n')
+	finally:
+		capture.close()
 	if dispatcher is not None and incident is not None:
 		job = dict(incident)
-		job['path'] = log_file_name
+		job['path'] = capture.path
 		dispatcher.submit(job)
 
 
 def capture_messages(command_to_run, patterns, capture_time=300, output_folder=DEFAULT_OUTPUT_FOLDER, compress_after_months=3, delete_after_months=0,capture_line_count_max=10000, notify_settings=None):
 	"""Tail the log file and process lines with buffer management."""
-	log_buffer = deque()  # Stores tuples of (read_time, line)
+	matcher = _as_matcher(patterns)
+	# Only the newest capture_line_count_max buffered lines are ever written, so hold no more.
+	buffer_cap = max(0, capture_line_count_max)
+	log_buffer = deque(maxlen=buffer_cap)  # Stores tuples of (read_time, line)
+	dropped_until = None  # read_time of the newest line the cap pushed out of log_buffer
 	log_compressor = Log_Compressor(output_folder, compress_after_months, delete_after_months)
 	capture_until = None
-	log_file_name = None
+	capture = None
 	captured_line_counter = 0
 	processed_line_counter = 0
 	incident = None
@@ -896,17 +972,28 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 			pending_read = None
 			while True:
 				try:
-					if pending_read is None:
-						pending_read = reader.submit(proc.stdout.readline)
-					remaining = None if capture_until is None else max(0, capture_until - time.time())
-					line = pending_read.result(timeout=remaining)
-					pending_read = None
+					if pending_read is None and capture_until is None:
+						# Nothing can time out, so read inline instead of waking the reader thread per line.
+						line = proc.stdout.readline()
+					else:
+						if pending_read is None:
+							pending_read = reader.submit(proc.stdout.readline)
+						deadline = capture_until
+						sync_due = capture.sync_due() if capture else None
+						if sync_due is not None and (deadline is None or sync_due < deadline):
+							deadline = sync_due
+						remaining = None if deadline is None else max(0, deadline - time.time())
+						line = pending_read.result(timeout=remaining)
+						pending_read = None
 				except ReadTimeout:
 					# Keep the same pending read while the source is quiet.
-					_end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
+					if capture and capture_until is not None and time.time() < capture_until:
+						capture.sync()
+						continue
+					_end_capture(capture, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
 					capture_until = None
 					captured_line_counter = 0
-					log_file_name = None
+					capture = None
 					continue
 				except KeyboardInterrupt:
 					print("Exiting.")
@@ -926,15 +1013,17 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 				read_time = time.time()
 				processed_line_counter += 1
 
-				# Maintain a 10-minute buffer
+				# Keep capture_time seconds of context, at most buffer_cap lines
 				while log_buffer and read_time - log_buffer[0][0] > capture_time:
 					log_buffer.popleft()
+				if len(log_buffer) == buffer_cap:
+					dropped_until = log_buffer[0][0] if log_buffer else read_time
 				log_buffer.append((read_time, line))
 
 				# if the current line matches the pattern, capture the logs for the next capture_time seconds
-				if match_patterns(line, patterns):
+				matched_pattern = matcher.find(line)
+				if matched_pattern is not None:
 					if not capture_until:
-						matched_pattern = get_matched_pattern(line, patterns)
 						# Dump the buffer to file at output_folder/{Year-Month}/{Day-Hour-Minute-Second}.log
 						log_folder = f"{output_folder}/{datetime.datetime.fromtimestamp(read_time).strftime('%Y-%m')}"
 						os.makedirs(log_folder, exist_ok=True)
@@ -945,59 +1034,56 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 							journal_file.write(f"{datetime.datetime.fromtimestamp(read_time).isoformat()} Captured logs for {matched_pattern} to {log_file_name}\n")
 						print(f"Capturing logs to {log_file_name}")
 						incident = {
-							'pattern': matched_pattern if matched_pattern is not None else '',
+							'pattern': matched_pattern,
 							'matched_line': line.strip(),
 							'hostname': _hostname(),
 						}
-						with open(log_file_name, "a") as log_file:
-							log_file.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
-							# Write the matched pattern and the line that matched
-							log_file.write(f"{bcolors.warning}Matched pattern: {bcolors.critical}{matched_pattern}{bcolors.ENDC}\n")
-							print(f"  Matched pattern: {matched_pattern}")
-							log_file.write(f"{bcolors.warning}Matched line:    {line}{bcolors.ENDC}")
-							print(f"  Matched line:    {line.strip()}")
-							log_file.write(f"{bcolors.warning}Captured at:     {datetime.datetime.now().isoformat()}{bcolors.ENDC}\n")
-							print(f"  Captured at:     {datetime.datetime.now().isoformat()}")
-							log_file.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
-							log_file.write(f"{bcolors.warning}Logs since {datetime.datetime.fromtimestamp(read_time - capture_time).isoformat()}:{bcolors.ENDC}\n")
-							skip_lines = 0
-							if len(log_buffer) > capture_line_count_max:
-								skip_lines = len(log_buffer) - capture_line_count_max
-								log_file.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Skipping cached {skip_lines} in-memory lines ...{bcolors.ENDC}\n")
-							for i in range(skip_lines, len(log_buffer)):
-								buffered_line = log_buffer[i][1]
-								if match_patterns(buffered_line, patterns):
-									log_file.write(f'{bcolors.critical}-> {buffered_line.strip()}{bcolors.ENDC}\n')
-								else:
-									log_file.write(buffered_line)
+						capture = CaptureFile(log_file_name, CAPTURE_SYNC_INTERVAL)
+						capture.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
+						# Write the matched pattern and the line that matched
+						capture.write(f"{bcolors.warning}Matched pattern: {bcolors.critical}{matched_pattern}{bcolors.ENDC}\n")
+						print(f"  Matched pattern: {matched_pattern}")
+						capture.write(f"{bcolors.warning}Matched line:    {line}{bcolors.ENDC}")
+						print(f"  Matched line:    {line.strip()}")
+						capture.write(f"{bcolors.warning}Captured at:     {datetime.datetime.now().isoformat()}{bcolors.ENDC}\n")
+						print(f"  Captured at:     {datetime.datetime.now().isoformat()}")
+						capture.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
+						capture.write(f"{bcolors.warning}Logs since {datetime.datetime.fromtimestamp(read_time - capture_time).isoformat()}:{bcolors.ENDC}\n")
+						if dropped_until is not None and read_time - dropped_until <= capture_time:
+							capture.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Older cached in-memory lines were dropped ...{bcolors.ENDC}\n")
+						for _buffered_time, buffered_line in log_buffer:
+							if matcher.find(buffered_line) is not None:
+								capture.write(f'{bcolors.critical}-> {buffered_line.strip()}{bcolors.ENDC}\n')
+							else:
+								capture.write(buffered_line)
 					else:
 						# Append the line to the current log file
-						with open(log_file_name, "a") as log_file:
-							log_file.write(f'{bcolors.critical}-> {line.strip()}{bcolors.ENDC}\n')
+						capture.write(f'{bcolors.critical}-> {line.strip()}{bcolors.ENDC}\n')
 						captured_line_counter += 1
+					capture.commit()
 					# If a match is found, set to capture the next capture_time seconds
 					capture_until = read_time + capture_time
-					if log_file_name and captured_line_counter > capture_line_count_max:
-						_end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
+					if captured_line_counter > capture_line_count_max:
+						_end_capture(capture, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
 						capture_until = None
 						captured_line_counter = 0
-						log_file_name = None
+						capture = None
 				# if the current line doesn't match the pattern, and capture_until is set and expired, reset capture_until and close the current log file
-				elif log_file_name and ((capture_until and read_time > capture_until) or captured_line_counter > capture_line_count_max):
-					_end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
+				elif capture and ((capture_until and read_time > capture_until) or captured_line_counter > capture_line_count_max):
+					_end_capture(capture, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
 					capture_until = None
 					captured_line_counter = 0
-					log_file_name = None
+					capture = None
 				# if the current line doesn't match the pattern, continue to write the logs to the current log file if capture_until is set and not expired
 				elif capture_until and read_time <= capture_until:
-					with open(log_file_name, "a") as log_file:
-						log_file.write(line)
+					capture.write(line)
+					capture.commit()
 					captured_line_counter += 1
 					if captured_line_counter > capture_line_count_max:
-						_end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
+						_end_capture(capture, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
 						capture_until = None
 						captured_line_counter = 0
-						log_file_name = None
+						capture = None
 				# if the current line doesn't match the pattern, and capture_until is not set, check for log compression
 				else:
 					log_compressor.compressLogs()
@@ -1005,8 +1091,8 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 					print(f"Processed {processed_line_counter} lines.",flush=True)
 	finally:
 		try:
-			if log_file_name:
-				_end_capture(log_file_name, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
+			if capture:
+				_end_capture(capture, captured_line_counter, capture_line_count_max, incident=incident, dispatcher=dispatcher)
 		finally:
 			if dispatcher is not None:
 				dispatcher.close()
@@ -1280,7 +1366,7 @@ def build_parser():
 	filt.add_argument('--filter_only', dest='filter_only', action='store_true', help='Only filter logs and do not wait for new lines. Uses cat instead of tail -F, or journalctl without --follow.')
 	filt.add_argument('--no-filter_only', dest='filter_only', action='store_false', help='Follow new log lines. Overrides filter_only from the config file.')
 	parser.add_argument('--compress-after-months', type=int, default=None, help='Compress logs after this many months. Default: 3 months.')
-	parser.add_argument('--capture_line_count_max', type=int, default=None, help='Maximum number of lines to capture before and after in a single log file. Default: 10000 lines. Note: The final file maybe 2 * this number.')
+	parser.add_argument('--capture_line_count_max', type=int, default=None, help='Maximum number of lines to capture before and after in a single log file. Default: 10000 lines. Note: The final file maybe 2 * this number. Also caps how many recent lines are held in memory.')
 	parser.add_argument('--delete-after-months', type=int, default=None, help='Delete logs after this many months. (will only delete files in YYYY-MM format) Default: 0 (never).')
 	svc = parser.add_mutually_exclusive_group()
 	svc.add_argument('--install-service', action='store_true', help='Install and enable a systemd unit for this command line. Warns and exits if systemd is not available.')

{firewatcher-1.58 → firewatcher-1.59}/tests/test_capture.py

@@ -1,4 +1,7 @@
+import collections
 import datetime
+import threading
+import time
 
 import firewatcher as fw
 from conftest import Clock, capture, incident_logs, ScriptedPopen, patch_popen
@@ -157,7 +160,7 @@ def test_capture_skips_excess_pre_match_buffer(tmp_path):
     body = ''.join(f'pre-{i}\n' for i in range(10)) + 'HIT\n'
     out = capture(tmp_path, body, capture_line_count_max=3)
     text = incident_logs(out)[0].read_text()
-    assert 'Skipping cached' in text
+    assert 'Older cached in-memory lines were dropped' in text
     assert 'pre-0' not in text
     assert 'pre-8' in text
     assert 'pre-9' in text
@@ -224,3 +227,121 @@ def test_capture_rematch_extends_window(tmp_path, monkeypatch):
     text = incident_logs(out)[0].read_text()
     assert '-> HIT 2' in text
     assert 'late' in text
+
+
+def run_scripted(tmp_path, monkeypatch, popen, **kw):
+    pat = tmp_path / 'p.txt'
+    pat.write_text('HIT\n')
+    opts = dict(capture_time=60, output_folder=str(tmp_path / 'out'), compress_after_months=0, delete_after_months=0)
+    opts.update(kw)
+    patch_popen(monkeypatch, popen)
+    fw.capture_messages(['unused'], fw.load_patterns([str(pat)]), **opts)
+    return tmp_path / 'out'
+
+
+def test_pre_match_buffer_is_capped_at_capture_line_count_max(tmp_path, monkeypatch):
+    class RecordingDeque(collections.deque):
+        peak = 0
+
+        def append(self, item):
+            super().append(item)
+            RecordingDeque.peak = max(RecordingDeque.peak, len(self))
+
+    monkeypatch.setattr(fw, 'deque', RecordingDeque)
+    capture(tmp_path, ''.join(f'noise-{i}\n' for i in range(50)), capture_line_count_max=5)
+    assert 0 < RecordingDeque.peak <= 5
+
+
+def test_capture_does_not_report_dropped_lines_that_aged_out(tmp_path, monkeypatch):
+    clock = Clock(1_000_000.0)
+    monkeypatch.setattr(fw.time, 'time', clock.time)
+    popen = ScriptedPopen([
+        ('old-1\n', 0), ('old-2\n', 0), ('old-3\n', 0),
+        ('mid\n', 70), ('HIT now\n', 0),
+    ], clock=clock)
+    out = run_scripted(tmp_path, monkeypatch, popen, capture_line_count_max=2)
+    text = incident_logs(out)[0].read_text()
+    assert 'mid' in text
+    assert 'old-3' not in text
+    assert 'Older cached in-memory lines were dropped' not in text
+
+
+def test_idle_reads_stay_on_the_calling_thread(tmp_path, monkeypatch):
+    threads = []
+
+    class RecordingPopen(ScriptedPopen):
+        def __init__(self, events):
+            super().__init__(events)
+            readline = self.stdout.readline
+
+            def recording_readline():
+                threads.append(threading.current_thread())
+                return readline()
+
+            self.stdout.readline = recording_readline
+
+    run_scripted(tmp_path, monkeypatch, RecordingPopen(['noise\n', 'HIT now\n', 'after\n']))
+    main = threading.current_thread()
+    assert threads[0] is main and threads[1] is main
+    assert threads[2] is not main
+
+
+def test_capture_opens_the_capture_file_once(tmp_path, monkeypatch):
+    opened = []
+    real_open = open
+
+    def counting_open(path, *args, **kwargs):
+        opened.append(str(path))
+        return real_open(path, *args, **kwargs)
+
+    monkeypatch.setattr(fw, 'open', counting_open, raising=False)
+    out = capture(tmp_path, 'HIT now\n' + ''.join(f'after-{i}\n' for i in range(20)))
+    cap = str(incident_logs(out)[0])
+    assert opened.count(cap) == 1
+    assert 'after-19' in incident_logs(out)[0].read_text()
+
+
+def test_capture_fsyncs_at_most_once_per_interval(tmp_path, monkeypatch):
+    clock = Clock(1_000_000.0)
+    monkeypatch.setattr(fw.time, 'time', clock.time)
+    synced_at = []
+    monkeypatch.setattr(fw.os, 'fsync', lambda fd: synced_at.append(clock.t))
+    popen = ScriptedPopen([
+        ('HIT now\n', 0), ('a\n', 0.125), ('b\n', 0.25), ('c\n', 0.875),
+    ], clock=clock)
+    run_scripted(tmp_path, monkeypatch, popen)
+    start = 1_000_000.0
+    assert synced_at == [start, start + 1.25, start + 1.25]
+
+
+def test_capture_fsyncs_pending_lines_when_source_goes_quiet(tmp_path, monkeypatch):
+    monkeypatch.setattr(fw, 'CAPTURE_SYNC_INTERVAL', 0.05)
+    events = []
+    monkeypatch.setattr(fw.os, 'fsync', lambda fd: events.append('fsync'))
+
+    class QuietStdout:
+        def __init__(self):
+            self.lines = ['HIT now\n', 'more\n']
+
+        def readline(self):
+            if self.lines:
+                return self.lines.pop(0)
+            time.sleep(0.5)
+            events.append('eof')
+            return ''
+
+    popen = ScriptedPopen([])
+    popen.stdout = QuietStdout()
+    run_scripted(tmp_path, monkeypatch, popen)
+    assert events == ['fsync', 'fsync', 'eof', 'fsync']
+
+
+def test_capture_survives_filesystems_without_fsync(tmp_path, monkeypatch):
+    def no_fsync(fd):
+        raise OSError(22, 'Invalid argument')
+
+    monkeypatch.setattr(fw.os, 'fsync', no_fsync)
+    out = capture(tmp_path, 'HIT now\nafter\n')
+    text = incident_logs(out)[0].read_text()
+    assert 'after' in text
+    assert 'End of capture' in text

{firewatcher-1.58 → firewatcher-1.59}/tests/test_cli.py

@@ -102,7 +102,7 @@ def test_parser_rejects_conflicting_service_flags():
 def test_module_has_no_nebula_product_name():
     src = open(fw.__file__, encoding='utf-8').read().lower()
     assert 'nebula' not in src
-    assert fw.version == '1.58'
+    assert fw.version == '1.59'
     assert fw.__version__ == fw.version
 
 
@@ -142,3 +142,13 @@ def test_main_install_defaults_pattern_dir(monkeypatch):
 def test_print_honors_explicit_flush_false(capsys):
     fw.print('no-force', flush=False)
     assert 'no-force' in capsys.readouterr().out
+
+
+def test_import_leaves_http_and_secret_modules_unloaded():
+    heavy = ('urllib.request', 'http.client', 'ssl', 'json', 'secrets')
+    code = (
+        'import sys; sys.path.insert(0, %r); import firewatcher; '
+        'print(" ".join(m for m in %r if m in sys.modules))'
+    ) % (os.path.dirname(os.path.abspath(fw.__file__)), heavy)
+    loaded = subprocess.run([sys.executable, '-I', '-S', '-c', code], capture_output=True, text=True, check=True).stdout.split()
+    assert loaded == []

{firewatcher-1.58 → firewatcher-1.59}/tests/test_matching.py

@@ -94,3 +94,36 @@ def test_match_regex_miss_falls_through_to_fnmatch():
     assert fw.get_matched_pattern('xx HIT yy', patterns) == '*HIT*'
     assert fw.match_patterns('nope', patterns) is False
     assert fw.get_matched_pattern('nope', patterns) is None
+
+
+def test_pattern_matcher_agrees_with_fnmatch_for_wildcards_and_literals():
+    import fnmatch
+    pats = ['*I/O error*', '*a?c*', '*[0-9]x*', '*back\\slash*', '*100%*', 'exact', '*[unclosed*', '***']
+    lines = ['sd I/O error', 'zzabczz', 'a?c', 'id 7x', 'back\\slash', 'at 100%', 'exact', 'exactly',
+             '[unclosed', 'nothing', '', 'multi\nline a1c\n']
+    for pat in pats:
+        matcher = fw.PatternMatcher({'fnmatch': {pat}})
+        for line in lines:
+            expected = pat if fnmatch.fnmatch(line, pat) else None
+            assert matcher.first_match(line) == expected, (pat, line)
+
+
+def test_matched_pattern_label_does_not_depend_on_pattern_order():
+    for order in (['*error*', '*disk*'], ['*disk*', '*error*']):
+        assert fw.get_matched_pattern('disk error', {'fnmatch': order}) == '*disk*'
+    regexes = [re.compile(r'sd[a-z]'), re.compile(r'ata\d')]
+    for order in (regexes, regexes[::-1]):
+        assert fw.get_matched_pattern('ata3 sdb fail', {'regex': order}) == 'ata3'
+
+
+def test_own_line_check_runs_only_for_candidate_matches(monkeypatch):
+    calls = []
+    real = fw._is_own_log_line
+    monkeypatch.setattr(fw, '_is_own_log_line', lambda line: calls.append(line) or real(line))
+    matcher = fw.PatternMatcher({'fnmatch': {'*panic*'}, 'regex': {re.compile(r'nvme\d')}})
+    for line in ('quiet line', 'another quiet line', 'host firewatcher[12]: nothing'):
+        assert matcher.find(line) is None
+    assert calls == []
+    assert matcher.find('host firewatcher[12]: panic') is None
+    assert matcher.find('kernel panic') == '*panic*'
+    assert len(calls) == 2

{firewatcher-1.58 → firewatcher-1.59}/tests/test_notify.py

@@ -2,6 +2,7 @@ import io
 import json
 import sys
 import urllib.error
+import urllib.request
 
 import pytest
 
@@ -93,7 +94,7 @@ def test_capture_notifies_without_llm(tmp_path, monkeypatch):
         calls.append((req, timeout))
         return Body(b'ok')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(tmp_path, '[notify]\nenabled = true\nntfy_url = https://ntfy.example/alerts\npriority = 4\ntoken = secret\n')
     pat = tmp_path / 'p.txt'
     pat.write_text('HIT\n')
@@ -164,7 +165,7 @@ def test_llm_notify_true_appends_report_and_pushes(tmp_path, monkeypatch):
             return chat(payload)
         return Body(b'ok')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(
         tmp_path,
         '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\napi_key = key\n'
@@ -202,7 +203,7 @@ def test_llm_decline_does_not_push(tmp_path, monkeypatch):
         calls.append(req.full_url)
         return chat(payload)
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(
         tmp_path,
         '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\n'
@@ -228,7 +229,7 @@ def test_llm_bad_json_and_http_error_fail_open(tmp_path, monkeypatch):
             return chat('not json')
         return Body(b'ok')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', bad_json)
+    monkeypatch.setattr(urllib.request, 'urlopen', bad_json)
     resolved = settings_from(
         tmp_path,
         '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\ntimeout = 3\n'
@@ -251,7 +252,7 @@ def test_llm_bad_json_and_http_error_fail_open(tmp_path, monkeypatch):
             raise urllib.error.HTTPError(req.full_url, 503, 'down', {}, io.BytesIO(b'busy'))
         return Body(b'ok')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', http_down)
+    monkeypatch.setattr(urllib.request, 'urlopen', http_down)
     path2 = tmp_path / 'cap2.log'
     path2.write_text('HIT again\n')
     dispatcher = fw.IncidentDispatcher(resolved, start_thread=True)
@@ -267,7 +268,7 @@ def test_ntfy_failure_is_journaled(tmp_path, monkeypatch):
     def urlopen(req, timeout=None):
         raise urllib.error.URLError('unreachable')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(tmp_path, '[notify]\nenabled = true\nntfy_url = https://ntfy.example/alerts\n')
     dispatcher = fw.IncidentDispatcher(resolved, start_thread=True)
     dispatcher.submit(incident(tmp_path / 'cap.log'))
@@ -283,7 +284,7 @@ def test_queue_full_skips_llm_and_notifies_immediately(tmp_path, monkeypatch):
         calls.append((req.full_url, timeout))
         return Body(b'ok')
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(
         tmp_path,
         '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\n'
@@ -302,7 +303,7 @@ def test_queue_full_skips_llm_and_notifies_immediately(tmp_path, monkeypatch):
 
 def test_notify_only_jobs_use_higher_priority(tmp_path, monkeypatch):
     silence_syslog(monkeypatch)
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', lambda *args, **kwargs: Body(b'ok'))
+    monkeypatch.setattr(urllib.request, 'urlopen', lambda *args, **kwargs: Body(b'ok'))
     resolved = settings_from(tmp_path, '[notify]\nenabled = true\nntfy_url = https://ntfy.example/alerts\n')
     dispatcher = fw.IncidentDispatcher(resolved, start_thread=False)
     dispatcher.submit(incident(tmp_path / 'cap.log'))
@@ -351,10 +352,10 @@ def test_helper_edges(tmp_path, monkeypatch, capsys):
     def urlopen(req, timeout=None):
         raise urllib.error.HTTPError(req.full_url, 500, 'err', {}, Boom())
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     with pytest.raises(RuntimeError, match='HTTP 500'):
         fw._urlopen_bytes('http://llm.example/v1', b'{}', {}, 1)
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', lambda req, timeout=None: Body(b'not-json'))
+    monkeypatch.setattr(urllib.request, 'urlopen', lambda req, timeout=None: Body(b'not-json'))
     settings = settings_from(tmp_path, '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\n')
     with pytest.raises(ValueError, match='unreadable'):
         fw.call_llm(settings, incident(path))
@@ -362,7 +363,7 @@ def test_helper_edges(tmp_path, monkeypatch, capsys):
 
 def test_overflow_without_notify_and_unwritable_failure_note(tmp_path, monkeypatch):
     records = silence_syslog(monkeypatch)
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError('no http')))
+    monkeypatch.setattr(urllib.request, 'urlopen', lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError('no http')))
     resolved = settings_from(tmp_path, '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\n')
     dispatcher = fw.IncidentDispatcher(resolved, start_thread=False)
     for index in range(fw.ANALYSIS_QUEUE_MAX):
@@ -385,7 +386,7 @@ def test_llm_without_notify_writes_report_only(tmp_path, monkeypatch):
         calls.append(req.full_url)
         return chat(payload)
 
-    monkeypatch.setattr(fw.urllib.request, 'urlopen', urlopen)
+    monkeypatch.setattr(urllib.request, 'urlopen', urlopen)
     resolved = settings_from(
         tmp_path,
         '[llm]\nenabled = true\nbase_url = http://llm.example/v1\nmodel = m\n',