firewatcher 1.60__tar.gz → 1.61__tar.gz

{firewatcher-1.60/firewatcher.egg-info → firewatcher-1.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firewatcher
-Version: 1.60
+Version: 1.61
 Summary: Watch logs for incident patterns and persist surrounding context off-box
 Home-page: https://github.com/yufei-pan/firewatcher
 Author: Yufei Pan
@@ -108,6 +108,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--llm-model NAME` | Model name |
 | `--llm-timeout SEC` | LLM HTTP timeout (default: 60) |
 | `--llm-max-bytes N` | Capture bytes sent to the model (default: 48000) |
+| `--llm-notes TEXT` | Short note sent to the model with each capture, such as what is normal on this host (default: none) |
 | `--notify` / `--no-notify` | Enable or disable ntfy pushes |
 | `--ntfy-url URL` | ntfy topic URL, such as `https://ntfy.sh/mytopic` |
 | `--ntfy-priority N` | ntfy priority 1–5 (default: 4) |
@@ -136,6 +137,7 @@ base_url = http://127.0.0.1:11434/v1
 model = llama3.1
 timeout = 60
 max_bytes = 48000
+notes = loop/dm devices from pytest and OOMs in capped test containers are expected
 
 [notify]
 enabled = false
@@ -159,6 +161,10 @@ Both features default off.
 - Both on: a push is sent only when the model sets `notify` to true.
 - If the LLM request times out, returns an HTTP error, or is not the expected JSON, and notifications are on, firewatcher still pushes, using the generated title.
 
+A capture larger than `max_bytes` is not sent whole. The model gets the first 2 KB of the file (the header and the oldest lines before the match; at most half of `max_bytes`), then a window around the matched line: a quarter of the remaining budget before it and the rest from it onward, with a `...` line wherever bytes were skipped. A window that reaches either end of the file gives its unused share to the other side. A match during a capture only extends that capture, so the matched lines after the window are sent too, in up to a quarter of the budget, newest first when they do not all fit, followed by the capture's last lines (often a retry that worked or a service back up) and its closing lines (line count, truncation), in up to an eighth of the budget.
+
+`notes` under `[llm]` (also accepted as `note`) is optional free text, empty by default. When set, it is sent with each capture as `Operator notes about this host: ...`, and the model is told it describes what is normal there. Whitespace and newlines in a note collapse to single spaces.
+
 The capture window closes even when no further log lines arrive. Analysis runs on a background worker so log following stays live. The queue holds four captures. Notify-only jobs run ahead of jobs that still need the model. When the queue is full, that incident skips the model and, if notifications are on, the capture thread sends the generated-title push itself.
 
 Lines firewatcher writes back to the journal are a single physical line containing `firewatcher-self`, so a summary that quotes `panic` or `I/O error` does not start another capture. The capture files themselves stay in the log stream's original format.

{firewatcher-1.60 → firewatcher-1.61}/README.md

@@ -81,6 +81,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--llm-model NAME` | Model name |
 | `--llm-timeout SEC` | LLM HTTP timeout (default: 60) |
 | `--llm-max-bytes N` | Capture bytes sent to the model (default: 48000) |
+| `--llm-notes TEXT` | Short note sent to the model with each capture, such as what is normal on this host (default: none) |
 | `--notify` / `--no-notify` | Enable or disable ntfy pushes |
 | `--ntfy-url URL` | ntfy topic URL, such as `https://ntfy.sh/mytopic` |
 | `--ntfy-priority N` | ntfy priority 1–5 (default: 4) |
@@ -109,6 +110,7 @@ base_url = http://127.0.0.1:11434/v1
 model = llama3.1
 timeout = 60
 max_bytes = 48000
+notes = loop/dm devices from pytest and OOMs in capped test containers are expected
 
 [notify]
 enabled = false
@@ -132,6 +134,10 @@ Both features default off.
 - Both on: a push is sent only when the model sets `notify` to true.
 - If the LLM request times out, returns an HTTP error, or is not the expected JSON, and notifications are on, firewatcher still pushes, using the generated title.
 
+A capture larger than `max_bytes` is not sent whole. The model gets the first 2 KB of the file (the header and the oldest lines before the match; at most half of `max_bytes`), then a window around the matched line: a quarter of the remaining budget before it and the rest from it onward, with a `...` line wherever bytes were skipped. A window that reaches either end of the file gives its unused share to the other side. A match during a capture only extends that capture, so the matched lines after the window are sent too, in up to a quarter of the budget, newest first when they do not all fit, followed by the capture's last lines (often a retry that worked or a service back up) and its closing lines (line count, truncation), in up to an eighth of the budget.
+
+`notes` under `[llm]` (also accepted as `note`) is optional free text, empty by default. When set, it is sent with each capture as `Operator notes about this host: ...`, and the model is told it describes what is normal there. Whitespace and newlines in a note collapse to single spaces.
+
 The capture window closes even when no further log lines arrive. Analysis runs on a background worker so log following stays live. The queue holds four captures. Notify-only jobs run ahead of jobs that still need the model. When the queue is full, that incident skips the model and, if notifications are on, the capture thread sends the generated-title push itself.
 
 Lines firewatcher writes back to the journal are a single physical line containing `firewatcher-self`, so a summary that quotes `panic` or `I/O error` does not start another capture. The capture files themselves stay in the log stream's original format.

{firewatcher-1.60 → firewatcher-1.61}/examples/firewatcher.conf

@@ -36,6 +36,8 @@ api_key =
 timeout = 60
 # Maximum capture bytes sent to the model.
 max_bytes = 48000
+# Optional note sent to the model with each capture, e.g. what is normal on this host.
+notes =
 
 [notify]
 # Enable pushes; when LLM analysis is on, the model decides whether to notify.

{firewatcher-1.60 → firewatcher-1.61/firewatcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firewatcher
-Version: 1.60
+Version: 1.61
 Summary: Watch logs for incident patterns and persist surrounding context off-box
 Home-page: https://github.com/yufei-pan/firewatcher
 Author: Yufei Pan
@@ -108,6 +108,7 @@ sudo firewatcher --install-service -o /mnt/logs/captured_messages \
 | `--llm-model NAME` | Model name |
 | `--llm-timeout SEC` | LLM HTTP timeout (default: 60) |
 | `--llm-max-bytes N` | Capture bytes sent to the model (default: 48000) |
+| `--llm-notes TEXT` | Short note sent to the model with each capture, such as what is normal on this host (default: none) |
 | `--notify` / `--no-notify` | Enable or disable ntfy pushes |
 | `--ntfy-url URL` | ntfy topic URL, such as `https://ntfy.sh/mytopic` |
 | `--ntfy-priority N` | ntfy priority 1–5 (default: 4) |
@@ -136,6 +137,7 @@ base_url = http://127.0.0.1:11434/v1
 model = llama3.1
 timeout = 60
 max_bytes = 48000
+notes = loop/dm devices from pytest and OOMs in capped test containers are expected
 
 [notify]
 enabled = false
@@ -159,6 +161,10 @@ Both features default off.
 - Both on: a push is sent only when the model sets `notify` to true.
 - If the LLM request times out, returns an HTTP error, or is not the expected JSON, and notifications are on, firewatcher still pushes, using the generated title.
 
+A capture larger than `max_bytes` is not sent whole. The model gets the first 2 KB of the file (the header and the oldest lines before the match; at most half of `max_bytes`), then a window around the matched line: a quarter of the remaining budget before it and the rest from it onward, with a `...` line wherever bytes were skipped. A window that reaches either end of the file gives its unused share to the other side. A match during a capture only extends that capture, so the matched lines after the window are sent too, in up to a quarter of the budget, newest first when they do not all fit, followed by the capture's last lines (often a retry that worked or a service back up) and its closing lines (line count, truncation), in up to an eighth of the budget.
+
+`notes` under `[llm]` (also accepted as `note`) is optional free text, empty by default. When set, it is sent with each capture as `Operator notes about this host: ...`, and the model is told it describes what is normal there. Whitespace and newlines in a note collapse to single spaces.
+
 The capture window closes even when no further log lines arrive. Analysis runs on a background worker so log following stays live. The queue holds four captures. Notify-only jobs run ahead of jobs that still need the model. When the queue is full, that incident skips the model and, if notifications are on, the capture thread sends the generated-title push itself.
 
 Lines firewatcher writes back to the journal are a single physical line containing `firewatcher-self`, so a summary that quotes `panic` or `I/O error` does not start another capture. The capture files themselves stay in the log stream's original format.

{firewatcher-1.60 → firewatcher-1.61}/firewatcher.py

@@ -21,7 +21,7 @@ import socket
 import syslog
 import threading
 
-version = '1.60'
+version = '1.61'
 __version__ = version
 
 DEFAULT_OUTPUT_FOLDER = '/var/log/captured_messages/'
@@ -53,6 +53,7 @@ LLM_SYSTEM_PROMPT = (
 	'Keys: "title" (short, one line), "summary" (plain text, a few sentences), '
 	'"notify" (boolean: true only if a person should look soon; '
 	'false for benign, expected, or duplicate-looking noise). '
+	'Operator notes about the host, when present, describe what is normal there. '
 	'Do not invent hosts, times, or causes that are not in the excerpt.'
 )
 
@@ -99,9 +100,16 @@ class Log_Compressor:
 		self.lastProcessTime = time.time()
 
 def print(*args, **kwargs):
-	'''Print with flush=True by default.'''
+	'''Print with flush=True by default. Text the stream can't encode (Python 3.6 under a C locale) is
+	escaped instead of stopping the watcher.'''
 	kwargs.setdefault('flush', True)
-	return builtins.print(*args, **kwargs)
+	sep = kwargs.pop('sep', None)
+	# One string, so a line that can't be encoded fails before any of it is written.
+	text = (' ' if sep is None else sep).join(str(arg) for arg in args)
+	try:
+		return builtins.print(text, **kwargs)
+	except UnicodeEncodeError:
+		return builtins.print(text.encode('ascii', 'backslashreplace').decode('ascii'), **kwargs)
 
 class bcolors:
 		HEADER = '\033[95m'
@@ -143,7 +151,7 @@ def load_patterns(pattern_files):
 			print(f"Pattern file not found: {pattern_file}")
 			continue
 		try:
-			with open(pattern_file, 'r') as f:
+			with open(pattern_file, 'r', encoding='utf-8', errors='replace') as f:
 				patterns = [line.strip() for line in f if line.strip()]
 		except Exception as e:
 			print(f"Error reading pattern file: {pattern_file}")
@@ -251,6 +259,7 @@ class Settings(object):
 		self.llm_api_key = ''
 		self.llm_timeout = 60
 		self.llm_max_bytes = 48000
+		self.llm_notes = ''
 		self.notify_enabled = False
 		self.ntfy_url = ''
 		self.ntfy_token = ''
@@ -340,6 +349,9 @@ def _read_config_file(path):
 	take('llm', 'api_key', 'llm_api_key', 'str')
 	take('llm', 'timeout', 'llm_timeout', 'int')
 	take('llm', 'max_bytes', 'llm_max_bytes', 'int')
+	take('llm', 'notes', 'llm_notes', 'str')
+	if not values.get('llm_notes'):
+		take('llm', 'note', 'llm_notes', 'str')
 	take('notify', 'enabled', 'notify_enabled', 'bool')
 	take('notify', 'ntfy_url', 'ntfy_url', 'str')
 	take('notify', 'token', 'ntfy_token', 'str')
@@ -384,7 +396,7 @@ def resolve_settings(args, environ=None):
 		settings.output_folder = file_values['output_folder']
 	if file_values.get('tail_lines'):
 		settings.tail_lines = file_values['tail_lines']
-	for key in ('llm_base_url', 'llm_model', 'llm_api_key', 'ntfy_url', 'ntfy_token'):
+	for key in ('llm_base_url', 'llm_model', 'llm_api_key', 'llm_notes', 'ntfy_url', 'ntfy_token'):
 		if key in file_values:
 			setattr(settings, key, file_values[key])
 	if getattr(args, 'pattern_file', None):
@@ -415,6 +427,8 @@ def resolve_settings(args, environ=None):
 		settings.llm_timeout = args.llm_timeout
 	if getattr(args, 'llm_max_bytes', None) is not None:
 		settings.llm_max_bytes = args.llm_max_bytes
+	if getattr(args, 'llm_notes', None) is not None:
+		settings.llm_notes = args.llm_notes
 	if getattr(args, 'notify', None) is not None:
 		settings.notify_enabled = bool(args.notify)
 	if getattr(args, 'ntfy_url', None) is not None:
@@ -427,6 +441,7 @@ def resolve_settings(args, environ=None):
 		settings.llm_api_key = environ[LLM_API_KEY_ENV]
 	if NTFY_TOKEN_ENV in environ:
 		settings.ntfy_token = environ[NTFY_TOKEN_ENV]
+	settings.llm_notes = _note_text(settings.llm_notes)
 	settings.llm_base_url = (settings.llm_base_url or '').strip()
 	settings.llm_model = (settings.llm_model or '').strip()
 	settings.ntfy_url = (settings.ntfy_url or '').strip()
@@ -452,6 +467,33 @@ def _ini_patterns(paths):
 	return first + ''.join('\n\t' + item for item in paths[1:])
 
 
+def _as_utf8(text):
+	"""Text that can be written or sent as UTF-8: bytes Python 3.6 decoded with surrogateescape (argv and
+	paths under a C locale) are decoded as UTF-8, and anything invalid is replaced."""
+	text = str(text)
+	try:
+		return text.encode('utf-8', 'surrogateescape').decode('utf-8', 'replace')
+	except UnicodeEncodeError:  # a lone surrogate that is not an escaped byte, e.g. from a JSON reply
+		return text.encode('utf-8', 'replace').decode('utf-8')
+
+
+def _strict_utf8(text):
+	"""Text for a config or unit file, with the bytes Python decoded from argv, paths or the environment
+	with surrogateescape restored (any byte the locale can't decode: non-UTF-8 bytes, or every non-ASCII
+	byte on Python 3.6 under a C locale). Bytes that are not UTF-8 are refused, not replaced: replacing
+	them would point the service at a different path. The message shows no value, since values include
+	the API key, the ntfy token and the private ntfy topic URL."""
+	try:
+		return text.encode('utf-8', 'surrogateescape').decode('utf-8')
+	except UnicodeError:
+		raise ConfigError('a path or value is not UTF-8, so it cannot go into a config or unit file')
+
+
+def _note_text(value):
+	"""A note as one line: any run of whitespace, newlines included, becomes one space."""
+	return ' '.join(str('' if value is None else value).split())
+
+
 def _ini_text(value):
 	return str('' if value is None else value).replace('\r', '').replace('\n', '').strip()
 
@@ -506,6 +548,8 @@ def render_config_text(settings):
 		'timeout = %s' % int(settings.llm_timeout),
 		'# Maximum capture bytes sent to the model.',
 		'max_bytes = %s' % int(settings.llm_max_bytes),
+		'# Optional note sent to the model with each capture, e.g. what is normal on this host.',
+		'notes = %s' % settings.llm_notes,
 		'',
 		'[notify]',
 		'# Enable pushes; when LLM analysis is on, the model decides whether to notify.',
@@ -525,10 +569,14 @@ def render_config_text(settings):
 
 
 def write_config_file(path, settings):
+	_write_private_file(path, _strict_utf8(render_config_text(settings)))
+
+
+def _write_private_file(path, text):
+	payload = text.encode('utf-8')
 	directory = os.path.dirname(path)
 	if directory:
 		os.makedirs(directory, exist_ok=True)
-	payload = render_config_text(settings).encode('utf-8')
 	fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
 	try:
 		os.write(fd, payload)
@@ -570,6 +618,8 @@ def _explicit_cli_args(raw):
 		parts.extend(['--llm-timeout', str(raw.llm_timeout)])
 	if getattr(raw, 'llm_max_bytes', None) is not None:
 		parts.extend(['--llm-max-bytes', str(raw.llm_max_bytes)])
+	if getattr(raw, 'llm_notes', None) is not None:
+		parts.append('--llm-notes=' + _note_text(raw.llm_notes))
 	if getattr(raw, 'notify', None) is True:
 		parts.append('--notify')
 	elif getattr(raw, 'notify', None) is False:
@@ -626,12 +676,12 @@ def _http_header(text, limit):
 
 
 def _notify_body(text, path):
-	suffix = '\n' + str(path)
+	suffix = '\n' + _as_utf8(path)
 	suffix_b = suffix.encode('utf-8')
 	if len(suffix_b) >= NTFY_BODY_MAX:
 		return suffix_b[:NTFY_BODY_MAX].decode('utf-8', 'ignore')
 	budget = NTFY_BODY_MAX - len(suffix_b)
-	body = _flatten(text).encode('utf-8')[:budget]
+	body = _as_utf8(_flatten(text)).encode('utf-8')[:budget]
 	return body.decode('utf-8', 'ignore') + suffix
 
 
@@ -689,17 +739,92 @@ def _parse_llm_object(content):
 	return {'title': title, 'summary': str(summary), 'notify': obj['notify']}
 
 
-def _capture_excerpt(path, max_bytes):
+def _excerpt_window(data, head_end, start, body, limit):
+	"""Bytes [lo, hi) around start: a quarter of body before it and the rest from it on, the unused share
+	of a side that reaches head_end or limit going to the other, cut at line ends. A bigger body never
+	gives a smaller window, which _capture_excerpt relies on."""
+	before = min(body // 4, start - head_end)
+	after = min(body - before, limit - start)
+	before = min(body - after, start - head_end)
+	lo = start - before
+	if lo > head_end:
+		# start - 1 is a newline here, so one is always found; a line too long for the share before the
+		# matched line is dropped, not cut. Its bytes stay unused: handing them to the after side would
+		# let a bigger body end the window earlier.
+		lo = data.find(b'\n', lo - 1, start) + 1
+	hi = start + after
+	if hi < limit:
+		nl = data.rfind(b'\n', start, hi)
+		hi = nl + 1 if nl != -1 else hi
+	return lo, hi
+
+
+def _capture_excerpt(path, max_bytes, matched_offset=None, later_offsets=(), footer_offset=None):
+	"""At most max_bytes of a capture for the model.
+
+	A capture that fits goes whole. Otherwise the model gets the header (the start of the file), a window
+	around the matched line (a quarter of the budget before it, the rest from it on), the matched lines
+	after the window (a match during a capture only extends it; up to a quarter of the budget, newest
+	first, so the end of an escalation is kept) and the capture's last lines with its footer (an eighth). Offsets are bytes into the file,
+	as capture_messages recorded them. Every skipped stretch is marked with a "..." line. Without a
+	matched offset it is head + tail.
+	"""
 	max_bytes = int(max_bytes)
 	if max_bytes < 1:
 		return ''
 	with open(path, 'rb') as handle:
 		data = handle.read()
-	if len(data) <= max_bytes:
+	size = len(data)
+	head_n = min(EXCERPT_HEAD_BYTES, max(1, max_bytes // 2))
+	sep = b'\n...\n'
+	if size <= max_bytes:
 		text = data.decode('utf-8', 'replace')
+	elif matched_offset is not None and 0 <= matched_offset < size and max_bytes - head_n > 2 * len(sep):
+		head_end = data.rfind(b'\n', 0, head_n) + 1 or head_n
+		# The matched line starts at a line start; if the offset is stale, snap to the line it is in.
+		start = max(head_end, data.rfind(b'\n', 0, matched_offset) + 1)
+		# Each piece pays for the "..." before it; the window's two are in body.
+		body = max_bytes - head_end - 2 * len(sep)
+		# The capture's last lines (often a retry that worked or a service back up) and its footer, in up
+		# to an eighth of the budget; the window and the later matches stop where they start.
+		limit = size
+		end_share = body // 8 - len(sep)
+		if footer_offset is not None and start < footer_offset < size and size - footer_offset <= end_share:
+			# A last raw log line may lack its newline: then the end piece starts at the footer.
+			limit = data.find(b'\n', size - end_share - 1, footer_offset) + 1 or footer_offset
+			if limit <= start:
+				limit = footer_offset
+			body -= size - limit + len(sep)
+		later = []
+		for pos in sorted(p for p in later_offsets or () if start < p < limit):
+			nl = data.find(b'\n', pos, limit)
+			later.append((pos, data[pos:limit if nl == -1 else nl + 1]))
+		# Room for the later matches (up to a quarter) comes first, and the newest outside the window that
+		# fit are kept. The window then gets the rest of the budget; it only grows, so whatever it held
+		# stays, and a kept line it now reaches is dropped and its bytes go back to the window.
+		room = min(sum(len(line) + len(sep) for _pos, line in later), body // 4)
+		lo, hi = _excerpt_window(data, head_end, start, body - room, limit)
+		kept, used = [], 0
+		for pos, line in reversed(later):
+			if pos >= hi and used + len(line) + len(sep) <= room:
+				kept.append((pos, line))
+				used += len(line) + len(sep)
+		while True:
+			lo, hi = _excerpt_window(data, head_end, start, body - used, limit)
+			if all(pos >= hi for pos, _line in kept):
+				break
+			kept = [(pos, line) for pos, line in kept if pos >= hi]
+			used = sum(len(line) + len(sep) for _pos, line in kept)
+		kept.sort()
+		pieces = [(lo, data[lo:hi])] + kept + ([(limit, data[limit:])] if limit < size else [])
+		out, end = [data[:head_end]], head_end
+		for pos, piece in pieces:
+			out.extend([sep, piece] if pos > end else [piece])
+			end = pos + len(piece)
+		if end < size:
+			out.append(sep)
+		text = b''.join(out).decode('utf-8', 'replace')
 	else:
-		head_n = min(EXCERPT_HEAD_BYTES, max(1, max_bytes // 2))
-		sep = b'\n...\n'
 		tail_n = max_bytes - head_n - len(sep)
 		if tail_n < 1:
 			sep = b''
@@ -715,11 +840,17 @@ def _capture_excerpt(path, max_bytes):
 
 def call_llm(settings, incident):
 	import json
-	excerpt = _capture_excerpt(incident['path'], settings.llm_max_bytes)
+	excerpt = _capture_excerpt(
+		incident['path'], settings.llm_max_bytes, incident.get('matched_offset'),
+		incident.get('later_offsets'), incident.get('footer_offset'))
+	notes = settings.llm_notes
 	user = (
-		'Host: %s\nMatched pattern: %s\nMatched line: %s\nCapture path: %s\n\nExcerpt:\n%s'
-		% (incident.get('hostname', ''), incident.get('pattern', ''), incident.get('matched_line', ''), incident.get('path', ''), excerpt)
+		'Host: %s\n%sMatched pattern: %s\nMatched line: %s\nCapture path: %s\n\nExcerpt:\n%s'
+		% (incident.get('hostname', ''), 'Operator notes about this host: %s\n' % notes if notes else '',
+			incident.get('pattern', ''), incident.get('matched_line', ''), incident.get('path', ''), excerpt)
 	)
+	# Not lone surrogates, which a JSON parser may refuse.
+	user = _as_utf8(user)
 	url = settings.llm_base_url.rstrip('/') + '/chat/completions'
 	payload = {
 		'model': settings.llm_model,
@@ -754,7 +885,7 @@ def send_ntfy(url, title, body, priority, token, timeout):
 
 def _append_note(path, text):
 	rows = str(text).splitlines() or ['']
-	with open(path, 'a', encoding='utf-8') as handle:
+	with open(path, 'a', encoding='utf-8', errors='replace') as handle:
 		handle.write('\n')
 		for row in rows:
 			handle.write('%s %s\n' % (SELF_MARKER, row))
@@ -893,13 +1024,18 @@ class CaptureFile(object):
 	def __init__(self, path, sync_interval):
 		self.path = path
 		self.sync_interval = sync_interval
-		self.handle = open(path, 'a')
+		self.handle = open(path, 'a', encoding='utf-8')
 		self.last_sync = None
 		self.dirty = False
 
 	def write(self, text):
 		self.handle.write(text)
 
+	def offset(self):
+		"""Bytes in the file so far, i.e. where the next write starts."""
+		self.handle.flush()
+		return self.handle.buffer.tell()
+
 	def commit(self):
 		self.handle.flush()
 		self.dirty = True
@@ -931,6 +1067,8 @@ def _end_capture(capture, captured_line_counter, capture_line_count_max, inciden
 	if not capture:
 		return
 	try:
+		if incident is not None:
+			incident['footer_offset'] = capture.offset()
 		if captured_line_counter > capture_line_count_max:
 			capture.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Truncated to {capture_line_count_max} lines.{bcolors.ENDC}\n")
 		capture.write(bcolors.warning + '-'*80 +bcolors.ENDC + '\n')
@@ -1030,7 +1168,7 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 						log_file_name = os.path.abspath(f"{log_folder}/{slugify(matched_pattern)}_{datetime.datetime.fromtimestamp(read_time).strftime('%Y-%m-%dT%H_%M_%S%z')}.log")
 						# also record a journal of all logs captured in {output_folder}/journal.log
 						journal_file_name = os.path.abspath(f"{output_folder}/journal.log")
-						with open(journal_file_name, "a") as journal_file:
+						with open(journal_file_name, "a", encoding='utf-8', errors='surrogateescape') as journal_file:
 							journal_file.write(f"{datetime.datetime.fromtimestamp(read_time).isoformat()} Captured logs for {matched_pattern} to {log_file_name}\n")
 						print(f"Capturing logs to {log_file_name}")
 						incident = {
@@ -1051,13 +1189,18 @@ def capture_messages(command_to_run, patterns, capture_time=300, output_folder=D
 						capture.write(f"{bcolors.warning}Logs since {datetime.datetime.fromtimestamp(read_time - capture_time).isoformat()}:{bcolors.ENDC}\n")
 						if dropped_until is not None and read_time - dropped_until <= capture_time:
 							capture.write(f"{bcolors.warning}Captured line count exceeded capture_line_count_max {capture_line_count_max}. Older cached in-memory lines were dropped ...{bcolors.ENDC}\n")
-						for _buffered_time, buffered_line in log_buffer:
+						# The newest buffered line is the trigger itself; remember where it starts for the LLM excerpt.
+						trigger_index = len(log_buffer) - 1
+						for index, (_buffered_time, buffered_line) in enumerate(log_buffer):
+							if index == trigger_index:
+								incident['matched_offset'] = capture.offset()
 							if matcher.find(buffered_line) is not None:
 								capture.write(f'{bcolors.critical}-> {buffered_line.strip()}{bcolors.ENDC}\n')
 							else:
 								capture.write(buffered_line)
 					else:
-						# Append the line to the current log file
+						# Append the line to the current log file; the LLM excerpt keeps these lines.
+						incident.setdefault('later_offsets', []).append(capture.offset())
 						capture.write(f'{bcolors.critical}-> {line.strip()}{bcolors.ENDC}\n')
 						captured_line_counter += 1
 					capture.commit()
@@ -1159,7 +1302,7 @@ def expand_pattern_sources(paths):
 
 
 def unit_filename(unit_name):
-	name = (unit_name or DEFAULT_UNIT_NAME).strip()
+	name = (unit_name or '').strip() or DEFAULT_UNIT_NAME
 	if not name.endswith('.service'):
 		name = name + '.service'
 	base = os.path.basename(name)
@@ -1168,6 +1311,16 @@ def unit_filename(unit_name):
 	return base
 
 
+def _check_new_unit_name(base):
+	"""ValueError unless systemd can load and enable a unit file named base: only its characters, at
+	most 255 of them, a name before the first '@' and an instance after it. For install only: uninstall
+	must still remove a unit an older firewatcher installed under another name."""
+	prefix, at, instance = base[:-len('.service')].partition('@')
+	if len(base) > 255 or not prefix or not re.match(r'[A-Za-z0-9:_.@\\-]+\Z', base) or (at and not instance):
+		raise ValueError('unit name must be one systemd can load and enable: letters, digits and : _ . @ - \\, '
+			'at most 255, and any @ with a name before it and an instance after it')
+
+
 def resolve_executable():
 	found = shutil.which('firewatcher')
 	if found:
@@ -1175,9 +1328,13 @@ def resolve_executable():
 	return os.path.abspath(sys.argv[0])
 
 
-def _quote_unit_arg(value):
-	value = str(value)
-	if value and not any(ch.isspace() or ch in '"\\' for ch in value):
+def _quote_unit_arg(value, program=False):
+	# systemd expands % specifiers in all of ExecStart and $ variables in its arguments (not the program
+	# path); a free-text argument must survive both.
+	value = str(value).replace('%', '%%')
+	if not program:
+		value = value.replace('$', '$$')
+	if value and not any(ch.isspace() or ch in '"\\\'' for ch in value):
 		return value
 	return '"' + value.replace('\\', '\\\\').replace('"', '\\"') + '"'
 
@@ -1194,7 +1351,7 @@ def render_unit_file(args, executable=None, config_exists=None, raw=None):
 		parts.extend(_explicit_cli_args(raw))
 		if not resolved.config_has_patterns and not getattr(raw, 'pattern_file', None):
 			parts.extend(resolved.pattern_file or [DEFAULT_PATTERNS_DIR])
-	exec_start = ' '.join([_quote_unit_arg(executable)] + [_quote_unit_arg(p) for p in parts])
+	exec_start = ' '.join([_quote_unit_arg(executable, program=True)] + [_quote_unit_arg(p) for p in parts])
 	after = ['network-online.target']
 	wants = ['network-online.target']
 	if not resolved.log_file:
@@ -1211,7 +1368,8 @@ def render_unit_file(args, executable=None, config_exists=None, raw=None):
 		'Wants=' + ' '.join(wants),
 	]
 	for mount in requires:
-		lines.append('RequiresMountsFor=' + mount)
+		# A path list: quoted and % escaped like an argument, but systemd substitutes no $ in it.
+		lines.append('RequiresMountsFor=' + _quote_unit_arg(mount, program=True))
 	stop_sec = max(30, int(resolved.llm_timeout) + int(resolved.ntfy_timeout) + 15)
 	lines.extend([
 		'',
@@ -1295,18 +1453,32 @@ def install_service(args, systemd_available=None, unit_dir=None, systemctl=None,
 		resolved.pattern_file = [DEFAULT_PATTERNS_DIR]
 	if unit_dir is None:
 		unit_dir = SYSTEMD_UNIT_DIR
+	config_existed = os.path.isfile(resolved.config_path)
+	executable = executable or resolve_executable()
+	# The unit name and both files are checked before anything is written: a refusal leaves nothing
+	# behind (a left-over config would make the next install render a different unit).
+	try:
+		unit_name = unit_filename(resolved.unit_name)
+		_check_new_unit_name(unit_name)
+	except ValueError as exc:
+		print(str(exc), file=sys.stderr)
+		return 2
+	try:
+		text = _strict_utf8(render_unit_file(resolved, executable=executable, config_exists=config_existed, raw=raw))
+		config_text = None if config_existed else _strict_utf8(render_config_text(resolved))
+	except ConfigError as exc:
+		print(str(exc), file=sys.stderr)
+		return 2
 	os.makedirs(resolved.output_folder, exist_ok=True)
 	_seed_example_patterns(resolved.pattern_file)
-	config_existed = os.path.isfile(resolved.config_path)
-	if not config_existed:
+	if config_text is not None:
 		try:
-			write_config_file(resolved.config_path, resolved)
+			_write_private_file(resolved.config_path, config_text)
 		except OSError as exc:
 			print('cannot write config %s: %s' % (resolved.config_path, exc), file=sys.stderr)
 			return 1
 	os.makedirs(unit_dir, exist_ok=True)
-	unit_path = os.path.join(unit_dir, unit_filename(resolved.unit_name))
-	text = render_unit_file(resolved, executable=executable, config_exists=config_existed, raw=raw)
+	unit_path = os.path.join(unit_dir, unit_name)
 	with open(unit_path, 'w', encoding='utf-8') as fh:
 		fh.write(text)
 	print(f"Wrote {unit_path}")
@@ -1316,11 +1488,11 @@ def install_service(args, systemd_available=None, unit_dir=None, systemctl=None,
 		return reload.returncode
 	if getattr(resolved, 'no_enable', False):
 		return 0
-	enable = _run_systemctl(systemctl, 'enable', '--now', unit_filename(resolved.unit_name))
+	enable = _run_systemctl(systemctl, 'enable', '--now', '--', unit_name)
 	if getattr(enable, 'returncode', 0):
 		print(f"systemctl enable --now failed (exit {enable.returncode})", file=sys.stderr)
 		return enable.returncode
-	print(f"Enabled and started {unit_filename(resolved.unit_name)}")
+	print(f"Enabled and started {unit_name}")
 	return 0
 
 
@@ -1334,7 +1506,7 @@ def uninstall_service(args, systemd_available=None, unit_dir=None, systemctl=Non
 		unit_dir = SYSTEMD_UNIT_DIR
 	name = unit_filename(args.unit_name)
 	unit_path = os.path.join(unit_dir, name)
-	disable = _run_systemctl(systemctl, 'disable', '--now', name)
+	disable = _run_systemctl(systemctl, 'disable', '--now', '--', name)
 	if getattr(disable, 'returncode', 0) not in (0, None) and os.path.exists(unit_path):
 		print(f"systemctl disable --now {name} exited {disable.returncode}", file=sys.stderr)
 	if os.path.exists(unit_path):
@@ -1384,6 +1556,7 @@ def build_parser():
 	parser.add_argument('--llm-model', default=None, help='Model name sent to the LLM API.')
 	parser.add_argument('--llm-timeout', type=int, default=None, help='LLM HTTP timeout in seconds. Default: 60.')
 	parser.add_argument('--llm-max-bytes', type=int, default=None, help='Maximum capture bytes sent to the LLM. Default: 48000.')
+	parser.add_argument('--llm-notes', default=None, metavar='TEXT', help='Short note sent to the LLM with each capture, for example what is normal on this host. Default: none.')
 	notify = parser.add_mutually_exclusive_group()
 	notify.add_argument('--notify', dest='notify', action='store_true', help='Push a notification for finished captures. With --llm, only when the model asks for one.')
 	notify.add_argument('--no-notify', dest='notify', action='store_false', help='Disable notifications, overriding the config file.')
@@ -1431,7 +1604,16 @@ def main(argv=None):
 		else:
 			parser.error('pattern file or directory required (for example /etc/firewatcher/patterns.d)')
 	if args.print_unit:
-		print(render_unit_file(resolved, raw=args), end='')
+		try:
+			text = _strict_utf8(render_unit_file(resolved, raw=args))
+		except ConfigError as exc:
+			print(str(exc), file=sys.stderr)
+			return 2
+		# As UTF-8 whatever the locale: the text is pasted into a unit file, where an escape would be read
+		# as a C escape.
+		sys.stdout.flush()
+		sys.stdout.buffer.write(text.encode('utf-8'))
+		sys.stdout.buffer.flush()
 		return 0
 	if args.install_service:
 		return install_service(resolved, raw=args)

{firewatcher-1.60 → firewatcher-1.61}/tests/conftest.py

@@ -10,11 +10,16 @@ if str(ROOT) not in sys.path:
 
 import firewatcher as fw  # noqa: E402
 
+# the real default, before the fixture below points it into each test's directory
+REAL_OUTPUT_FOLDER = fw.DEFAULT_OUTPUT_FOLDER
+
 
 @pytest.fixture(autouse=True)
 def isolated_firewatcher_config(tmp_path, monkeypatch):
-    """Keep tests off the host config and any secret environment variables."""
+    """Keep tests off the host config, output and pattern folders, and any secret environment variables."""
     monkeypatch.setattr(fw, 'DEFAULT_CONFIG_PATH', str(tmp_path / 'firewatcher.conf'))
+    monkeypatch.setattr(fw, 'DEFAULT_OUTPUT_FOLDER', str(tmp_path / 'captured_messages') + os.sep)
+    monkeypatch.setattr(fw, 'DEFAULT_PATTERNS_DIR', str(tmp_path / 'patterns.d'))
     monkeypatch.delenv(fw.LLM_API_KEY_ENV, raising=False)
     monkeypatch.delenv(fw.NTFY_TOKEN_ENV, raising=False)
 
@@ -44,7 +49,7 @@ def write_log(tmp_path, text, name='src.log'):
     if isinstance(text, bytes):
         src.write_bytes(text)
     else:
-        src.write_text(text)
+        src.write_text(text, encoding='utf-8')
     return src