JSONL-LOGGER-LIB 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jsonl_logger_lib-1.1.2/JSONL_LOGGER.py +2622 -0
- jsonl_logger_lib-1.1.2/JSONL_LOGGER_LIB.egg-info/PKG-INFO +262 -0
- jsonl_logger_lib-1.1.2/JSONL_LOGGER_LIB.egg-info/SOURCES.txt +9 -0
- jsonl_logger_lib-1.1.2/JSONL_LOGGER_LIB.egg-info/dependency_links.txt +1 -0
- jsonl_logger_lib-1.1.2/JSONL_LOGGER_LIB.egg-info/requires.txt +5 -0
- jsonl_logger_lib-1.1.2/JSONL_LOGGER_LIB.egg-info/top_level.txt +1 -0
- jsonl_logger_lib-1.1.2/LICENSE +21 -0
- jsonl_logger_lib-1.1.2/PKG-INFO +262 -0
- jsonl_logger_lib-1.1.2/README.md +235 -0
- jsonl_logger_lib-1.1.2/pyproject.toml +48 -0
- jsonl_logger_lib-1.1.2/setup.cfg +4 -0
|
@@ -0,0 +1,2622 @@
|
|
|
1
|
+
# FILE: JSONL_LOGGER.py
|
|
2
|
+
# PURPOSE: Custom logging system with JSONL file output
|
|
3
|
+
# GOAL: Provide structured logging to JSONL files
|
|
4
|
+
# RUNS TO: log_info() / log_warn() / log_error() / log_metric() / send_notification() / send_notification_async()
|
|
5
|
+
#
|
|
6
|
+
# ┌────────────────────────────────────────────────────────────────────────────────┐
|
|
7
|
+
# │ PERFORMANCE TEST RESULTS (10 RUN AVERAGE) │
|
|
8
|
+
# ├───────────────────┬──────────┬──────────┬────────────┬─────────────────────────┤
|
|
9
|
+
# │ Test │ Logs │ Time │ Throughput │ Status │
|
|
10
|
+
# ├───────────────────┼──────────┼──────────┼────────────┼─────────────────────────┤
|
|
11
|
+
# │ Single-thread │ 10,000 │ 1.04 sec │ 9,643/sec │ ✅ PASS │
|
|
12
|
+
# │ Multi-thread │ 100,000 │ 16.87 sec│ 5,902/sec │ ✅ PASS │
|
|
13
|
+
# └───────────────────┴──────────┴──────────┴────────────┴─────────────────────────┘
|
|
14
|
+
# SYSTEM: Ubuntu 24.04.4 LTS | AMD Ryzen 7 5800H (16 cores, 13Gi RAM) | Python 3.12.3
|
|
15
|
+
# TESTED: 2026-04-03 12:38 UTC
|
|
16
|
+
|
|
17
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
18
|
+
# QUICK START
|
|
19
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
20
|
+
#
|
|
21
|
+
# Set environment variables in .env:
|
|
22
|
+
# PROJECT_DIRECTORY=/path/to/your/project
|
|
23
|
+
# LOGS_LOCAL_TIMEZONE=Asia/Kolkata
|
|
24
|
+
# LOGGER_FILE_NAME=orders # Optional: default log file name (default: "LOGS")
|
|
25
|
+
#
|
|
26
|
+
# Import and use:
|
|
27
|
+
# from JSONL_LOGGER import log_info, log_warn, log_error, log_metric
|
|
28
|
+
#
|
|
29
|
+
# log_info("User logged in", user_id=123, email="user@example.com")
|
|
30
|
+
# log_warn("Rate limit approaching", remaining=10, reset_seconds=60)
|
|
31
|
+
# log_error("Payment failed", error_code=500, error="insufficient_funds")
|
|
32
|
+
# log_metric("api_latency_ms", 142.5, unit="ms", endpoint="/checkout", method="POST")
|
|
33
|
+
# send_notification("Application started", source_file="main.py")
|
|
34
|
+
# await send_notification_async("Deployment completed", source_file="deploy.py")
|
|
35
|
+
#
|
|
36
|
+
# Output: {PROJECT_DIRECTORY}/_LOGS_DIRECTORY/{YYYY_MM_DD}/LOGS/{logfile_name}.jsonl
|
|
37
|
+
#
|
|
38
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
39
|
+
|
|
40
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
41
|
+
# KEY FEATURES & DESIGN DECISIONS
|
|
42
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
43
|
+
#
|
|
44
|
+
# 1. Queue-Based Async Logging
|
|
45
|
+
# - Uses stdlib queue.Queue for thread-safe log buffering
|
|
46
|
+
# - Background thread writes to disk; API calls return immediately
|
|
47
|
+
# - WHY: Keeps logging latency off the caller's hot path — disk I/O never
|
|
48
|
+
# blocks the application thread
|
|
49
|
+
#
|
|
50
|
+
# 2. Retry Helper for I/O Failures
|
|
51
|
+
# - _with_file_retry() wraps file writes with exponential backoff (3 attempts)
|
|
52
|
+
# - Business function _flush_buffer() is clean of retry loop
|
|
53
|
+
# - WHY: Retry logic isolated in helper per RULE retry-mechanics
|
|
54
|
+
#
|
|
55
|
+
# 3. Per-Module Log Files
|
|
56
|
+
# - Each module gets its own JSONL file keyed by LOGGER_FILE_NAME
|
|
57
|
+
# - WHY: Per-module files enable independent retention, rotation, and grep
|
|
58
|
+
#
|
|
59
|
+
# 4. JSONL Format
|
|
60
|
+
# - Each log entry is valid JSON on a single line
|
|
61
|
+
# - WHY: JSON supports nested structures; each line independently parseable
|
|
62
|
+
#
|
|
63
|
+
# 5. Dual-Timestamp
|
|
64
|
+
# - Every log entry includes both UTC and local timestamps
|
|
65
|
+
# - WHY: UTC for machine sorting; local for human readability
|
|
66
|
+
#
|
|
67
|
+
# 6. Errors-Only and Warnings-Only Dual-Write
|
|
68
|
+
# - log_error() → {module}.jsonl + {module}.errors.jsonl
|
|
69
|
+
# - log_warn() → {module}.jsonl + {module}.warn.jsonl
|
|
70
|
+
# - WHY: Fast triage without grep — separate files are always current
|
|
71
|
+
#
|
|
72
|
+
# 7. Opt-In Signal Handlers
|
|
73
|
+
# - LOGGER_REGISTER_SIGNALS=true enables SIGINT/SIGTERM flush
|
|
74
|
+
# - WHY: Default-off avoids silently overriding host app handlers
|
|
75
|
+
#
|
|
76
|
+
# 8. Caller Detection via inspect
|
|
77
|
+
# - Auto-detects module name and source file from call stack
|
|
78
|
+
# - WHY: Zero-config ergonomics; explicit params bypass inspect overhead
|
|
79
|
+
#
|
|
80
|
+
# 9. Zero Data Loss Design
|
|
81
|
+
# - Buffer capped at LOGGER_MAX_BUFFER_SIZE; oldest entries dropped on overflow
|
|
82
|
+
# - WHY: Prevents unbounded memory growth on persistent disk failures
|
|
83
|
+
#
|
|
84
|
+
# 10. Module-Level Notification Log
|
|
85
|
+
# - send_notification() writes to main_logger.jsonl
|
|
86
|
+
# - WHY: Uniform JSONL format — same parser, same tooling, same retention
|
|
87
|
+
#
|
|
88
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
89
|
+
# SECTION MAP:
|
|
90
|
+
# SECTION 1 · CONSTANTS & CONFIG → LOGGING_CONFIG, module-level state
|
|
91
|
+
# SECTION 2 · CALLER DETECTION → _get_logfile_name(), _get_actual_source_file()
|
|
92
|
+
# SECTION 3 · TIMESTAMP & DEBUG → _get_timestamp(), _debug_print(), _warn_non_primitive_fields()
|
|
93
|
+
# SECTION 4 · PATH SETUP → _get_log_path(), _ensure_directory()
|
|
94
|
+
# SECTION 5 · CUSTOM FORMATTERS → ColoredFormatter, UniformLevelFormatter
|
|
95
|
+
# SECTION 6 · RETRY HELPER → _with_file_retry()
|
|
96
|
+
# SECTION 7 · QUEUE & WRITER → QueueHandler, _writer_worker(), _flush_buffer()
|
|
97
|
+
# SECTION 8 · SHUTDOWN HANDLERS → _flush_logs(), _chain_signal_handler()
|
|
98
|
+
# SECTION 9 · LOGGER INIT → _init_logger()
|
|
99
|
+
# SECTION 10 · PUBLIC API → log_info(), log_warn(), log_error(), log_metric()
|
|
100
|
+
# SECTION 11 · NOTIFICATION API → send_notification(), send_notification_async()
|
|
101
|
+
# SECTION 12 · PERFORMANCE TEST → run_performance_test()
|
|
102
|
+
|
|
103
|
+
# ── IMPORTS ──────────────────────────────────────────────────────────────────
|
|
104
|
+
# Three groups: stdlib · third-party · internal
|
|
105
|
+
|
|
106
|
+
# ── stdlib ───────────────────────────────────────────────────────────────────
|
|
107
|
+
import asyncio
|
|
108
|
+
import atexit
|
|
109
|
+
import inspect
|
|
110
|
+
import logging
|
|
111
|
+
import os
|
|
112
|
+
import queue
|
|
113
|
+
import signal
|
|
114
|
+
import sys
|
|
115
|
+
import threading
|
|
116
|
+
import time
|
|
117
|
+
from datetime import datetime, timezone
|
|
118
|
+
from pathlib import Path
|
|
119
|
+
from typing import Any, Callable
|
|
120
|
+
from unittest.mock import patch
|
|
121
|
+
|
|
122
|
+
# ── third-party ──────────────────────────────────────────────────────────────
|
|
123
|
+
from dotenv import load_dotenv
|
|
124
|
+
|
|
125
|
+
# WHY: load_dotenv() runs here — after all imports, before constants — so env
|
|
126
|
+
# vars are populated before any os.getenv() call in the constants block
|
|
127
|
+
load_dotenv()
|
|
128
|
+
|
|
129
|
+
# WHY: Shorten level names for compact, uniform JSONL output; "WARN"/"ERRO" are
|
|
130
|
+
# 4 chars like "INFO" — makes logs visually aligned and easier to scan
|
|
131
|
+
logging.addLevelName(logging.WARNING, "WARN")
|
|
132
|
+
logging.addLevelName(logging.ERROR, "ERRO")
|
|
133
|
+
|
|
134
|
+
# ── CHANGE 1 of 3 — register METR as a custom log level ──────────────────────
|
|
135
|
+
METRIC_LEVEL: int = 25
|
|
136
|
+
logging.addLevelName(METRIC_LEVEL, "METR")
|
|
137
|
+
# WHY: Custom level 25 sits between INFO (20) and WARNING (30) so metric records
|
|
138
|
+
# are distinguishable from audit INFO at the handler level.
|
|
139
|
+
|
|
140
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
141
|
+
try:
|
|
142
|
+
import pytest
|
|
143
|
+
except ImportError:
|
|
144
|
+
pytest = None
|
|
145
|
+
|
|
146
|
+
if pytest is not None:
|
|
147
|
+
|
|
148
|
+
class TestImportValidation:
|
|
149
|
+
"""WHY: _get_log_path() validates PROJECT_DIRECTORY before any log write —
|
|
150
|
+
a missing or invalid path should raise ValueError with actionable message."""
|
|
151
|
+
|
|
152
|
+
def test_get_log_path_validates_project_directory(self) -> None:
|
|
153
|
+
import subprocess
|
|
154
|
+
|
|
155
|
+
shared_lib_dir = os.path.dirname(os.path.abspath(__file__))
|
|
156
|
+
env = os.environ.copy()
|
|
157
|
+
env.pop("PROJECT_DIRECTORY", None)
|
|
158
|
+
env["PYTHONPATH"] = shared_lib_dir
|
|
159
|
+
# _get_log_path() raises ValueError when PROJECT_DIRECTORY is missing;
|
|
160
|
+
# import itself succeeds because path validation is lazy (on first emit)
|
|
161
|
+
result = subprocess.run(
|
|
162
|
+
["python3", "-c", "import JSONL_LOGGER as m; m._get_log_path('test')"],
|
|
163
|
+
capture_output=True,
|
|
164
|
+
text=True,
|
|
165
|
+
env=env,
|
|
166
|
+
cwd="/tmp",
|
|
167
|
+
)
|
|
168
|
+
assert result.returncode != 0, (
|
|
169
|
+
f"Expected non-zero returncode, got {result.returncode}"
|
|
170
|
+
)
|
|
171
|
+
assert "PROJECT_DIRECTORY" in result.stderr, (
|
|
172
|
+
f"Expected 'PROJECT_DIRECTORY' in stderr, got: {result.stderr}"
|
|
173
|
+
)
|
|
174
|
+
assert result.returncode != 0, (
|
|
175
|
+
f"Expected non-zero returncode, got {result.returncode}"
|
|
176
|
+
)
|
|
177
|
+
assert "PROJECT_DIRECTORY" in result.stderr, (
|
|
178
|
+
f"Expected 'PROJECT_DIRECTORY' in stderr, got: {result.stderr}"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# ── SECTION 1 · CONSTANTS & CONFIG ───────────────────────────────────────────
|
|
183
|
+
# Goal: Define all configuration values and module-level state in one place
|
|
184
|
+
|
|
185
|
+
LOGGER_FILE_NAME: str = os.getenv("LOGGER_FILE_NAME", "TEST_LOGGER")
|
|
186
|
+
# WHY: Sentinel default used when no caller sets LOGGER_FILE_NAME — "TEST_LOGGER"
|
|
187
|
+
# signals test/REPL usage so logs don't silently land in a production module file
|
|
188
|
+
|
|
189
|
+
DEBUG_PRINT: bool = False
|
|
190
|
+
# WHY: Disabled by default to reduce noise; enable during development only
|
|
191
|
+
|
|
192
|
+
CONSOLE_LOGGING_ENABLED: bool = (
|
|
193
|
+
os.getenv("CONSOLE_LOGGING_ENABLED", "false").lower() == "true"
|
|
194
|
+
)
|
|
195
|
+
# WHY: Off by default to keep log output clean; enable via env var when debugging
|
|
196
|
+
|
|
197
|
+
BUFFER_SIZE: int = 1000
|
|
198
|
+
# WHY: Larger buffer = fewer open()/write() calls = better throughput under burst load
|
|
199
|
+
|
|
200
|
+
FLUSH_INTERVAL: float = 0.05
|
|
201
|
+
# WHY: 50ms - faster response for flush while keeping syscall overhead low
|
|
202
|
+
|
|
203
|
+
RETRY_MAX_ATTEMPTS: int = 3
|
|
204
|
+
# WHY: Three attempts cover transient disk/NFS blips without delaying the writer loop
|
|
205
|
+
|
|
206
|
+
RETRY_BACKOFF_BASE: float = 0.1
|
|
207
|
+
# WHY: 100ms base → 100ms, 200ms, 400ms; fast enough for transient errors
|
|
208
|
+
|
|
209
|
+
PROJECT_DIRECTORY: str = os.path.expandvars(os.getenv("PROJECT_DIRECTORY", ""))
|
|
210
|
+
# WHY: Required — log root must be explicit; empty string triggers ValueError in
|
|
211
|
+
# _get_log_path() with actionable message rather than silent write to wrong path
|
|
212
|
+
|
|
213
|
+
_DEFAULT_LOGFILE: str = "LOGS"
|
|
214
|
+
# WHY: Default logfile name — can override via LOGGER_FILE_NAME env var or explicit param
|
|
215
|
+
|
|
216
|
+
LOGS_LOCAL_TIMEZONE: str = os.getenv("LOGS_LOCAL_TIMEZONE", "")
|
|
217
|
+
# WHY: Local timezone for logging. Set via LOGS_LOCAL_TIMEZONE env var.
|
|
218
|
+
|
|
219
|
+
_LOGS_DIRECTORY: str = "_LOGS_DIRECTORY"
|
|
220
|
+
# WHY: Hardcoded — this is the one canonical log subdirectory for this project.
|
|
221
|
+
|
|
222
|
+
LOGGER_REGISTER_SIGNALS: bool = (
|
|
223
|
+
os.getenv("LOGGER_REGISTER_SIGNALS", "false").lower() == "true"
|
|
224
|
+
)
|
|
225
|
+
# WHY: Opt-in avoids silently overriding host app handlers (FastAPI, Gunicorn, etc.)
|
|
226
|
+
|
|
227
|
+
LOGGER_DAEMON_THREAD: bool = os.getenv("LOGGER_DAEMON_THREAD", "true").lower() == "true"
|
|
228
|
+
# WHY: Daemon=True is the default for fast process exit
|
|
229
|
+
|
|
230
|
+
LOGGER_MAX_BUFFER_SIZE: int = int(os.getenv("LOGGER_MAX_BUFFER_SIZE", "200000"))
|
|
231
|
+
# WHY: Hard cap prevents unbounded memory growth when disk fails persistently
|
|
232
|
+
|
|
233
|
+
# Logger implementation: stdlib logging (use extra={} for structured fields)
|
|
234
|
+
# Logger name: __name__ — from DOMAIN_RULES.md §2 Logging Strategy
|
|
235
|
+
logger: logging.Logger | None = None
|
|
236
|
+
# WHY: Initialized to None at module level; _init_logger() assigns at import time
|
|
237
|
+
|
|
238
|
+
_RESERVED_LOG_KEYS: frozenset[str] = frozenset(
|
|
239
|
+
{
|
|
240
|
+
"name",
|
|
241
|
+
"msg",
|
|
242
|
+
"args",
|
|
243
|
+
"levelname",
|
|
244
|
+
"levelno",
|
|
245
|
+
"pathname",
|
|
246
|
+
"filename",
|
|
247
|
+
"module",
|
|
248
|
+
"exc_info",
|
|
249
|
+
"exc_text",
|
|
250
|
+
"stack_info",
|
|
251
|
+
"lineno",
|
|
252
|
+
"funcName",
|
|
253
|
+
"created",
|
|
254
|
+
"msecs",
|
|
255
|
+
"relativeCreated",
|
|
256
|
+
"thread",
|
|
257
|
+
"threadName",
|
|
258
|
+
"processName",
|
|
259
|
+
"process",
|
|
260
|
+
"taskName",
|
|
261
|
+
"message",
|
|
262
|
+
"asctime",
|
|
263
|
+
}
|
|
264
|
+
)
|
|
265
|
+
# WHY: Single source of truth for stdlib LogRecord attrs
|
|
266
|
+
|
|
267
|
+
_PRIMITIVE_TYPES = (str, int, float, bool, type(None))
|
|
268
|
+
# WHY: Defines the set of types that serialize cleanly to JSONL
|
|
269
|
+
|
|
270
|
+
_log_queue: queue.Queue | None = None
|
|
271
|
+
_writer_thread: threading.Thread | None = None
|
|
272
|
+
_buffers: dict[str, list[str]] = {}
|
|
273
|
+
_buffer_lock: threading.Lock = threading.Lock()
|
|
274
|
+
_shutdown: bool = False
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# ── SECTION 2 · CALLER DETECTION ─────────────────────────────────────────────
|
|
278
|
+
# Goal: Resolve the source module name for each log entry
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _get_logfile_name() -> str:
|
|
282
|
+
"""[TIER 2] Resolve the logging module name from the call stack.
|
|
283
|
+
|
|
284
|
+
WHY: Zero-config ergonomics — callers don't need to pass module_name manually.
|
|
285
|
+
Checks LOGGER_FILE_NAME in caller's globals first (grouping pattern), then
|
|
286
|
+
falls back to the caller's filename without .py extension.
|
|
287
|
+
"""
|
|
288
|
+
frame = None
|
|
289
|
+
try:
|
|
290
|
+
frame = inspect.currentframe()
|
|
291
|
+
if frame is None:
|
|
292
|
+
return "unknown_file"
|
|
293
|
+
|
|
294
|
+
# WHY: Skip two frames — log_info/warn/error/metric, then this function
|
|
295
|
+
frame = frame.f_back # → log_info/warn/error/metric
|
|
296
|
+
if frame is None:
|
|
297
|
+
return "unknown_file"
|
|
298
|
+
frame = frame.f_back # → actual caller
|
|
299
|
+
if frame is None:
|
|
300
|
+
return "unknown_file"
|
|
301
|
+
|
|
302
|
+
if "LOGGER_FILE_NAME" in frame.f_globals:
|
|
303
|
+
return frame.f_globals["LOGGER_FILE_NAME"]
|
|
304
|
+
|
|
305
|
+
file_path = frame.f_code.co_filename
|
|
306
|
+
if file_path and file_path != __file__:
|
|
307
|
+
return Path(file_path).name.replace(".py", "")
|
|
308
|
+
|
|
309
|
+
return "unknown_file"
|
|
310
|
+
except Exception as e:
|
|
311
|
+
# WHY: Exception (not bare except) — frame inspection can fail under
|
|
312
|
+
# restricted interpreters; returning a sentinel is safer than crashing.
|
|
313
|
+
return "unknown_file"
|
|
314
|
+
finally:
|
|
315
|
+
if frame is not None:
|
|
316
|
+
del frame
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
320
|
+
if pytest is not None:
|
|
321
|
+
|
|
322
|
+
class TestGetLogfileName:
|
|
323
|
+
"""WHY: Caller detection drives every log entry's module_name field — a wrong
|
|
324
|
+
detection silently misroutes logs to the wrong JSONL file."""
|
|
325
|
+
|
|
326
|
+
def test_get_logfile_name(self) -> None:
|
|
327
|
+
assert pytest is not None # Type guard for pyright
|
|
328
|
+
import types
|
|
329
|
+
|
|
330
|
+
# ── LOGGER_FILE_NAME in globals takes priority over filename ──────────
|
|
331
|
+
fake_frame = types.SimpleNamespace(
|
|
332
|
+
f_globals={"LOGGER_FILE_NAME": "MY_APP"},
|
|
333
|
+
f_back=None,
|
|
334
|
+
f_code=types.SimpleNamespace(co_filename="irrelevant.py"),
|
|
335
|
+
)
|
|
336
|
+
with patch("inspect.currentframe") as mock_cf:
|
|
337
|
+
inner = types.SimpleNamespace(
|
|
338
|
+
f_back=types.SimpleNamespace(f_back=fake_frame)
|
|
339
|
+
)
|
|
340
|
+
mock_cf.return_value = inner
|
|
341
|
+
result = _get_logfile_name()
|
|
342
|
+
assert result == "MY_APP", f"Expected 'MY_APP', got {result!r}"
|
|
343
|
+
|
|
344
|
+
# ── no LOGGER_FILE_NAME falls back to caller filename ─────────────────
|
|
345
|
+
result = _get_logfile_name()
|
|
346
|
+
assert result != "unknown_file", (
|
|
347
|
+
"Should detect caller filename, not 'unknown_file'"
|
|
348
|
+
)
|
|
349
|
+
assert len(result) > 0, "Filename must be non-empty"
|
|
350
|
+
|
|
351
|
+
# ── exception during frame walk returns "unknown_file" ────────────────
|
|
352
|
+
with patch("inspect.currentframe", side_effect=RuntimeError("no frames")):
|
|
353
|
+
result = _get_logfile_name()
|
|
354
|
+
assert result == "unknown_file", (
|
|
355
|
+
f"Expected 'unknown_file' on exception, got {result!r}"
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
_SKIP_FRAME_PATHS: tuple[str, ...] = (
|
|
360
|
+
"/asyncio/",
|
|
361
|
+
"asyncio/",
|
|
362
|
+
"asyncio.py",
|
|
363
|
+
"runners.py",
|
|
364
|
+
"base_events.py",
|
|
365
|
+
"events.py",
|
|
366
|
+
"/concurrent/",
|
|
367
|
+
"/threading.py",
|
|
368
|
+
"/site-packages/",
|
|
369
|
+
)
|
|
370
|
+
_SKIP_FRAME_NAMES: frozenset[str] = frozenset(
|
|
371
|
+
{"runners", "base_events", "events", "asyncio", "threading"}
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _get_caller_module() -> str:
|
|
376
|
+
"""[TIER 2] Get the Python module name (e.g., 'orders', 'payments') from call stack.
|
|
377
|
+
|
|
378
|
+
Uses __name__ from caller's module globals.
|
|
379
|
+
"""
|
|
380
|
+
frame = None
|
|
381
|
+
try:
|
|
382
|
+
frame = inspect.currentframe()
|
|
383
|
+
if frame is None:
|
|
384
|
+
return "unknown_module"
|
|
385
|
+
|
|
386
|
+
# Skip frames: log_info → this function → actual caller
|
|
387
|
+
for _ in range(2):
|
|
388
|
+
frame = frame.f_back
|
|
389
|
+
if frame is None:
|
|
390
|
+
return "unknown_module"
|
|
391
|
+
|
|
392
|
+
module_name = frame.f_globals.get("__name__", "unknown_module")
|
|
393
|
+
return module_name.split(".")[-1]
|
|
394
|
+
|
|
395
|
+
except Exception as e:
|
|
396
|
+
# WHY: Exception (not bare except) — frame inspection can fail under
|
|
397
|
+
# restricted interpreters; returning a sentinel is safer than crashing.
|
|
398
|
+
return "unknown_module"
|
|
399
|
+
finally:
|
|
400
|
+
if frame is not None:
|
|
401
|
+
del frame
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _get_actual_source_file() -> str:
|
|
405
|
+
"""[TIER 2] Get the actual Python filename, bypassing LOGGER_FILE_NAME grouping.
|
|
406
|
+
|
|
407
|
+
WHY: Skips asyncio/stdlib internal frames so async callers don't produce
|
|
408
|
+
"events" as the detected source.
|
|
409
|
+
"""
|
|
410
|
+
frame = None
|
|
411
|
+
try:
|
|
412
|
+
frame = inspect.currentframe()
|
|
413
|
+
if frame is None:
|
|
414
|
+
return "unknown_source"
|
|
415
|
+
|
|
416
|
+
while frame is not None:
|
|
417
|
+
co_filename = getattr(getattr(frame, "f_code", None), "co_filename", None)
|
|
418
|
+
|
|
419
|
+
if co_filename is None:
|
|
420
|
+
frame = frame.f_back
|
|
421
|
+
continue
|
|
422
|
+
|
|
423
|
+
if co_filename == __file__:
|
|
424
|
+
frame = frame.f_back
|
|
425
|
+
continue
|
|
426
|
+
|
|
427
|
+
if any(skip in co_filename for skip in _SKIP_FRAME_PATHS):
|
|
428
|
+
frame = frame.f_back
|
|
429
|
+
continue
|
|
430
|
+
|
|
431
|
+
filename = Path(co_filename).stem
|
|
432
|
+
if not filename or filename in _SKIP_FRAME_NAMES:
|
|
433
|
+
frame = frame.f_back
|
|
434
|
+
continue
|
|
435
|
+
|
|
436
|
+
return filename
|
|
437
|
+
|
|
438
|
+
return Path(__file__).stem # __main__ fallback
|
|
439
|
+
|
|
440
|
+
except Exception as e:
|
|
441
|
+
# WHY: Exception (not bare except) — frame inspection can fail under
|
|
442
|
+
# restricted interpreters; returning a sentinel is safer than crashing.
|
|
443
|
+
return "unknown_source"
|
|
444
|
+
finally:
|
|
445
|
+
if frame is not None:
|
|
446
|
+
del frame
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
450
|
+
if pytest is not None:
|
|
451
|
+
|
|
452
|
+
class TestGetActualSourceFile:
|
|
453
|
+
"""WHY: source_file must reflect the actual calling Python file, not the
|
|
454
|
+
LOGGER_FILE_NAME group alias — without this, dual-source tracking fails."""
|
|
455
|
+
|
|
456
|
+
def test_get_actual_source_file(self) -> None:
|
|
457
|
+
assert pytest is not None # Type guard for pyright
|
|
458
|
+
import types
|
|
459
|
+
|
|
460
|
+
# ── bypasses LOGGER_FILE_NAME and returns actual filename ─────────────
|
|
461
|
+
def _make_frame(co_filename: str, f_back):
|
|
462
|
+
return types.SimpleNamespace(
|
|
463
|
+
f_code=types.SimpleNamespace(co_filename=co_filename),
|
|
464
|
+
f_globals={},
|
|
465
|
+
f_back=f_back,
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
fake_caller = _make_frame("actual_module.py", None)
|
|
469
|
+
frame_b = _make_frame(__file__, fake_caller)
|
|
470
|
+
frame_a = _make_frame(__file__, frame_b)
|
|
471
|
+
|
|
472
|
+
with patch("inspect.currentframe", return_value=frame_a):
|
|
473
|
+
result = _get_actual_source_file()
|
|
474
|
+
assert result == "actual_module", (
|
|
475
|
+
f"Expected 'actual_module', got {result!r}"
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
# ── exception during frame walk returns "unknown_source" ──────────────
|
|
479
|
+
with patch("inspect.currentframe", side_effect=RuntimeError("no frames")):
|
|
480
|
+
result = _get_actual_source_file()
|
|
481
|
+
assert result == "unknown_source", (
|
|
482
|
+
f"Expected 'unknown_source' on exception, got {result!r}"
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
# ── SECTION 3 · TIMESTAMP & FIELD VALIDATION ─────────────────────────────────
|
|
487
|
+
# Goal: Provide timestamp formatting and warn on non-serializable extra fields
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _get_timestamp() -> dict[str, str]:
|
|
491
|
+
"""[TIER 3] Return both UTC and local timestamps in ISO-8601 format.
|
|
492
|
+
|
|
493
|
+
WHY: ISO-8601 with Z suffix ensures consistent, sortable timestamps.
|
|
494
|
+
Local time with offset helps readability while UTC ensures cross-machine consistency.
|
|
495
|
+
"""
|
|
496
|
+
from zoneinfo import ZoneInfo
|
|
497
|
+
|
|
498
|
+
utc_time = datetime.now(timezone.utc)
|
|
499
|
+
utc_str = utc_time.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
500
|
+
|
|
501
|
+
try:
|
|
502
|
+
if not LOGS_LOCAL_TIMEZONE:
|
|
503
|
+
raise ValueError(
|
|
504
|
+
"LOGS_LOCAL_TIMEZONE must be set in .env\n"
|
|
505
|
+
"⚠️ Common mistake: LOGS_LOCAL_TIMEZONE =Asia/Kolkata has a space before =\n"
|
|
506
|
+
" Correct format: LOGS_LOCAL_TIMEZONE=Asia/Kolkata"
|
|
507
|
+
)
|
|
508
|
+
tz = ZoneInfo(LOGS_LOCAL_TIMEZONE)
|
|
509
|
+
local_time = utc_time.astimezone(tz)
|
|
510
|
+
local_str = local_time.strftime("%Y-%m-%dT%H:%M:%S.%f")[
|
|
511
|
+
:-3
|
|
512
|
+
] + local_time.strftime("%z")
|
|
513
|
+
except ValueError:
|
|
514
|
+
raise
|
|
515
|
+
except Exception as e:
|
|
516
|
+
# WHY: Exception (not bare except) — timezone lookup can fail for invalid
|
|
517
|
+
# zone names; falling back to UTC keeps logging functional.
|
|
518
|
+
local_str = utc_str
|
|
519
|
+
|
|
520
|
+
return {"utc": utc_str, "local": local_str}
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
524
|
+
if pytest is not None:
|
|
525
|
+
|
|
526
|
+
class TestGetTimestamp:
|
|
527
|
+
"""WHY: Timestamp format is the contract shared with every log parser and Rust
|
|
528
|
+
sibling — a wrong format silently breaks all downstream tooling."""
|
|
529
|
+
|
|
530
|
+
def test_get_timestamp(self) -> None:
|
|
531
|
+
assert pytest is not None # Type guard for pyright
|
|
532
|
+
import re
|
|
533
|
+
from datetime import timedelta
|
|
534
|
+
import JSONL_LOGGER as mod
|
|
535
|
+
|
|
536
|
+
original_tz = mod.LOGS_LOCAL_TIMEZONE
|
|
537
|
+
try:
|
|
538
|
+
mod.LOGS_LOCAL_TIMEZONE = "Asia/Kolkata"
|
|
539
|
+
|
|
540
|
+
# ── returns dict with utc and local keys ───────────────────────────────
|
|
541
|
+
result = mod._get_timestamp()
|
|
542
|
+
assert isinstance(result, dict), (
|
|
543
|
+
f"Expected dict, got {type(result).__name__}"
|
|
544
|
+
)
|
|
545
|
+
assert "utc" in result, "Missing 'utc' key"
|
|
546
|
+
assert "local" in result, "Missing 'local' key"
|
|
547
|
+
|
|
548
|
+
# ── UTC ends with Z (UTC marker) ─────────────────────────────────────
|
|
549
|
+
assert result["utc"].endswith("Z"), (
|
|
550
|
+
f"UTC timestamp must end with 'Z', got {result['utc']!r}"
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
# ── UTC matches ISO-8601 with millisecond precision ───────────────────
|
|
554
|
+
pattern = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$"
|
|
555
|
+
assert re.match(pattern, result["utc"]), (
|
|
556
|
+
f"Unexpected UTC format: {result['utc']!r}"
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
# ── local has offset (e.g., +0530 or -0800) ───────────────────────────
|
|
560
|
+
assert re.match(r".+\d{4}$", result["local"]), (
|
|
561
|
+
f"Unexpected local format: {result['local']!r}"
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
# ── reflects real clock, not a stale cached value ─────────────────────
|
|
565
|
+
before = datetime.now(timezone.utc)
|
|
566
|
+
result2 = mod._get_timestamp()
|
|
567
|
+
after = datetime.now(timezone.utc)
|
|
568
|
+
parsed = datetime.fromisoformat(result2["utc"].replace("Z", "+00:00"))
|
|
569
|
+
assert (before - timedelta(milliseconds=1)) <= parsed <= after, (
|
|
570
|
+
f"Timestamp {parsed} outside expected range [{before}..{after}]"
|
|
571
|
+
)
|
|
572
|
+
finally:
|
|
573
|
+
mod.LOGS_LOCAL_TIMEZONE = original_tz
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _debug_print(message: str) -> None:
|
|
577
|
+
"""[TIER 3] Print internal debug messages to stderr when DEBUG_PRINT is enabled.
|
|
578
|
+
|
|
579
|
+
WHY: Tier 3 only — development-time visibility without polluting production logs.
|
|
580
|
+
"""
|
|
581
|
+
if DEBUG_PRINT:
|
|
582
|
+
print(f"[DEBUG] {message}", file=sys.stderr)
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
586
|
+
if pytest is not None:
|
|
587
|
+
|
|
588
|
+
class TestDebugPrint:
|
|
589
|
+
"""WHY: _debug_print is the only development visibility mechanism — if it leaks
|
|
590
|
+
to production stderr or is silenced when needed, debugging becomes impossible."""
|
|
591
|
+
|
|
592
|
+
def test_debug_print(self) -> None:
|
|
593
|
+
assert pytest is not None # Type guard for pyright
|
|
594
|
+
import io
|
|
595
|
+
import JSONL_LOGGER as mod
|
|
596
|
+
|
|
597
|
+
# ── writes to stderr when DEBUG_PRINT=True ────────────────────────────
|
|
598
|
+
original = mod.DEBUG_PRINT
|
|
599
|
+
mod.DEBUG_PRINT = True
|
|
600
|
+
try:
|
|
601
|
+
buf = io.StringIO()
|
|
602
|
+
with patch("sys.stderr", buf):
|
|
603
|
+
_debug_print("hello debug")
|
|
604
|
+
assert "[DEBUG] hello debug" in buf.getvalue(), (
|
|
605
|
+
f"Expected debug output, got: {buf.getvalue()!r}"
|
|
606
|
+
)
|
|
607
|
+
finally:
|
|
608
|
+
mod.DEBUG_PRINT = original
|
|
609
|
+
|
|
610
|
+
# ── silent when DEBUG_PRINT=False (production default) ───────────────
|
|
611
|
+
mod.DEBUG_PRINT = False
|
|
612
|
+
try:
|
|
613
|
+
buf = io.StringIO()
|
|
614
|
+
with patch("sys.stderr", buf):
|
|
615
|
+
_debug_print("should not appear")
|
|
616
|
+
assert buf.getvalue() == "", (
|
|
617
|
+
f"Expected empty output, got: {buf.getvalue()!r}"
|
|
618
|
+
)
|
|
619
|
+
finally:
|
|
620
|
+
mod.DEBUG_PRINT = original
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _warn_non_primitive_fields(extra_fields: dict[str, Any], caller: str) -> None:
|
|
624
|
+
"""[TIER 3] Emit a stderr warning for any extra_field value that is not a primitive type.
|
|
625
|
+
|
|
626
|
+
WHY: json.dumps(default=str) silently converts datetimes, lists, and custom
|
|
627
|
+
classes to their string repr. This hides bugs and creates unqueryable JSONL fields.
|
|
628
|
+
"""
|
|
629
|
+
for k, v in extra_fields.items():
|
|
630
|
+
if not isinstance(v, _PRIMITIVE_TYPES):
|
|
631
|
+
print(
|
|
632
|
+
f"[LOGGER WARN] Field '{k}' in {caller} has type {type(v).__name__!r} "
|
|
633
|
+
f"— will be stringified in JSONL. Consider explicit serialization.",
|
|
634
|
+
file=sys.stderr,
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
639
|
+
if pytest is not None:
|
|
640
|
+
|
|
641
|
+
class TestWarnNonPrimitiveFields:
|
|
642
|
+
"""WHY: Silent stringification of complex types produces unqueryable JSONL
|
|
643
|
+
fields — the warning is the only signal callers get."""
|
|
644
|
+
|
|
645
|
+
def test_warn_non_primitive_fields(self) -> None:
|
|
646
|
+
assert pytest is not None # Type guard for pyright
|
|
647
|
+
import io
|
|
648
|
+
|
|
649
|
+
# ── list value emits warning with field name and type ─────────────────
|
|
650
|
+
buf = io.StringIO()
|
|
651
|
+
with patch("sys.stderr", buf):
|
|
652
|
+
_warn_non_primitive_fields({"tags": [1, 2, 3]}, "my_module")
|
|
653
|
+
assert "tags" in buf.getvalue(), (
|
|
654
|
+
f"Expected 'tags' in warning, got: {buf.getvalue()!r}"
|
|
655
|
+
)
|
|
656
|
+
assert "list" in buf.getvalue(), (
|
|
657
|
+
f"Expected 'list' in warning, got: {buf.getvalue()!r}"
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
# ── dict value emits warning ──────────────────────────────────────────
|
|
661
|
+
buf = io.StringIO()
|
|
662
|
+
with patch("sys.stderr", buf):
|
|
663
|
+
_warn_non_primitive_fields({"meta": {"a": 1}}, "my_module")
|
|
664
|
+
assert "meta" in buf.getvalue(), (
|
|
665
|
+
f"Expected 'meta' in warning, got: {buf.getvalue()!r}"
|
|
666
|
+
)
|
|
667
|
+
assert "dict" in buf.getvalue(), (
|
|
668
|
+
f"Expected 'dict' in warning, got: {buf.getvalue()!r}"
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
# ── datetime value emits warning ──────────────────────────────────────
|
|
672
|
+
buf = io.StringIO()
|
|
673
|
+
with patch("sys.stderr", buf):
|
|
674
|
+
_warn_non_primitive_fields({"ts": datetime.now()}, "my_module")
|
|
675
|
+
assert "ts" in buf.getvalue(), (
|
|
676
|
+
f"Expected 'ts' in warning, got: {buf.getvalue()!r}"
|
|
677
|
+
)
|
|
678
|
+
assert "datetime" in buf.getvalue(), (
|
|
679
|
+
f"Expected 'datetime' in warning, got: {buf.getvalue()!r}"
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
# ── primitive types (str/int/float/bool/None) emit no warning ─────────
|
|
683
|
+
buf = io.StringIO()
|
|
684
|
+
with patch("sys.stderr", buf):
|
|
685
|
+
_warn_non_primitive_fields(
|
|
686
|
+
{"s": "ok", "i": 1, "f": 1.5, "b": True, "n": None}, "my_module"
|
|
687
|
+
)
|
|
688
|
+
assert buf.getvalue() == "", (
|
|
689
|
+
f"Expected no warnings for primitives, got: {buf.getvalue()!r}"
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
# ── warning includes caller module name for fast triage ───────────────
|
|
693
|
+
buf = io.StringIO()
|
|
694
|
+
with patch("sys.stderr", buf):
|
|
695
|
+
_warn_non_primitive_fields({"x": [1]}, "payments_module")
|
|
696
|
+
assert "payments_module" in buf.getvalue(), (
|
|
697
|
+
f"Expected caller name in warning, got: {buf.getvalue()!r}"
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
# ── SECTION 4 · PATH SETUP ───────────────────────────────────────────────────
|
|
702
|
+
# Goal: Resolve and create the directory path for each module's log file
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _ensure_directory(path: Path, mode: int = 0o755) -> None:
|
|
706
|
+
"""[TIER 2] Create directory and all parents if they do not exist.
|
|
707
|
+
|
|
708
|
+
WHY: 0o755 gives rwxr-xr-x — owner can write, others can read/execute.
|
|
709
|
+
"""
|
|
710
|
+
path.mkdir(parents=True, exist_ok=True, mode=mode)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _get_log_path(logfile_name: str | None = None, suffix: str = "") -> Path:
|
|
714
|
+
"""[TIER 2] Resolve the JSONL file path for a given logfile name and optional suffix.
|
|
715
|
+
|
|
716
|
+
WHY: Centralised path logic means all three file types (audit, errors, metrics)
|
|
717
|
+
follow the same directory and naming convention automatically.
|
|
718
|
+
"""
|
|
719
|
+
logfile_name = logfile_name or os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
720
|
+
logfile_name = logfile_name.replace(".py", "")
|
|
721
|
+
|
|
722
|
+
if not PROJECT_DIRECTORY:
|
|
723
|
+
raise ValueError(
|
|
724
|
+
"PROJECT_DIRECTORY must be set in .env\n"
|
|
725
|
+
"⚠️ Common mistake: PROJECT_DIRECTORY =/path has a space before =\n"
|
|
726
|
+
" Correct format: PROJECT_DIRECTORY=/path/to/logs"
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
base_dir = Path(PROJECT_DIRECTORY)
|
|
730
|
+
if not base_dir.exists():
|
|
731
|
+
raise ValueError(f"PROJECT_DIRECTORY '{base_dir}' does not exist")
|
|
732
|
+
if not os.access(base_dir, os.W_OK):
|
|
733
|
+
raise ValueError(f"PROJECT_DIRECTORY '{base_dir}' is not writable")
|
|
734
|
+
|
|
735
|
+
today = datetime.now(timezone.utc).strftime("%Y_%m_%d")
|
|
736
|
+
log_dir = base_dir / _LOGS_DIRECTORY / today / "LOGS"
|
|
737
|
+
_ensure_directory(log_dir)
|
|
738
|
+
return log_dir / f"{logfile_name}{suffix}.jsonl"
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
742
|
+
if pytest is not None:
|
|
743
|
+
|
|
744
|
+
class TestGetLogPath:
|
|
745
|
+
"""WHY: Path construction is the spine of the logging system — wrong paths mean
|
|
746
|
+
silent data loss."""
|
|
747
|
+
|
|
748
|
+
def test_get_log_path(self) -> None:
|
|
749
|
+
assert pytest is not None # Type guard for pyright
|
|
750
|
+
import tempfile
|
|
751
|
+
import JSONL_LOGGER as mod
|
|
752
|
+
from datetime import datetime, timezone
|
|
753
|
+
|
|
754
|
+
# ── missing PROJECT_DIRECTORY raises ValueError ───────────────────────
|
|
755
|
+
original_pd = mod.PROJECT_DIRECTORY
|
|
756
|
+
mod.PROJECT_DIRECTORY = ""
|
|
757
|
+
try:
|
|
758
|
+
try:
|
|
759
|
+
_get_log_path("mymodule")
|
|
760
|
+
assert False, "Expected ValueError for missing PROJECT_DIRECTORY"
|
|
761
|
+
except ValueError as e:
|
|
762
|
+
assert "PROJECT_DIRECTORY" in str(e), (
|
|
763
|
+
f"Expected 'PROJECT_DIRECTORY' in error, got: {e!r}"
|
|
764
|
+
)
|
|
765
|
+
finally:
|
|
766
|
+
mod.PROJECT_DIRECTORY = original_pd
|
|
767
|
+
|
|
768
|
+
# ── non-existent PROJECT_DIRECTORY raises ValueError ─────────────────
|
|
769
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", "/nonexistent/path/abc123"):
|
|
770
|
+
try:
|
|
771
|
+
_get_log_path("mymodule")
|
|
772
|
+
assert False, "Expected ValueError for non-existent path"
|
|
773
|
+
except ValueError as e:
|
|
774
|
+
assert "does not exist" in str(e), (
|
|
775
|
+
f"Expected 'does not exist' in error, got: {e!r}"
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
# ── _LOGS_DIRECTORY is always _LOGS_DIRECTORY — hardcoded ─────────────
|
|
779
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
780
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
781
|
+
result = _get_log_path("mymodule")
|
|
782
|
+
assert "_LOGS_DIRECTORY" in str(result), (
|
|
783
|
+
f"Expected '_LOGS_DIRECTORY' in path, got: {result!r}"
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
# ── plain suffix="" produces {module}.jsonl ───────────────────────────
|
|
787
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
788
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
789
|
+
result = _get_log_path("mymodule", suffix="")
|
|
790
|
+
assert result.name == "mymodule.jsonl", (
|
|
791
|
+
f"Expected 'mymodule.jsonl', got {result.name!r}"
|
|
792
|
+
)
|
|
793
|
+
|
|
794
|
+
# ── .errors suffix produces {module}.errors.jsonl ────────────────────
|
|
795
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
796
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
797
|
+
result = _get_log_path("mymodule", suffix=".errors")
|
|
798
|
+
assert result.name == "mymodule.errors.jsonl", (
|
|
799
|
+
f"Expected 'mymodule.errors.jsonl', got {result.name!r}"
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
# ── .metrics suffix produces {module}.metrics.jsonl ──────────────────
|
|
803
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
804
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
805
|
+
result = _get_log_path("mymodule", suffix=".metrics")
|
|
806
|
+
assert result.name == "mymodule.metrics.jsonl", (
|
|
807
|
+
f"Expected 'mymodule.metrics.jsonl', got {result.name!r}"
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
# ── dated subdirectory present in path ───────────────────────────────
|
|
811
|
+
today = datetime.now(timezone.utc).strftime("%Y_%m_%d")
|
|
812
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
813
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
814
|
+
result = _get_log_path("mymodule")
|
|
815
|
+
assert today in str(result), (
|
|
816
|
+
f"Expected date {today} in path, got: {result!r}"
|
|
817
|
+
)
|
|
818
|
+
|
|
819
|
+
# ── .py extension stripped from module_name ────────────────────────────
|
|
820
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
821
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
822
|
+
result = _get_log_path("mymodule.py", suffix="")
|
|
823
|
+
assert result.name == "mymodule.jsonl", (
|
|
824
|
+
f"Expected 'mymodule.jsonl' (stripped .py), got {result.name!r}"
|
|
825
|
+
)
|
|
826
|
+
|
|
827
|
+
# ── dated directory is created if not yet present ────────────────────
|
|
828
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
829
|
+
orig_pd = mod.PROJECT_DIRECTORY
|
|
830
|
+
mod.PROJECT_DIRECTORY = tmp
|
|
831
|
+
try:
|
|
832
|
+
result = _get_log_path("mymodule")
|
|
833
|
+
assert result.parent.exists(), (
|
|
834
|
+
f"Log directory should exist: {result.parent}"
|
|
835
|
+
)
|
|
836
|
+
finally:
|
|
837
|
+
mod.PROJECT_DIRECTORY = orig_pd
|
|
838
|
+
|
|
839
|
+
# ── LOGS subdirectory present after date in path ──────────────────────
|
|
840
|
+
today = datetime.now(timezone.utc).strftime("%Y_%m_%d")
|
|
841
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
842
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
843
|
+
result = _get_log_path("mymodule")
|
|
844
|
+
path_parts = result.parts
|
|
845
|
+
date_idx = path_parts.index(today)
|
|
846
|
+
assert path_parts[date_idx + 1] == "LOGS", (
|
|
847
|
+
f"Expected 'LOGS' after date, got {path_parts[date_idx + 1]!r}"
|
|
848
|
+
)
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
# ── SECTION 5 · CUSTOM FORMATTERS ────────────────────────────────────────────
|
|
852
|
+
# Goal: Format log records for colored console output and JSONL file output
|
|
853
|
+
# RULE OVERRIDE: §PYTHON no-oop — logging.Formatter requires class inheritance;
|
|
854
|
+
# there is no functional formatter API in stdlib logging.
|
|
855
|
+
# Restore when: stdlib logging adds a hook-based formatter protocol.
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
class ColoredFormatter(logging.Formatter):
|
|
859
|
+
"""Formatter for colored console output with emoji level indicators.
|
|
860
|
+
|
|
861
|
+
WHY: Visual differentiation at a glance during development.
|
|
862
|
+
"""
|
|
863
|
+
|
|
864
|
+
GREEN = "\033[92m"
|
|
865
|
+
YELLOW = "\033[93m"
|
|
866
|
+
RED = "\033[91m"
|
|
867
|
+
RESET = "\033[0m"
|
|
868
|
+
|
|
869
|
+
EMOJI_MAP = {
|
|
870
|
+
"INFO": "🟢",
|
|
871
|
+
"WARN": "🟡",
|
|
872
|
+
"ERRO": "🔴",
|
|
873
|
+
"METR": "📊",
|
|
874
|
+
"CRITICAL": "🔴",
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
878
|
+
emoji = self.EMOJI_MAP.get(record.levelname, "")
|
|
879
|
+
color = self.RESET
|
|
880
|
+
if record.levelno >= logging.ERROR:
|
|
881
|
+
color = self.RED
|
|
882
|
+
elif record.levelno == logging.WARNING:
|
|
883
|
+
color = self.YELLOW
|
|
884
|
+
elif record.levelno == logging.INFO:
|
|
885
|
+
color = self.GREEN
|
|
886
|
+
|
|
887
|
+
timestamp = _get_timestamp()
|
|
888
|
+
extra = {
|
|
889
|
+
k: v
|
|
890
|
+
for k, v in record.__dict__.items()
|
|
891
|
+
if k not in _RESERVED_LOG_KEYS and not k.startswith("_")
|
|
892
|
+
}
|
|
893
|
+
module_name = extra.get("module_name", Path(record.pathname).name)
|
|
894
|
+
source_file = extra.get("source_file", "unknown")
|
|
895
|
+
return (
|
|
896
|
+
f"{timestamp['utc']} {color}{record.levelname}{self.RESET} "
|
|
897
|
+
f"{emoji} [{module_name}:{source_file}] {record.getMessage()}"
|
|
898
|
+
)
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
902
|
+
if pytest is not None:
|
|
903
|
+
|
|
904
|
+
class TestColoredFormatter:
|
|
905
|
+
"""WHY: Console formatting is the developer's primary real-time log view —
|
|
906
|
+
wrong colours or missing emojis make level discrimination impossible."""
|
|
907
|
+
|
|
908
|
+
def _make_record(
|
|
909
|
+
self, level: int, message: str, **extra: Any
|
|
910
|
+
) -> logging.LogRecord:
|
|
911
|
+
record = logging.LogRecord(
|
|
912
|
+
name="test",
|
|
913
|
+
level=level,
|
|
914
|
+
pathname="mymodule.py",
|
|
915
|
+
lineno=1,
|
|
916
|
+
msg=message,
|
|
917
|
+
args=(),
|
|
918
|
+
exc_info=None,
|
|
919
|
+
)
|
|
920
|
+
for k, v in extra.items():
|
|
921
|
+
setattr(record, k, v)
|
|
922
|
+
return record
|
|
923
|
+
|
|
924
|
+
def test_colored_formatter_format(self) -> None:
|
|
925
|
+
assert pytest is not None # Type guard for pyright
|
|
926
|
+
fmt = ColoredFormatter()
|
|
927
|
+
|
|
928
|
+
# ── INFO → green ANSI code + green emoji ─────────────────────────────
|
|
929
|
+
result = fmt.format(self._make_record(logging.INFO, "hello"))
|
|
930
|
+
assert "\033[92m" in result, "Expected GREEN ANSI code"
|
|
931
|
+
assert "🟢" in result, "Expected green emoji"
|
|
932
|
+
|
|
933
|
+
# ── WARNING → yellow ANSI code + yellow emoji ─────────────────────────
|
|
934
|
+
result = fmt.format(self._make_record(logging.WARNING, "watch out"))
|
|
935
|
+
assert "\033[93m" in result, "Expected YELLOW ANSI code"
|
|
936
|
+
assert "🟡" in result, "Expected yellow emoji"
|
|
937
|
+
|
|
938
|
+
# ── ERROR → red ANSI code + red emoji ────────────────────────────────
|
|
939
|
+
result = fmt.format(self._make_record(logging.ERROR, "boom"))
|
|
940
|
+
assert "\033[91m" in result, "Expected RED ANSI code"
|
|
941
|
+
assert "🔴" in result, "Expected red emoji"
|
|
942
|
+
|
|
943
|
+
# ── METR → chart emoji ────────────────────────────────────────────────
|
|
944
|
+
metr = self._make_record(METRIC_LEVEL, "api_latency_ms=142ms")
|
|
945
|
+
metr.levelname = "METR"
|
|
946
|
+
assert "📊" in fmt.format(metr), "Expected chart emoji for METR"
|
|
947
|
+
|
|
948
|
+
# ── message text survives formatting ──────────────────────────────────
|
|
949
|
+
result = fmt.format(self._make_record(logging.INFO, "user logged in"))
|
|
950
|
+
assert "user logged in" in result, "Message text should survive formatting"
|
|
951
|
+
|
|
952
|
+
# ── RESET present to prevent terminal colour bleed ────────────────────
|
|
953
|
+
result = fmt.format(self._make_record(logging.INFO, "hello"))
|
|
954
|
+
assert "\033[0m" in result, "Expected RESET ANSI code"
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
class UniformLevelFormatter(logging.Formatter):
|
|
958
|
+
"""Formatter that serialises each log record as a single-line JSON object.
|
|
959
|
+
|
|
960
|
+
WHY: JSONL is independently parseable per line, supports nested types,
|
|
961
|
+
and avoids CSV escaping issues.
|
|
962
|
+
"""
|
|
963
|
+
|
|
964
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
965
|
+
import json as _json
|
|
966
|
+
|
|
967
|
+
timestamp = _get_timestamp()
|
|
968
|
+
extra = {
|
|
969
|
+
k: v
|
|
970
|
+
for k, v in record.__dict__.items()
|
|
971
|
+
if k not in _RESERVED_LOG_KEYS and not k.startswith("_")
|
|
972
|
+
}
|
|
973
|
+
logfile_name = extra.pop(
|
|
974
|
+
"logfile_name", os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
975
|
+
)
|
|
976
|
+
module_name = extra.pop("module_name", _get_caller_module())
|
|
977
|
+
source_file = extra.pop("source_file", "unknown")
|
|
978
|
+
source = extra.pop("source", None)
|
|
979
|
+
|
|
980
|
+
log_entry = {
|
|
981
|
+
"timestamp": timestamp["utc"],
|
|
982
|
+
"timestamp_local": timestamp["local"],
|
|
983
|
+
"level": record.levelname,
|
|
984
|
+
"logfile_name": logfile_name,
|
|
985
|
+
"module_name": module_name,
|
|
986
|
+
"source_file": source_file,
|
|
987
|
+
**({"source": source} if source is not None else {}),
|
|
988
|
+
"message": record.getMessage(),
|
|
989
|
+
**extra,
|
|
990
|
+
}
|
|
991
|
+
return _json.dumps(log_entry, default=str, separators=(",", ":"))
|
|
992
|
+
# WHY: separators=(',', ':') produces compact JSON matching serde_json output
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
996
|
+
if pytest is not None:
|
|
997
|
+
|
|
998
|
+
class TestUniformLevelFormatter:
|
|
999
|
+
"""WHY: JSONL shape is the shared contract between this module, Rust siblings,
|
|
1000
|
+
and every downstream log parser."""
|
|
1001
|
+
|
|
1002
|
+
def _make_record(
|
|
1003
|
+
self, level: int, message: str, levelname: str | None = None, **extra: Any
|
|
1004
|
+
) -> logging.LogRecord:
|
|
1005
|
+
record = logging.LogRecord(
|
|
1006
|
+
name="test",
|
|
1007
|
+
level=level,
|
|
1008
|
+
pathname="mymodule.py",
|
|
1009
|
+
lineno=1,
|
|
1010
|
+
msg=message,
|
|
1011
|
+
args=(),
|
|
1012
|
+
exc_info=None,
|
|
1013
|
+
)
|
|
1014
|
+
if levelname:
|
|
1015
|
+
record.levelname = levelname
|
|
1016
|
+
for k, v in extra.items():
|
|
1017
|
+
setattr(record, k, v)
|
|
1018
|
+
return record
|
|
1019
|
+
|
|
1020
|
+
def test_uniform_level_formatter_format(self) -> None:
|
|
1021
|
+
assert pytest is not None # Type guard for pyright
|
|
1022
|
+
import json
|
|
1023
|
+
|
|
1024
|
+
fmt = UniformLevelFormatter()
|
|
1025
|
+
|
|
1026
|
+
# ── output is valid JSON ──────────────────────────────────────────────
|
|
1027
|
+
record = self._make_record(
|
|
1028
|
+
logging.INFO, "hello", module_name="mod", source_file="src"
|
|
1029
|
+
)
|
|
1030
|
+
parsed = json.loads(fmt.format(record))
|
|
1031
|
+
assert isinstance(parsed, dict), (
|
|
1032
|
+
f"Expected dict, got {type(parsed).__name__}"
|
|
1033
|
+
)
|
|
1034
|
+
|
|
1035
|
+
# ── required top-level keys present ──────────────────────────────────
|
|
1036
|
+
for key in (
|
|
1037
|
+
"timestamp",
|
|
1038
|
+
"timestamp_local",
|
|
1039
|
+
"level",
|
|
1040
|
+
"logfile_name",
|
|
1041
|
+
"module_name",
|
|
1042
|
+
"source_file",
|
|
1043
|
+
"message",
|
|
1044
|
+
):
|
|
1045
|
+
assert key in parsed, f"Missing required key: {key}"
|
|
1046
|
+
|
|
1047
|
+
# ── compact separators (no spaces) ────────────────────────────────────
|
|
1048
|
+
result = fmt.format(record)
|
|
1049
|
+
assert ", " not in result, "Found ', ' — should use compact ',' separator"
|
|
1050
|
+
assert ": " not in result, "Found ': ' — should use compact ':' separator"
|
|
1051
|
+
|
|
1052
|
+
# ── WARNING serialises to renamed levelname "WARN" ────────────────────
|
|
1053
|
+
record = self._make_record(
|
|
1054
|
+
logging.WARNING, "watch out", module_name="mod", source_file="src"
|
|
1055
|
+
)
|
|
1056
|
+
parsed = json.loads(fmt.format(record))
|
|
1057
|
+
assert parsed["level"] == "WARN", (
|
|
1058
|
+
f"Expected 'WARN', got {parsed['level']!r}"
|
|
1059
|
+
)
|
|
1060
|
+
|
|
1061
|
+
# ── custom METR level serialises to "METR" ────────────────────────────
|
|
1062
|
+
record = self._make_record(
|
|
1063
|
+
METRIC_LEVEL,
|
|
1064
|
+
"latency=100ms",
|
|
1065
|
+
levelname="METR",
|
|
1066
|
+
module_name="mod",
|
|
1067
|
+
source_file="src",
|
|
1068
|
+
)
|
|
1069
|
+
assert json.loads(fmt.format(record))["level"] == "METR", (
|
|
1070
|
+
"Expected 'METR' level for metric records"
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
# ── extra structured fields survive into JSONL ─────────────────────────
|
|
1074
|
+
record = self._make_record(
|
|
1075
|
+
logging.INFO, "login", module_name="mod", source_file="src", user_id=42
|
|
1076
|
+
)
|
|
1077
|
+
assert json.loads(fmt.format(record)).get("user_id") == 42, (
|
|
1078
|
+
"Expected user_id=42 in JSONL"
|
|
1079
|
+
)
|
|
1080
|
+
|
|
1081
|
+
# ── reserved LogRecord internals do not leak into JSONL ───────────────
|
|
1082
|
+
record = self._make_record(
|
|
1083
|
+
logging.INFO, "hello", module_name="mod", source_file="src"
|
|
1084
|
+
)
|
|
1085
|
+
parsed = json.loads(fmt.format(record))
|
|
1086
|
+
for reserved in ("lineno", "funcName", "thread", "processName", "msecs"):
|
|
1087
|
+
assert reserved not in parsed, f"Reserved key leaked: {reserved}"
|
|
1088
|
+
|
|
1089
|
+
# ── source field absent when not set ─────────────────────────────────
|
|
1090
|
+
record = self._make_record(
|
|
1091
|
+
logging.INFO, "hello", module_name="mod", source_file="src"
|
|
1092
|
+
)
|
|
1093
|
+
assert "source" not in json.loads(fmt.format(record)), (
|
|
1094
|
+
"source field should be absent when not set"
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1097
|
+
# ── source field present when set ─────────────────────────────────────
|
|
1098
|
+
record = self._make_record(
|
|
1099
|
+
logging.INFO,
|
|
1100
|
+
"hello",
|
|
1101
|
+
module_name="mod",
|
|
1102
|
+
source_file="src",
|
|
1103
|
+
source="myservice",
|
|
1104
|
+
)
|
|
1105
|
+
assert json.loads(fmt.format(record)).get("source") == "myservice", (
|
|
1106
|
+
"Expected source='myservice'"
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1109
|
+
# ── non-serialisable value stringified, not crashed ───────────────────
|
|
1110
|
+
record = self._make_record(
|
|
1111
|
+
logging.INFO, "hello", module_name="mod", source_file="src"
|
|
1112
|
+
)
|
|
1113
|
+
record.weird = object()
|
|
1114
|
+
result = fmt.format(record)
|
|
1115
|
+
parsed = json.loads(result)
|
|
1116
|
+
assert "weird" in parsed, (
|
|
1117
|
+
"Non-serialisable value should be stringified, not dropped"
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
# ── SECTION 6 · RETRY HELPER ─────────────────────────────────────────────────
|
|
1122
|
+
# Goal: Isolate retry mechanics so business functions stay free of retry loops
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
def _with_file_retry(write_fn: Callable[[], None], log_file: str) -> list[str] | None:
|
|
1126
|
+
"""[TIER 2] Attempt write_fn() up to RETRY_MAX_ATTEMPTS times with exponential backoff.
|
|
1127
|
+
|
|
1128
|
+
WHY: Retry logic lives here — not in _flush_buffer — per RULE retry-mechanics.
|
|
1129
|
+
Business functions must not contain retry loops.
|
|
1130
|
+
"""
|
|
1131
|
+
for attempt in range(RETRY_MAX_ATTEMPTS):
|
|
1132
|
+
try:
|
|
1133
|
+
write_fn()
|
|
1134
|
+
return None # Success — nothing to re-buffer
|
|
1135
|
+
except Exception as e:
|
|
1136
|
+
# WHY: Exception (not bare except) — all non-system-exit errors are
|
|
1137
|
+
# treated as transient for file I/O per DOMAIN_RULES retry policy.
|
|
1138
|
+
if attempt < RETRY_MAX_ATTEMPTS - 1:
|
|
1139
|
+
delay = RETRY_BACKOFF_BASE * (2**attempt)
|
|
1140
|
+
_debug_print(
|
|
1141
|
+
f"Retry {attempt + 1}/{RETRY_MAX_ATTEMPTS} for {log_file} "
|
|
1142
|
+
f"after {delay}s: {e}"
|
|
1143
|
+
)
|
|
1144
|
+
time.sleep(delay)
|
|
1145
|
+
else:
|
|
1146
|
+
_debug_print(
|
|
1147
|
+
f"Failed to write logs to {log_file} after "
|
|
1148
|
+
f"{RETRY_MAX_ATTEMPTS} attempts: {e}"
|
|
1149
|
+
)
|
|
1150
|
+
return [] # Signals caller that all attempts failed
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1154
|
+
if pytest is not None:
|
|
1155
|
+
|
|
1156
|
+
class TestWithFileRetry:
|
|
1157
|
+
"""WHY: _with_file_retry is the only disk-failure safety net — a wrong return
|
|
1158
|
+
value on exhaustion causes silent log loss."""
|
|
1159
|
+
|
|
1160
|
+
def test_with_file_retry(self) -> None:
|
|
1161
|
+
assert pytest is not None # Type guard for pyright
|
|
1162
|
+
# ── success on first attempt returns None ─────────────────────────────
|
|
1163
|
+
calls = []
|
|
1164
|
+
|
|
1165
|
+
def write_fn() -> None:
|
|
1166
|
+
calls.append(1)
|
|
1167
|
+
|
|
1168
|
+
result = _with_file_retry(write_fn, "test.jsonl")
|
|
1169
|
+
assert result is None, f"Expected None on success, got {result!r}"
|
|
1170
|
+
assert len(calls) == 1, f"Expected 1 call, got {len(calls)}"
|
|
1171
|
+
|
|
1172
|
+
# ── success on third attempt (transient failures) returns None ────────
|
|
1173
|
+
attempt_count = [0]
|
|
1174
|
+
|
|
1175
|
+
def write_fn_transient() -> None:
|
|
1176
|
+
attempt_count[0] += 1
|
|
1177
|
+
if attempt_count[0] < 3:
|
|
1178
|
+
raise OSError("disk busy")
|
|
1179
|
+
|
|
1180
|
+
with patch("time.sleep"):
|
|
1181
|
+
result = _with_file_retry(write_fn_transient, "test.jsonl")
|
|
1182
|
+
assert result is None, (
|
|
1183
|
+
f"Expected None after transient recovery, got {result!r}"
|
|
1184
|
+
)
|
|
1185
|
+
assert attempt_count[0] == 3, f"Expected 3 attempts, got {attempt_count[0]}"
|
|
1186
|
+
|
|
1187
|
+
# ── total exhaustion returns empty list (re-buffer sentinel) ──────────
|
|
1188
|
+
with patch("time.sleep"):
|
|
1189
|
+
result = _with_file_retry(
|
|
1190
|
+
lambda: (_ for _ in ()).throw(OSError("disk full")), "test.jsonl"
|
|
1191
|
+
)
|
|
1192
|
+
assert result == [], f"Expected [] on exhaustion, got {result!r}"
|
|
1193
|
+
|
|
1194
|
+
# ── any exception type is retried up to max attempts ──────────────────
|
|
1195
|
+
perm_count = [0]
|
|
1196
|
+
|
|
1197
|
+
def write_fn_perm() -> None:
|
|
1198
|
+
perm_count[0] += 1
|
|
1199
|
+
raise PermissionError("not allowed")
|
|
1200
|
+
|
|
1201
|
+
with patch("time.sleep"):
|
|
1202
|
+
_with_file_retry(write_fn_perm, "test.jsonl")
|
|
1203
|
+
assert perm_count[0] == RETRY_MAX_ATTEMPTS, (
|
|
1204
|
+
f"Expected {RETRY_MAX_ATTEMPTS} attempts, got {perm_count[0]}"
|
|
1205
|
+
)
|
|
1206
|
+
|
|
1207
|
+
# ── backoff delays grow exponentially ─────────────────────────────────
|
|
1208
|
+
sleep_calls: list[float] = []
|
|
1209
|
+
|
|
1210
|
+
with patch("time.sleep", side_effect=lambda d: sleep_calls.append(d)):
|
|
1211
|
+
_with_file_retry(
|
|
1212
|
+
lambda: (_ for _ in ()).throw(OSError("error")), "test.jsonl"
|
|
1213
|
+
)
|
|
1214
|
+
assert len(sleep_calls) == RETRY_MAX_ATTEMPTS - 1, (
|
|
1215
|
+
f"Expected {RETRY_MAX_ATTEMPTS - 1} sleep calls, got {len(sleep_calls)}"
|
|
1216
|
+
)
|
|
1217
|
+
assert sleep_calls[1] == sleep_calls[0] * 2, (
|
|
1218
|
+
f"Expected exponential backoff: {sleep_calls[1]} == {sleep_calls[0]} * 2"
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
# ── SECTION 7 · QUEUE & WRITER ───────────────────────────────────────────────
|
|
1223
|
+
# Goal: Buffer log entries and flush them to disk on a background thread
|
|
1224
|
+
# RULE OVERRIDE: §PYTHON no-oop — QueueHandler inherits from logging.Handler
|
|
1225
|
+
# which requires class-based inheritance; no functional alternative exists.
|
|
1226
|
+
# Restore when: stdlib logging adds a hook-based handler protocol.
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
class QueueHandler(logging.Handler):
|
|
1230
|
+
"""Handler that enqueues formatted log records for background disk writing.
|
|
1231
|
+
|
|
1232
|
+
WHY: Decouples the caller's thread from disk I/O — emit() returns in microseconds.
|
|
1233
|
+
"""
|
|
1234
|
+
|
|
1235
|
+
def __init__(self, q: queue.Queue, log_suffix: str = "") -> None:
|
|
1236
|
+
super().__init__()
|
|
1237
|
+
self.queue = q
|
|
1238
|
+
self.log_suffix = log_suffix
|
|
1239
|
+
# WHY: log_suffix routes entries to different files sharing one writer thread
|
|
1240
|
+
|
|
1241
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
1242
|
+
try:
|
|
1243
|
+
msg = self.format(record)
|
|
1244
|
+
extra = {
|
|
1245
|
+
k: v
|
|
1246
|
+
for k, v in record.__dict__.items()
|
|
1247
|
+
if k not in _RESERVED_LOG_KEYS and not k.startswith("_")
|
|
1248
|
+
}
|
|
1249
|
+
logfile_name = extra.get(
|
|
1250
|
+
"logfile_name", os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
1251
|
+
)
|
|
1252
|
+
log_path = _get_log_path(logfile_name, suffix=self.log_suffix)
|
|
1253
|
+
self.queue.put_nowait((str(log_path), msg))
|
|
1254
|
+
except queue.Full:
|
|
1255
|
+
print("Queue full, dropping log", file=sys.stderr)
|
|
1256
|
+
except Exception as e:
|
|
1257
|
+
# WHY: Exception (not bare except) — queuing failures should not crash
|
|
1258
|
+
# the logging system; printing to stderr is the last-resort signal.
|
|
1259
|
+
print(f"Error queuing log: {e}", file=sys.stderr)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1263
|
+
if pytest is not None:
|
|
1264
|
+
|
|
1265
|
+
class TestQueueHandlerEmit:
|
|
1266
|
+
"""WHY: QueueHandler.emit is the only path from a log call to the writer thread —
|
|
1267
|
+
a wrong tuple shape or silent raise causes silent data loss."""
|
|
1268
|
+
|
|
1269
|
+
def test_emit(self) -> None:
|
|
1270
|
+
assert pytest is not None # Type guard for pyright
|
|
1271
|
+
import tempfile
|
|
1272
|
+
import io
|
|
1273
|
+
|
|
1274
|
+
# ── puts (path, message) tuple onto queue ─────────────────────────────
|
|
1275
|
+
q = queue.Queue()
|
|
1276
|
+
handler = QueueHandler(q, log_suffix="")
|
|
1277
|
+
handler.setFormatter(UniformLevelFormatter())
|
|
1278
|
+
record = logging.LogRecord(
|
|
1279
|
+
name="test",
|
|
1280
|
+
level=logging.INFO,
|
|
1281
|
+
pathname="mod.py",
|
|
1282
|
+
lineno=1,
|
|
1283
|
+
msg="hello",
|
|
1284
|
+
args=(),
|
|
1285
|
+
exc_info=None,
|
|
1286
|
+
)
|
|
1287
|
+
record.module_name = "testmod"
|
|
1288
|
+
record.source_file = "testmod"
|
|
1289
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1290
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
1291
|
+
handler.emit(record)
|
|
1292
|
+
assert not q.empty(), "Queue should not be empty after emit"
|
|
1293
|
+
path_str, msg_str = q.get_nowait()
|
|
1294
|
+
assert path_str.endswith(".jsonl"), (
|
|
1295
|
+
f"Expected .jsonl path, got {path_str!r}"
|
|
1296
|
+
)
|
|
1297
|
+
assert "hello" in msg_str, f"Expected 'hello' in message, got {msg_str!r}"
|
|
1298
|
+
|
|
1299
|
+
# ── .errors suffix routes to {module}.errors.jsonl ───────────────────
|
|
1300
|
+
q = queue.Queue()
|
|
1301
|
+
handler = QueueHandler(q, log_suffix=".errors")
|
|
1302
|
+
handler.setFormatter(UniformLevelFormatter())
|
|
1303
|
+
record = logging.LogRecord(
|
|
1304
|
+
name="test",
|
|
1305
|
+
level=logging.ERROR,
|
|
1306
|
+
pathname="mod.py",
|
|
1307
|
+
lineno=1,
|
|
1308
|
+
msg="boom",
|
|
1309
|
+
args=(),
|
|
1310
|
+
exc_info=None,
|
|
1311
|
+
)
|
|
1312
|
+
record.module_name = "testmod"
|
|
1313
|
+
record.source_file = "testmod"
|
|
1314
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1315
|
+
with patch("JSONL_LOGGER.PROJECT_DIRECTORY", tmp):
|
|
1316
|
+
handler.emit(record)
|
|
1317
|
+
path_str, _ = q.get_nowait()
|
|
1318
|
+
assert ".errors.jsonl" in path_str, (
|
|
1319
|
+
f"Expected '.errors.jsonl' in path, got {path_str!r}"
|
|
1320
|
+
)
|
|
1321
|
+
|
|
1322
|
+
# ── full queue prints to stderr, does not raise ───────────────────────
|
|
1323
|
+
q = queue.Queue(maxsize=1)
|
|
1324
|
+
q.put_nowait(("dummy", "dummy"))
|
|
1325
|
+
handler = QueueHandler(q, log_suffix="")
|
|
1326
|
+
handler.setFormatter(UniformLevelFormatter())
|
|
1327
|
+
record = logging.LogRecord(
|
|
1328
|
+
name="test",
|
|
1329
|
+
level=logging.INFO,
|
|
1330
|
+
pathname="mod.py",
|
|
1331
|
+
lineno=1,
|
|
1332
|
+
msg="overflow",
|
|
1333
|
+
args=(),
|
|
1334
|
+
exc_info=None,
|
|
1335
|
+
)
|
|
1336
|
+
record.module_name = "testmod"
|
|
1337
|
+
record.source_file = "testmod"
|
|
1338
|
+
buf = io.StringIO()
|
|
1339
|
+
with patch("sys.stderr", buf):
|
|
1340
|
+
with patch(
|
|
1341
|
+
"JSONL_LOGGER._get_log_path", return_value=Path("/tmp/test.jsonl")
|
|
1342
|
+
):
|
|
1343
|
+
handler.emit(record)
|
|
1344
|
+
assert "Queue full" in buf.getvalue(), (
|
|
1345
|
+
f"Expected 'Queue full' in stderr, got: {buf.getvalue()!r}"
|
|
1346
|
+
)
|
|
1347
|
+
|
|
1348
|
+
|
|
1349
|
+
def _writer_worker(q: queue.Queue) -> None:
|
|
1350
|
+
"""[TIER 2] Background thread: dequeue log entries and flush them to per-module files.
|
|
1351
|
+
|
|
1352
|
+
WHY: Single writer thread per queue eliminates concurrent open() calls.
|
|
1353
|
+
"""
|
|
1354
|
+
global _shutdown, _buffers
|
|
1355
|
+
|
|
1356
|
+
while not _shutdown or not q.empty():
|
|
1357
|
+
try:
|
|
1358
|
+
log_path_str, msg = q.get(timeout=FLUSH_INTERVAL)
|
|
1359
|
+
|
|
1360
|
+
paths_to_flush = []
|
|
1361
|
+
with _buffer_lock:
|
|
1362
|
+
if log_path_str not in _buffers:
|
|
1363
|
+
_buffers[log_path_str] = []
|
|
1364
|
+
_buffers[log_path_str].append(msg)
|
|
1365
|
+
|
|
1366
|
+
for path, buf in _buffers.items():
|
|
1367
|
+
if len(buf) >= BUFFER_SIZE:
|
|
1368
|
+
paths_to_flush.append(path)
|
|
1369
|
+
|
|
1370
|
+
for path in paths_to_flush:
|
|
1371
|
+
_flush_buffer(path)
|
|
1372
|
+
|
|
1373
|
+
q.task_done()
|
|
1374
|
+
|
|
1375
|
+
except queue.Empty:
|
|
1376
|
+
with _buffer_lock:
|
|
1377
|
+
paths_to_flush = list(_buffers.keys())
|
|
1378
|
+
|
|
1379
|
+
for path in paths_to_flush:
|
|
1380
|
+
_flush_buffer(path)
|
|
1381
|
+
except Exception as e:
|
|
1382
|
+
# WHY: Exception (not bare except) — writer errors should not crash
|
|
1383
|
+
# the thread; debug print surfaces the issue for investigation.
|
|
1384
|
+
_debug_print(f"Writer error: {e}")
|
|
1385
|
+
try:
|
|
1386
|
+
q.task_done()
|
|
1387
|
+
except ValueError:
|
|
1388
|
+
pass
|
|
1389
|
+
|
|
1390
|
+
# WHY: Final sweep on shutdown — ensures no entries remain after _shutdown=True
|
|
1391
|
+
with _buffer_lock:
|
|
1392
|
+
paths_to_flush = list(_buffers.keys())
|
|
1393
|
+
|
|
1394
|
+
for path in paths_to_flush:
|
|
1395
|
+
_flush_buffer(path)
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _flush_buffer(log_file: str) -> None:
|
|
1399
|
+
"""[TIER 2] Write buffered log lines for one file to disk via the retry helper.
|
|
1400
|
+
|
|
1401
|
+
NOTE: This function acquires _buffer_lock internally. Do NOT call it while
|
|
1402
|
+
already holding the lock.
|
|
1403
|
+
"""
|
|
1404
|
+
global _buffers
|
|
1405
|
+
|
|
1406
|
+
with _buffer_lock:
|
|
1407
|
+
if log_file not in _buffers or not _buffers[log_file]:
|
|
1408
|
+
return
|
|
1409
|
+
|
|
1410
|
+
to_write = _buffers[log_file][:]
|
|
1411
|
+
_buffers[log_file] = []
|
|
1412
|
+
|
|
1413
|
+
def write_fn() -> None:
|
|
1414
|
+
with open(log_file, "a") as f:
|
|
1415
|
+
for line in to_write:
|
|
1416
|
+
f.write(line + "\n")
|
|
1417
|
+
|
|
1418
|
+
failed = _with_file_retry(write_fn, log_file)
|
|
1419
|
+
|
|
1420
|
+
if failed is not None:
|
|
1421
|
+
with _buffer_lock:
|
|
1422
|
+
combined = to_write + _buffers.get(log_file, [])
|
|
1423
|
+
if len(combined) > LOGGER_MAX_BUFFER_SIZE:
|
|
1424
|
+
dropped = len(combined) - LOGGER_MAX_BUFFER_SIZE
|
|
1425
|
+
combined = combined[-LOGGER_MAX_BUFFER_SIZE:]
|
|
1426
|
+
print(
|
|
1427
|
+
f"[LOGGER WARN] Buffer for '{log_file}' exceeded "
|
|
1428
|
+
f"{LOGGER_MAX_BUFFER_SIZE} entries — dropped {dropped} oldest.",
|
|
1429
|
+
file=sys.stderr,
|
|
1430
|
+
)
|
|
1431
|
+
_buffers[log_file] = combined
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1435
|
+
if pytest is not None:
|
|
1436
|
+
|
|
1437
|
+
class TestFlushBuffer:
|
|
1438
|
+
"""WHY: _flush_buffer is the bridge between in-memory buffers and disk —
|
|
1439
|
+
a double-write, silent drop, or missed buffer cap allows data loss or OOM."""
|
|
1440
|
+
|
|
1441
|
+
def test_flush_buffer(self) -> None:
|
|
1442
|
+
assert pytest is not None # Type guard for pyright
|
|
1443
|
+
import JSONL_LOGGER as mod
|
|
1444
|
+
import io
|
|
1445
|
+
|
|
1446
|
+
# ── no-op on empty buffer — no open() call made ───────────────────────
|
|
1447
|
+
original = mod._buffers.copy()
|
|
1448
|
+
mod._buffers["fake.jsonl"] = []
|
|
1449
|
+
try:
|
|
1450
|
+
with patch("builtins.open") as mock_open:
|
|
1451
|
+
_flush_buffer("fake.jsonl")
|
|
1452
|
+
mock_open.assert_not_called()
|
|
1453
|
+
finally:
|
|
1454
|
+
mod._buffers = original
|
|
1455
|
+
|
|
1456
|
+
# ── buffer cleared after successful write ─────────────────────────────
|
|
1457
|
+
original = mod._buffers.copy()
|
|
1458
|
+
mod._buffers["fake.jsonl"] = ["line1", "line2"]
|
|
1459
|
+
try:
|
|
1460
|
+
with patch("builtins.open") as mock_open:
|
|
1461
|
+
mock_open.return_value.__enter__ = lambda self: (
|
|
1462
|
+
mock_open.return_value
|
|
1463
|
+
)
|
|
1464
|
+
mock_open.return_value.__exit__ = lambda self, *args: None
|
|
1465
|
+
with patch("JSONL_LOGGER._with_file_retry", return_value=None):
|
|
1466
|
+
_flush_buffer("fake.jsonl")
|
|
1467
|
+
assert mod._buffers.get("fake.jsonl", []) == [], (
|
|
1468
|
+
"Buffer should be empty after successful flush"
|
|
1469
|
+
)
|
|
1470
|
+
finally:
|
|
1471
|
+
mod._buffers = original
|
|
1472
|
+
|
|
1473
|
+
# ── lines re-buffered on retry exhaustion ─────────────────────────────
|
|
1474
|
+
original = mod._buffers.copy()
|
|
1475
|
+
mod._buffers["fake.jsonl"] = ["line1", "line2"]
|
|
1476
|
+
try:
|
|
1477
|
+
with patch("JSONL_LOGGER._with_file_retry", return_value=[]):
|
|
1478
|
+
_flush_buffer("fake.jsonl")
|
|
1479
|
+
assert len(mod._buffers.get("fake.jsonl", [])) == 2, (
|
|
1480
|
+
"Lines should be re-buffered on retry exhaustion"
|
|
1481
|
+
)
|
|
1482
|
+
finally:
|
|
1483
|
+
mod._buffers = original
|
|
1484
|
+
|
|
1485
|
+
# ── oldest entries dropped when buffer exceeds cap ────────────────────
|
|
1486
|
+
original_buffers = mod._buffers.copy()
|
|
1487
|
+
original_cap = mod.LOGGER_MAX_BUFFER_SIZE
|
|
1488
|
+
mod.LOGGER_MAX_BUFFER_SIZE = 3
|
|
1489
|
+
try:
|
|
1490
|
+
buf = io.StringIO()
|
|
1491
|
+
with patch("sys.stderr", buf):
|
|
1492
|
+
with patch("JSONL_LOGGER._with_file_retry", return_value=[]):
|
|
1493
|
+
mod._buffers["fake.jsonl"] = ["a", "b", "c", "d"]
|
|
1494
|
+
_flush_buffer("fake.jsonl")
|
|
1495
|
+
remaining = mod._buffers.get("fake.jsonl", [])
|
|
1496
|
+
assert len(remaining) <= mod.LOGGER_MAX_BUFFER_SIZE, (
|
|
1497
|
+
f"Buffer should not exceed cap {mod.LOGGER_MAX_BUFFER_SIZE}, got {len(remaining)}"
|
|
1498
|
+
)
|
|
1499
|
+
finally:
|
|
1500
|
+
mod._buffers = original_buffers
|
|
1501
|
+
mod.LOGGER_MAX_BUFFER_SIZE = original_cap
|
|
1502
|
+
|
|
1503
|
+
# ── stderr warning emitted when buffer cap exceeded ───────────────────
|
|
1504
|
+
original_buffers = mod._buffers.copy()
|
|
1505
|
+
original_cap = mod.LOGGER_MAX_BUFFER_SIZE
|
|
1506
|
+
mod.LOGGER_MAX_BUFFER_SIZE = 2
|
|
1507
|
+
mod._buffers["fake.jsonl"] = ["a", "b", "c", "d"]
|
|
1508
|
+
try:
|
|
1509
|
+
buf = io.StringIO()
|
|
1510
|
+
with patch("sys.stderr", buf):
|
|
1511
|
+
with patch("JSONL_LOGGER._with_file_retry", return_value=[]):
|
|
1512
|
+
_flush_buffer("fake.jsonl")
|
|
1513
|
+
assert (
|
|
1514
|
+
"dropped" in buf.getvalue().lower()
|
|
1515
|
+
or "exceeded" in buf.getvalue().lower()
|
|
1516
|
+
), f"Expected buffer overflow warning, got: {buf.getvalue()!r}"
|
|
1517
|
+
finally:
|
|
1518
|
+
mod._buffers = original_buffers
|
|
1519
|
+
mod.LOGGER_MAX_BUFFER_SIZE = original_cap
|
|
1520
|
+
|
|
1521
|
+
|
|
1522
|
+
# ── SECTION 8 · SHUTDOWN HANDLERS ────────────────────────────────────────────
|
|
1523
|
+
# Goal: Ensure all pending log entries reach disk before process exit
|
|
1524
|
+
|
|
1525
|
+
|
|
1526
|
+
def _flush_logs() -> None:
|
|
1527
|
+
"""[TIER 2] Drain the log queue and join the writer thread before process exit.
|
|
1528
|
+
|
|
1529
|
+
WHY: queue.join() blocks until every put() has a matching task_done().
|
|
1530
|
+
"""
|
|
1531
|
+
if _log_queue is None:
|
|
1532
|
+
return
|
|
1533
|
+
|
|
1534
|
+
global _shutdown
|
|
1535
|
+
_shutdown = True
|
|
1536
|
+
|
|
1537
|
+
if _writer_thread and _writer_thread.is_alive():
|
|
1538
|
+
_writer_thread.join(timeout=5)
|
|
1539
|
+
|
|
1540
|
+
try:
|
|
1541
|
+
while True:
|
|
1542
|
+
try:
|
|
1543
|
+
log_path, msg = _log_queue.get_nowait()
|
|
1544
|
+
with _buffer_lock:
|
|
1545
|
+
if log_path not in _buffers:
|
|
1546
|
+
_buffers[log_path] = []
|
|
1547
|
+
_buffers[log_path].append(msg)
|
|
1548
|
+
_log_queue.task_done()
|
|
1549
|
+
except queue.Empty:
|
|
1550
|
+
break
|
|
1551
|
+
except Exception as e:
|
|
1552
|
+
# WHY: Exception (not bare except) — drain failures should not crash
|
|
1553
|
+
# shutdown; best-effort flush is better than nothing.
|
|
1554
|
+
_debug_print(f"Error draining queue during shutdown: {e}")
|
|
1555
|
+
|
|
1556
|
+
with _buffer_lock:
|
|
1557
|
+
paths = list(_buffers.keys())
|
|
1558
|
+
for path in paths:
|
|
1559
|
+
_flush_buffer(path)
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1563
|
+
if pytest is not None:
|
|
1564
|
+
|
|
1565
|
+
class TestFlushLogs:
|
|
1566
|
+
"""WHY: _flush_logs is the last-mile guarantee that buffered entries reach disk
|
|
1567
|
+
before process exit."""
|
|
1568
|
+
|
|
1569
|
+
def test_flush_logs(self) -> None:
|
|
1570
|
+
assert pytest is not None # Type guard for pyright
|
|
1571
|
+
import JSONL_LOGGER as mod
|
|
1572
|
+
|
|
1573
|
+
# ── sets _shutdown flag to True ───────────────────────────────────────
|
|
1574
|
+
original = mod._shutdown
|
|
1575
|
+
try:
|
|
1576
|
+
mod._shutdown = False
|
|
1577
|
+
with (
|
|
1578
|
+
patch.object(mod._writer_thread, "is_alive", return_value=False),
|
|
1579
|
+
):
|
|
1580
|
+
_flush_logs()
|
|
1581
|
+
assert mod._shutdown is True, "_shutdown should be True after flush"
|
|
1582
|
+
finally:
|
|
1583
|
+
mod._shutdown = original
|
|
1584
|
+
|
|
1585
|
+
# ── drains queue and flushes buffers when thread is alive ───────────────
|
|
1586
|
+
original_shutdown = mod._shutdown
|
|
1587
|
+
try:
|
|
1588
|
+
mod._shutdown = False
|
|
1589
|
+
with (
|
|
1590
|
+
patch.object(mod._log_queue, "get_nowait") as mock_get,
|
|
1591
|
+
patch.object(mod._log_queue, "task_done"),
|
|
1592
|
+
patch.object(mod._writer_thread, "is_alive", return_value=True),
|
|
1593
|
+
patch.object(mod._writer_thread, "join"),
|
|
1594
|
+
patch("JSONL_LOGGER._flush_buffer"),
|
|
1595
|
+
):
|
|
1596
|
+
_flush_logs()
|
|
1597
|
+
mock_get.assert_called(), ("Queue should be drained during flush")
|
|
1598
|
+
finally:
|
|
1599
|
+
mod._shutdown = original_shutdown
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def _chain_signal_handler(signum: int, frame: Any, previous_handler: Any) -> None:
|
|
1603
|
+
"""[TIER 2] Flush logs then call the previously registered signal handler.
|
|
1604
|
+
|
|
1605
|
+
WHY: Naive signal override breaks host apps (FastAPI, Gunicorn, Click).
|
|
1606
|
+
Chaining preserves the full shutdown chain.
|
|
1607
|
+
"""
|
|
1608
|
+
_flush_logs()
|
|
1609
|
+
if callable(previous_handler):
|
|
1610
|
+
previous_handler(signum, frame)
|
|
1611
|
+
elif previous_handler == signal.default_int_handler:
|
|
1612
|
+
raise KeyboardInterrupt
|
|
1613
|
+
elif previous_handler not in (signal.SIG_IGN, signal.SIG_DFL):
|
|
1614
|
+
signal.signal(signum, previous_handler)
|
|
1615
|
+
os.kill(os.getpid(), signum)
|
|
1616
|
+
|
|
1617
|
+
|
|
1618
|
+
# ── SECTION 9 · LOGGER INITIALIZATION ────────────────────────────────────────
|
|
1619
|
+
# Goal: Construct and configure the logger and its handlers exactly once at import
|
|
1620
|
+
|
|
1621
|
+
|
|
1622
|
+
def _init_logger() -> logging.Logger:
|
|
1623
|
+
"""[TIER 2] Build the module logger with queue-based handlers for all output files.
|
|
1624
|
+
|
|
1625
|
+
WHY: logger is a module singleton so all callers share the same queue.
|
|
1626
|
+
"""
|
|
1627
|
+
global _log_queue, _writer_thread
|
|
1628
|
+
|
|
1629
|
+
_logger = logging.getLogger(__name__)
|
|
1630
|
+
_logger.setLevel(logging.DEBUG)
|
|
1631
|
+
_logger.handlers.clear()
|
|
1632
|
+
|
|
1633
|
+
_log_queue = queue.Queue(maxsize=100000)
|
|
1634
|
+
|
|
1635
|
+
_writer_thread = threading.Thread(
|
|
1636
|
+
target=_writer_worker,
|
|
1637
|
+
args=(_log_queue,),
|
|
1638
|
+
daemon=LOGGER_DAEMON_THREAD,
|
|
1639
|
+
name="JSONL_LOGGER_writer",
|
|
1640
|
+
)
|
|
1641
|
+
_writer_thread.start()
|
|
1642
|
+
|
|
1643
|
+
atexit.register(_flush_logs)
|
|
1644
|
+
|
|
1645
|
+
if LOGGER_REGISTER_SIGNALS:
|
|
1646
|
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
1647
|
+
previous = signal.getsignal(sig)
|
|
1648
|
+
signal.signal(
|
|
1649
|
+
sig,
|
|
1650
|
+
lambda signum, frame, prev=previous: _chain_signal_handler(
|
|
1651
|
+
signum, frame, prev
|
|
1652
|
+
),
|
|
1653
|
+
)
|
|
1654
|
+
|
|
1655
|
+
if CONSOLE_LOGGING_ENABLED:
|
|
1656
|
+
console_handler = logging.StreamHandler()
|
|
1657
|
+
console_handler.setLevel(logging.DEBUG)
|
|
1658
|
+
console_handler.setFormatter(ColoredFormatter())
|
|
1659
|
+
_logger.addHandler(console_handler)
|
|
1660
|
+
|
|
1661
|
+
file_handler = QueueHandler(_log_queue, log_suffix="")
|
|
1662
|
+
file_handler.setLevel(logging.DEBUG)
|
|
1663
|
+
file_handler.setFormatter(UniformLevelFormatter())
|
|
1664
|
+
_logger.addHandler(file_handler)
|
|
1665
|
+
|
|
1666
|
+
errors_handler = QueueHandler(_log_queue, log_suffix=".errors")
|
|
1667
|
+
errors_handler.setLevel(logging.ERROR)
|
|
1668
|
+
errors_handler.setFormatter(UniformLevelFormatter())
|
|
1669
|
+
_logger.addHandler(errors_handler)
|
|
1670
|
+
|
|
1671
|
+
warn_handler = QueueHandler(_log_queue, log_suffix=".warn")
|
|
1672
|
+
warn_handler.setLevel(logging.WARNING)
|
|
1673
|
+
warn_handler.addFilter(lambda record: record.levelno == logging.WARNING)
|
|
1674
|
+
warn_handler.setFormatter(UniformLevelFormatter())
|
|
1675
|
+
_logger.addHandler(warn_handler)
|
|
1676
|
+
|
|
1677
|
+
return _logger
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
# WHY: Initialized automatically at import.
|
|
1681
|
+
# All callers share the same queue and writer thread.
|
|
1682
|
+
logger = _init_logger()
|
|
1683
|
+
|
|
1684
|
+
_metrics_logger: logging.Logger = logging.getLogger(f"{__name__}.metrics")
|
|
1685
|
+
_metrics_logger.setLevel(logging.DEBUG)
|
|
1686
|
+
_metrics_logger.handlers.clear()
|
|
1687
|
+
_metrics_logger.propagate = False
|
|
1688
|
+
|
|
1689
|
+
_metrics_file_handler = QueueHandler(_log_queue, log_suffix=".metrics") # type: ignore[arg-type]
|
|
1690
|
+
_metrics_file_handler.setLevel(logging.DEBUG)
|
|
1691
|
+
_metrics_file_handler.setFormatter(UniformLevelFormatter())
|
|
1692
|
+
_metrics_logger.addHandler(_metrics_file_handler)
|
|
1693
|
+
|
|
1694
|
+
_notifications_logger: logging.Logger = logging.getLogger(f"{__name__}.notifications")
|
|
1695
|
+
_notifications_logger.setLevel(logging.DEBUG)
|
|
1696
|
+
_notifications_logger.handlers.clear()
|
|
1697
|
+
_notifications_logger.propagate = False
|
|
1698
|
+
|
|
1699
|
+
_notifications_file_handler = QueueHandler(_log_queue, log_suffix="") # type: ignore[arg-type]
|
|
1700
|
+
_notifications_file_handler.setLevel(logging.DEBUG)
|
|
1701
|
+
_notifications_file_handler.setFormatter(UniformLevelFormatter())
|
|
1702
|
+
_notifications_logger.addHandler(_notifications_file_handler)
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
# ── SECTION 10 · PUBLIC API ───────────────────────────────────────────────────
|
|
1706
|
+
# Goal: Expose four clean logging functions for all application callers
|
|
1707
|
+
|
|
1708
|
+
|
|
1709
|
+
def log_info(
|
|
1710
|
+
message: str,
|
|
1711
|
+
logfile_name: str | None = None,
|
|
1712
|
+
module_name: str | None = None,
|
|
1713
|
+
**extra_fields: Any,
|
|
1714
|
+
) -> None:
|
|
1715
|
+
"""[TIER 1] Record an informational event in the module's audit log.
|
|
1716
|
+
|
|
1717
|
+
WHY: Primary audit trail for application events — covers the happy path.
|
|
1718
|
+
"""
|
|
1719
|
+
if logfile_name is None:
|
|
1720
|
+
logfile_name = os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
1721
|
+
if module_name is None:
|
|
1722
|
+
module_name = _get_caller_module()
|
|
1723
|
+
source_file = _get_actual_source_file()
|
|
1724
|
+
_warn_non_primitive_fields(extra_fields, module_name)
|
|
1725
|
+
assert logger is not None # Initialized at import time
|
|
1726
|
+
logger.info(
|
|
1727
|
+
message,
|
|
1728
|
+
extra={
|
|
1729
|
+
"logfile_name": logfile_name,
|
|
1730
|
+
"module_name": module_name,
|
|
1731
|
+
"source_file": source_file,
|
|
1732
|
+
**extra_fields,
|
|
1733
|
+
},
|
|
1734
|
+
)
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1738
|
+
if pytest is not None:
|
|
1739
|
+
|
|
1740
|
+
class TestLogInfo:
|
|
1741
|
+
"""WHY: log_info is the highest-frequency call in the public API — a wrong level,
|
|
1742
|
+
dropped message, or missing dual-source entry silently corrupts every audit trail."""
|
|
1743
|
+
|
|
1744
|
+
def test_log_info(self) -> None:
|
|
1745
|
+
assert pytest is not None # Type guard for pyright
|
|
1746
|
+
# ── explicit logfile_name → correct file routing ────────────────────────
|
|
1747
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1748
|
+
log_info("test message", logfile_name="test_file.py")
|
|
1749
|
+
mock_handle.assert_called_once()
|
|
1750
|
+
record = mock_handle.call_args[0][0]
|
|
1751
|
+
assert record.levelno == logging.INFO, (
|
|
1752
|
+
f"Expected INFO level, got {record.levelno}"
|
|
1753
|
+
)
|
|
1754
|
+
assert record.getMessage() == "test message", (
|
|
1755
|
+
f"Expected 'test message', got {record.getMessage()!r}"
|
|
1756
|
+
)
|
|
1757
|
+
|
|
1758
|
+
# ── no logfile_name/module_name → auto-detects non-empty values ─────────
|
|
1759
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1760
|
+
log_info("test message")
|
|
1761
|
+
record = mock_handle.call_args[0][0]
|
|
1762
|
+
assert record.module_name is not None, (
|
|
1763
|
+
"module_name should be auto-detected"
|
|
1764
|
+
)
|
|
1765
|
+
assert len(record.module_name) > 0, "module_name should be non-empty"
|
|
1766
|
+
|
|
1767
|
+
# ── empty message logs without error ──────────────────────────────────
|
|
1768
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1769
|
+
log_info("")
|
|
1770
|
+
mock_handle.assert_called_once()
|
|
1771
|
+
assert mock_handle.call_args[0][0].getMessage() == "", (
|
|
1772
|
+
"Empty message should log without error"
|
|
1773
|
+
)
|
|
1774
|
+
|
|
1775
|
+
# ── special characters and emoji preserved intact ─────────────────────
|
|
1776
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1777
|
+
log_info('Message with 🐍 emoji and "quotes"')
|
|
1778
|
+
msg = mock_handle.call_args[0][0].getMessage()
|
|
1779
|
+
assert "🐍" in msg, "Emoji should be preserved"
|
|
1780
|
+
assert "quotes" in msg, "Quotes should be preserved"
|
|
1781
|
+
|
|
1782
|
+
# ── 10 000-char message not truncated ─────────────────────────────────
|
|
1783
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1784
|
+
log_info("x" * 10000)
|
|
1785
|
+
assert len(mock_handle.call_args[0][0].getMessage()) == 10000, (
|
|
1786
|
+
"10k char message should not be truncated"
|
|
1787
|
+
)
|
|
1788
|
+
|
|
1789
|
+
# ── dual-source: logfile_name, module_name and source_file all present ───
|
|
1790
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1791
|
+
log_info(
|
|
1792
|
+
"test message", logfile_name="LOGS", module_name="grouped_name"
|
|
1793
|
+
)
|
|
1794
|
+
record = mock_handle.call_args[0][0]
|
|
1795
|
+
assert hasattr(record, "logfile_name"), "Missing logfile_name attr"
|
|
1796
|
+
assert hasattr(record, "module_name"), "Missing module_name attr"
|
|
1797
|
+
assert hasattr(record, "source_file"), "Missing source_file attr"
|
|
1798
|
+
assert record.logfile_name == "LOGS", (
|
|
1799
|
+
f"Expected 'LOGS', got {record.logfile_name!r}"
|
|
1800
|
+
)
|
|
1801
|
+
assert record.module_name == "grouped_name", (
|
|
1802
|
+
f"Expected 'grouped_name', got {record.module_name!r}"
|
|
1803
|
+
)
|
|
1804
|
+
assert record.source_file is not None, "source_file should not be None"
|
|
1805
|
+
assert len(record.source_file) > 0, "source_file should be non-empty"
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
def log_warn(
|
|
1809
|
+
message: str,
|
|
1810
|
+
logfile_name: str | None = None,
|
|
1811
|
+
module_name: str | None = None,
|
|
1812
|
+
**extra_fields: Any,
|
|
1813
|
+
) -> None:
|
|
1814
|
+
"""[TIER 1] Record a recoverable anomaly or approaching-limit condition.
|
|
1815
|
+
|
|
1816
|
+
WHY: Warning-level events signal conditions that don't fail the current operation
|
|
1817
|
+
but indicate the system is under stress.
|
|
1818
|
+
"""
|
|
1819
|
+
if logfile_name is None:
|
|
1820
|
+
logfile_name = os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
1821
|
+
if module_name is None:
|
|
1822
|
+
module_name = _get_caller_module()
|
|
1823
|
+
source_file = _get_actual_source_file()
|
|
1824
|
+
_warn_non_primitive_fields(extra_fields, module_name)
|
|
1825
|
+
assert logger is not None # Initialized at import time
|
|
1826
|
+
logger.warning(
|
|
1827
|
+
message,
|
|
1828
|
+
extra={
|
|
1829
|
+
"logfile_name": logfile_name,
|
|
1830
|
+
"module_name": module_name,
|
|
1831
|
+
"source_file": source_file,
|
|
1832
|
+
**extra_fields,
|
|
1833
|
+
},
|
|
1834
|
+
)
|
|
1835
|
+
|
|
1836
|
+
|
|
1837
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1838
|
+
if pytest is not None:
|
|
1839
|
+
|
|
1840
|
+
class TestLogWarn:
|
|
1841
|
+
"""WHY: log_warn signals approaching-limit conditions — a wrong level or dropped
|
|
1842
|
+
message causes silent monitoring gaps."""
|
|
1843
|
+
|
|
1844
|
+
def test_log_warn(self) -> None:
|
|
1845
|
+
assert pytest is not None # Type guard for pyright
|
|
1846
|
+
# ── explicit logfile_name → WARNING level and correct message ───────────
|
|
1847
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1848
|
+
log_warn("warning message", logfile_name="test_file.py")
|
|
1849
|
+
mock_handle.assert_called_once()
|
|
1850
|
+
record = mock_handle.call_args[0][0]
|
|
1851
|
+
assert record.levelno == logging.WARNING, (
|
|
1852
|
+
f"Expected WARNING level, got {record.levelno}"
|
|
1853
|
+
)
|
|
1854
|
+
assert record.getMessage() == "warning message", (
|
|
1855
|
+
f"Expected 'warning message', got {record.getMessage()!r}"
|
|
1856
|
+
)
|
|
1857
|
+
|
|
1858
|
+
# ── no logfile_name/module_name → auto-detects non-empty values ─────────
|
|
1859
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1860
|
+
log_warn("warning message")
|
|
1861
|
+
assert len(mock_handle.call_args[0][0].module_name) > 0, (
|
|
1862
|
+
"module_name should be auto-detected and non-empty"
|
|
1863
|
+
)
|
|
1864
|
+
|
|
1865
|
+
# ── empty message logs at WARNING without error ───────────────────────
|
|
1866
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1867
|
+
log_warn("")
|
|
1868
|
+
assert mock_handle.call_args[0][0].levelno == logging.WARNING, (
|
|
1869
|
+
"Empty message should log at WARNING level"
|
|
1870
|
+
)
|
|
1871
|
+
|
|
1872
|
+
# ── special characters and emoji preserved ────────────────────────────
|
|
1873
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1874
|
+
log_warn("Warning: ⚠️ system overloaded")
|
|
1875
|
+
assert "⚠️" in mock_handle.call_args[0][0].getMessage(), (
|
|
1876
|
+
"Emoji should be preserved in warning"
|
|
1877
|
+
)
|
|
1878
|
+
|
|
1879
|
+
# ── 10 000-char message not truncated ─────────────────────────────────
|
|
1880
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1881
|
+
log_warn("w" * 10000)
|
|
1882
|
+
assert len(mock_handle.call_args[0][0].getMessage()) == 10000, (
|
|
1883
|
+
"10k char message should not be truncated"
|
|
1884
|
+
)
|
|
1885
|
+
|
|
1886
|
+
# ── dual-source: logfile_name, module_name and source_file all present ───
|
|
1887
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1888
|
+
log_warn(
|
|
1889
|
+
"warning message", logfile_name="LOGS", module_name="grouped_name"
|
|
1890
|
+
)
|
|
1891
|
+
record = mock_handle.call_args[0][0]
|
|
1892
|
+
assert hasattr(record, "logfile_name"), "Missing logfile_name attr"
|
|
1893
|
+
assert hasattr(record, "module_name"), "Missing module_name attr"
|
|
1894
|
+
assert hasattr(record, "source_file"), "Missing source_file attr"
|
|
1895
|
+
assert record.logfile_name == "LOGS", (
|
|
1896
|
+
f"Expected 'LOGS', got {record.logfile_name!r}"
|
|
1897
|
+
)
|
|
1898
|
+
assert record.module_name == "grouped_name", (
|
|
1899
|
+
f"Expected 'grouped_name', got {record.module_name!r}"
|
|
1900
|
+
)
|
|
1901
|
+
assert record.source_file is not None, "source_file should not be None"
|
|
1902
|
+
assert len(record.source_file) > 0, "source_file should be non-empty"
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
def log_error(
|
|
1906
|
+
message: str,
|
|
1907
|
+
logfile_name: str | None = None,
|
|
1908
|
+
module_name: str | None = None,
|
|
1909
|
+
**extra_fields: Any,
|
|
1910
|
+
) -> None:
|
|
1911
|
+
"""[TIER 1] Record a failure requiring manual investigation.
|
|
1912
|
+
|
|
1913
|
+
Dual-write: every call appends to BOTH:
|
|
1914
|
+
· {module}.jsonl — full audit log alongside info/warn entries
|
|
1915
|
+
· {module}.errors.jsonl — errors only, for fast triage without grep
|
|
1916
|
+
|
|
1917
|
+
WHY: Two files serve two workflows — errors.jsonl for on-call triage,
|
|
1918
|
+
main audit file for root-cause investigation.
|
|
1919
|
+
"""
|
|
1920
|
+
if logfile_name is None:
|
|
1921
|
+
logfile_name = os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
1922
|
+
if module_name is None:
|
|
1923
|
+
module_name = _get_caller_module()
|
|
1924
|
+
source_file = _get_actual_source_file()
|
|
1925
|
+
_warn_non_primitive_fields(extra_fields, module_name)
|
|
1926
|
+
assert logger is not None # Initialized at import time
|
|
1927
|
+
logger.error(
|
|
1928
|
+
message,
|
|
1929
|
+
extra={
|
|
1930
|
+
"logfile_name": logfile_name,
|
|
1931
|
+
"module_name": module_name,
|
|
1932
|
+
"source_file": source_file,
|
|
1933
|
+
**extra_fields,
|
|
1934
|
+
},
|
|
1935
|
+
)
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
1939
|
+
if pytest is not None:
|
|
1940
|
+
|
|
1941
|
+
class TestLogError:
|
|
1942
|
+
"""WHY: log_error dual-writes to both audit and errors files — a wrong level,
|
|
1943
|
+
missed dual-write, or info/warn leaking into errors.jsonl defeats fast-triage."""
|
|
1944
|
+
|
|
1945
|
+
def test_log_error(self) -> None:
|
|
1946
|
+
assert pytest is not None # Type guard for pyright
|
|
1947
|
+
# ── explicit logfile_name → ERROR level and correct message ─────────────
|
|
1948
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1949
|
+
log_error("error message", logfile_name="test_file.py")
|
|
1950
|
+
mock_handle.assert_called_once()
|
|
1951
|
+
record = mock_handle.call_args[0][0]
|
|
1952
|
+
assert record.levelno == logging.ERROR, (
|
|
1953
|
+
f"Expected ERROR level, got {record.levelno}"
|
|
1954
|
+
)
|
|
1955
|
+
assert record.getMessage() == "error message", (
|
|
1956
|
+
f"Expected 'error message', got {record.getMessage()!r}"
|
|
1957
|
+
)
|
|
1958
|
+
|
|
1959
|
+
# ── no logfile_name/module_name → auto-detects non-empty values ─────────
|
|
1960
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1961
|
+
log_error("error message")
|
|
1962
|
+
assert len(mock_handle.call_args[0][0].module_name) > 0, (
|
|
1963
|
+
"module_name should be auto-detected and non-empty"
|
|
1964
|
+
)
|
|
1965
|
+
|
|
1966
|
+
# ── empty message logs at ERROR without crash ─────────────────────────
|
|
1967
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1968
|
+
log_error("")
|
|
1969
|
+
assert mock_handle.call_args[0][0].levelno == logging.ERROR, (
|
|
1970
|
+
"Empty message should log at ERROR level"
|
|
1971
|
+
)
|
|
1972
|
+
|
|
1973
|
+
# ── special characters and emoji preserved ────────────────────────────
|
|
1974
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1975
|
+
log_error("Error: ❌ failed with code 0xFF")
|
|
1976
|
+
assert "❌" in mock_handle.call_args[0][0].getMessage(), (
|
|
1977
|
+
"Emoji should be preserved in error"
|
|
1978
|
+
)
|
|
1979
|
+
|
|
1980
|
+
# ── 10 000-char message not truncated ─────────────────────────────────
|
|
1981
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
1982
|
+
log_error("e" * 10000)
|
|
1983
|
+
assert len(mock_handle.call_args[0][0].getMessage()) == 10000, (
|
|
1984
|
+
"10k char message should not be truncated"
|
|
1985
|
+
)
|
|
1986
|
+
|
|
1987
|
+
# ── dual-write: both main (DEBUG) and errors-only (ERROR) handlers present
|
|
1988
|
+
assert logger is not None # Initialized at import time
|
|
1989
|
+
assert len(logger.handlers) >= 2, (
|
|
1990
|
+
f"Expected at least 2 handlers, got {len(logger.handlers)}"
|
|
1991
|
+
)
|
|
1992
|
+
handler_levels = sorted(h.level for h in logger.handlers)
|
|
1993
|
+
assert logging.DEBUG in handler_levels, (
|
|
1994
|
+
"Missing main handler at DEBUG level"
|
|
1995
|
+
)
|
|
1996
|
+
assert logging.ERROR in handler_levels, (
|
|
1997
|
+
"Missing errors-only handler at ERROR level"
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
# ── info/warn do not reach errors-only handler ────────────────────────
|
|
2001
|
+
errors_handler = next(
|
|
2002
|
+
h for h in logger.handlers if h.level == logging.ERROR
|
|
2003
|
+
)
|
|
2004
|
+
with patch.object(errors_handler, "emit") as mock_emit:
|
|
2005
|
+
log_info("should not reach errors handler")
|
|
2006
|
+
log_warn("should not reach errors handler either")
|
|
2007
|
+
mock_emit.assert_not_called()
|
|
2008
|
+
|
|
2009
|
+
# ── dual-source: logfile_name, module_name and source_file all present ───
|
|
2010
|
+
with patch.object(logger, "handle") as mock_handle:
|
|
2011
|
+
log_error(
|
|
2012
|
+
"error message", logfile_name="LOGS", module_name="grouped_name"
|
|
2013
|
+
)
|
|
2014
|
+
record = mock_handle.call_args[0][0]
|
|
2015
|
+
assert hasattr(record, "logfile_name"), "Missing logfile_name attr"
|
|
2016
|
+
assert hasattr(record, "module_name"), "Missing module_name attr"
|
|
2017
|
+
assert hasattr(record, "source_file"), "Missing source_file attr"
|
|
2018
|
+
assert record.logfile_name == "LOGS", (
|
|
2019
|
+
f"Expected 'LOGS', got {record.logfile_name!r}"
|
|
2020
|
+
)
|
|
2021
|
+
assert record.module_name == "grouped_name", (
|
|
2022
|
+
f"Expected 'grouped_name', got {record.module_name!r}"
|
|
2023
|
+
)
|
|
2024
|
+
assert record.source_file is not None, "source_file should not be None"
|
|
2025
|
+
assert len(record.source_file) > 0, "source_file should be non-empty"
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
def log_metric(
|
|
2029
|
+
metric_name: str,
|
|
2030
|
+
value: int | float,
|
|
2031
|
+
unit: str = "",
|
|
2032
|
+
logfile_name: str | None = None,
|
|
2033
|
+
module_name: str | None = None,
|
|
2034
|
+
**tags: Any,
|
|
2035
|
+
) -> None:
|
|
2036
|
+
"""[TIER 1] Emit a structured numeric metric to a separate .metrics.jsonl file.
|
|
2037
|
+
|
|
2038
|
+
WHY: Metrics and audit logs have different retention and forwarding needs;
|
|
2039
|
+
a separate file lets monitoring tools ingest metrics independently.
|
|
2040
|
+
"""
|
|
2041
|
+
if logfile_name is None:
|
|
2042
|
+
logfile_name = os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
2043
|
+
if module_name is None:
|
|
2044
|
+
module_name = _get_caller_module()
|
|
2045
|
+
source_file = _get_actual_source_file()
|
|
2046
|
+
_warn_non_primitive_fields(tags, module_name)
|
|
2047
|
+
message = f"{metric_name}={value}{unit}"
|
|
2048
|
+
_metrics_logger.log(
|
|
2049
|
+
METRIC_LEVEL,
|
|
2050
|
+
message,
|
|
2051
|
+
extra={
|
|
2052
|
+
"logfile_name": logfile_name,
|
|
2053
|
+
"module_name": module_name,
|
|
2054
|
+
"source_file": source_file,
|
|
2055
|
+
"metric_name": metric_name,
|
|
2056
|
+
"value": value,
|
|
2057
|
+
"unit": unit,
|
|
2058
|
+
**tags,
|
|
2059
|
+
},
|
|
2060
|
+
)
|
|
2061
|
+
|
|
2062
|
+
|
|
2063
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
2064
|
+
if pytest is not None:
|
|
2065
|
+
|
|
2066
|
+
class TestLogMetric:
|
|
2067
|
+
"""WHY: log_metric emits to a separate .metrics.jsonl file at METR level — if it
|
|
2068
|
+
bleeds into audit log or drops structured fields, monitoring tools receive corrupt data."""
|
|
2069
|
+
|
|
2070
|
+
def test_log_metric(self) -> None:
|
|
2071
|
+
assert pytest is not None # Type guard for pyright
|
|
2072
|
+
# ── float value → METR level and correct message ─────────────────────
|
|
2073
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2074
|
+
log_metric(
|
|
2075
|
+
"api_latency_ms", 142.5, unit="ms", module_name="test_module"
|
|
2076
|
+
)
|
|
2077
|
+
record = mock_handle.call_args[0][0]
|
|
2078
|
+
assert record.levelno == METRIC_LEVEL, (
|
|
2079
|
+
f"Expected METRIC_LEVEL ({METRIC_LEVEL}), got {record.levelno}"
|
|
2080
|
+
)
|
|
2081
|
+
assert record.levelname == "METR", (
|
|
2082
|
+
f"Expected 'METR' levelname, got {record.levelname!r}"
|
|
2083
|
+
)
|
|
2084
|
+
assert record.getMessage() == "api_latency_ms=142.5ms", (
|
|
2085
|
+
f"Expected 'api_latency_ms=142.5ms', got {record.getMessage()!r}"
|
|
2086
|
+
)
|
|
2087
|
+
|
|
2088
|
+
# ── integer value stored as int (no coercion to float) ────────────────
|
|
2089
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2090
|
+
log_metric("queue_depth", 83, module_name="test_module")
|
|
2091
|
+
assert mock_handle.call_args[0][0].value == 83, (
|
|
2092
|
+
f"Expected int 83, got {mock_handle.call_args[0][0].value!r}"
|
|
2093
|
+
)
|
|
2094
|
+
|
|
2095
|
+
# ── tags attached as queryable structured fields ───────────────────────
|
|
2096
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2097
|
+
log_metric(
|
|
2098
|
+
"cache_hits",
|
|
2099
|
+
500,
|
|
2100
|
+
module_name="test_module",
|
|
2101
|
+
region="us-east-1",
|
|
2102
|
+
env="prod",
|
|
2103
|
+
)
|
|
2104
|
+
record = mock_handle.call_args[0][0]
|
|
2105
|
+
assert record.region == "us-east-1", (
|
|
2106
|
+
f"Expected region='us-east-1', got {record.region!r}"
|
|
2107
|
+
)
|
|
2108
|
+
assert record.env == "prod", f"Expected env='prod', got {record.env!r}"
|
|
2109
|
+
|
|
2110
|
+
# ── no module_name → auto-detects non-empty caller name ─────────────────
|
|
2111
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2112
|
+
log_metric("error_rate", 0.02)
|
|
2113
|
+
assert len(mock_handle.call_args[0][0].module_name) > 0, (
|
|
2114
|
+
"module_name should be auto-detected and non-empty"
|
|
2115
|
+
)
|
|
2116
|
+
|
|
2117
|
+
# ── missing unit defaults to "" not None ──────────────────────────────
|
|
2118
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2119
|
+
log_metric("active_sessions", 1200, module_name="test_module")
|
|
2120
|
+
assert mock_handle.call_args[0][0].unit == "", (
|
|
2121
|
+
f"Expected empty string for unit, got {mock_handle.call_args[0][0].unit!r}"
|
|
2122
|
+
)
|
|
2123
|
+
|
|
2124
|
+
# ── zero value not filtered out ───────────────────────────────────────
|
|
2125
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2126
|
+
log_metric("failed_requests", 0, module_name="test_module")
|
|
2127
|
+
assert mock_handle.call_args[0][0].value == 0, (
|
|
2128
|
+
"Zero value should not be filtered out"
|
|
2129
|
+
)
|
|
2130
|
+
|
|
2131
|
+
# ── negative value accepted ───────────────────────────────────────────
|
|
2132
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2133
|
+
log_metric("delta_users", -42, module_name="test_module")
|
|
2134
|
+
assert mock_handle.call_args[0][0].value == -42, (
|
|
2135
|
+
f"Expected -42, got {mock_handle.call_args[0][0].value!r}"
|
|
2136
|
+
)
|
|
2137
|
+
|
|
2138
|
+
# ── metric_name stored as structured field, not only in message ────────
|
|
2139
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2140
|
+
log_metric("db_query_time_ms", 55.3, module_name="test_module")
|
|
2141
|
+
assert mock_handle.call_args[0][0].metric_name == "db_query_time_ms", (
|
|
2142
|
+
f"Expected 'db_query_time_ms', got {mock_handle.call_args[0][0].metric_name!r}"
|
|
2143
|
+
)
|
|
2144
|
+
|
|
2145
|
+
# ── does not fire audit logger ────────────────────────────────────────
|
|
2146
|
+
with (
|
|
2147
|
+
patch.object(logger, "handle") as audit_mock,
|
|
2148
|
+
patch.object(_metrics_logger, "handle"),
|
|
2149
|
+
):
|
|
2150
|
+
log_metric("some_metric", 1.0, module_name="test_module")
|
|
2151
|
+
audit_mock.assert_not_called(), ("Metric should not fire audit logger")
|
|
2152
|
+
|
|
2153
|
+
# ── dual-source: module_name and source_file both present ───────────────
|
|
2154
|
+
with patch.object(_metrics_logger, "handle") as mock_handle:
|
|
2155
|
+
log_metric("test_metric", 42, module_name="grouped_name")
|
|
2156
|
+
record = mock_handle.call_args[0][0]
|
|
2157
|
+
assert hasattr(record, "module_name"), "Missing module_name attr"
|
|
2158
|
+
assert hasattr(record, "source_file"), "Missing source_file attr"
|
|
2159
|
+
assert record.module_name == "grouped_name", (
|
|
2160
|
+
f"Expected 'grouped_name', got {record.module_name!r}"
|
|
2161
|
+
)
|
|
2162
|
+
assert record.source_file is not None, "source_file should not be None"
|
|
2163
|
+
assert len(record.source_file) > 0, "source_file should be non-empty"
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
# ── SECTION 11 · NOTIFICATION API ────────────────────────────────────────────
|
|
2167
|
+
# Goal: Write structured JSONL notifications to main_logger.jsonl
|
|
2168
|
+
|
|
2169
|
+
|
|
2170
|
+
def send_notification(
|
|
2171
|
+
message: str,
|
|
2172
|
+
logfile_name: str | None = None,
|
|
2173
|
+
module_name: str | None = None,
|
|
2174
|
+
source_file: str | None = None,
|
|
2175
|
+
) -> None:
|
|
2176
|
+
"""[TIER 1] Synchronously emit a structured notification to main_logger.jsonl.
|
|
2177
|
+
|
|
2178
|
+
WHY: Notifications are cross-module operational signals that belong in one
|
|
2179
|
+
shared JSONL file regardless of which module calls them.
|
|
2180
|
+
"""
|
|
2181
|
+
if logfile_name is None:
|
|
2182
|
+
logfile_name = os.getenv("LOGGER_FILE_NAME", _DEFAULT_LOGFILE)
|
|
2183
|
+
if module_name is None:
|
|
2184
|
+
module_name = _get_caller_module()
|
|
2185
|
+
resolved_source_file = (
|
|
2186
|
+
source_file if source_file is not None else _get_actual_source_file()
|
|
2187
|
+
)
|
|
2188
|
+
|
|
2189
|
+
_notifications_logger.info(
|
|
2190
|
+
message,
|
|
2191
|
+
extra={
|
|
2192
|
+
"logfile_name": logfile_name,
|
|
2193
|
+
"module_name": module_name,
|
|
2194
|
+
"source_file": resolved_source_file,
|
|
2195
|
+
},
|
|
2196
|
+
)
|
|
2197
|
+
|
|
2198
|
+
|
|
2199
|
+
async def send_notification_async(
|
|
2200
|
+
message: str, logfile_name: str | None = None, module_name: str | None = None
|
|
2201
|
+
) -> None:
|
|
2202
|
+
"""[TIER 1] Asynchronously emit a structured notification to main_logger.jsonl.
|
|
2203
|
+
|
|
2204
|
+
WHY: Async callers must not block the event loop. run_in_executor() offloads
|
|
2205
|
+
the call to a thread pool thread.
|
|
2206
|
+
"""
|
|
2207
|
+
source_file = _get_actual_source_file()
|
|
2208
|
+
loop = asyncio.get_event_loop()
|
|
2209
|
+
await loop.run_in_executor(
|
|
2210
|
+
None, send_notification, message, logfile_name, module_name, source_file
|
|
2211
|
+
)
|
|
2212
|
+
|
|
2213
|
+
|
|
2214
|
+
# ── TESTS ───────────────────────────────────────────────────────────────────
|
|
2215
|
+
if pytest is not None:
|
|
2216
|
+
|
|
2217
|
+
class TestSendNotification:
|
|
2218
|
+
"""WHY: send_notification is the cross-module operational signal path —
|
|
2219
|
+
routing to the wrong logger or losing source_file corrupts downstream tools."""
|
|
2220
|
+
|
|
2221
|
+
def test_send_notification(self) -> None:
|
|
2222
|
+
assert pytest is not None # Type guard for pyright
|
|
2223
|
+
import asyncio
|
|
2224
|
+
import inspect as _inspect
|
|
2225
|
+
|
|
2226
|
+
# ── fires _notifications_logger at INFO level ─────────────────────────
|
|
2227
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2228
|
+
send_notification("Deployment complete")
|
|
2229
|
+
mock_handle.assert_called_once()
|
|
2230
|
+
assert mock_handle.call_args[0][0].levelno == logging.INFO, (
|
|
2231
|
+
f"Expected INFO level, got {mock_handle.call_args[0][0].levelno}"
|
|
2232
|
+
)
|
|
2233
|
+
|
|
2234
|
+
# ── message text preserved on record ─────────────────────────────────
|
|
2235
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2236
|
+
send_notification("Deployment complete")
|
|
2237
|
+
assert (
|
|
2238
|
+
mock_handle.call_args[0][0].getMessage() == "Deployment complete"
|
|
2239
|
+
), "Message text should be preserved"
|
|
2240
|
+
|
|
2241
|
+
# ── logfile_name/module_name present when not provided ─────────────────
|
|
2242
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2243
|
+
send_notification("any message")
|
|
2244
|
+
record = mock_handle.call_args[0][0]
|
|
2245
|
+
assert hasattr(record, "logfile_name"), "Missing logfile_name attr"
|
|
2246
|
+
assert hasattr(record, "module_name"), "Missing module_name attr"
|
|
2247
|
+
assert record.module_name is not None, "module_name should not be None"
|
|
2248
|
+
assert len(record.module_name) > 0, "module_name should be non-empty"
|
|
2249
|
+
|
|
2250
|
+
# ── source_file reflects the actual calling module ────────────────────
|
|
2251
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2252
|
+
send_notification("any message")
|
|
2253
|
+
record = mock_handle.call_args[0][0]
|
|
2254
|
+
assert hasattr(record, "source_file"), "Missing source_file attr"
|
|
2255
|
+
assert len(record.source_file) > 0, "source_file should be non-empty"
|
|
2256
|
+
assert not hasattr(record, "source"), (
|
|
2257
|
+
"redundant 'source' field must be absent"
|
|
2258
|
+
)
|
|
2259
|
+
|
|
2260
|
+
# ── empty message writes without error ────────────────────────────────
|
|
2261
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2262
|
+
send_notification("")
|
|
2263
|
+
mock_handle.assert_called_once()
|
|
2264
|
+
assert mock_handle.call_args[0][0].getMessage() == "", (
|
|
2265
|
+
"Empty message should write without error"
|
|
2266
|
+
)
|
|
2267
|
+
|
|
2268
|
+
# ── emoji and unicode preserved intact ───────────────────────────────
|
|
2269
|
+
with patch.object(_notifications_logger, "handle") as mock_handle:
|
|
2270
|
+
send_notification("🚀 Deploy: región=us-east-1")
|
|
2271
|
+
msg = mock_handle.call_args[0][0].getMessage()
|
|
2272
|
+
assert "🚀" in msg, "Emoji should be preserved"
|
|
2273
|
+
assert "región" in msg, "Unicode should be preserved"
|
|
2274
|
+
|
|
2275
|
+
# ── does not fire audit logger ────────────────────────────────────────
|
|
2276
|
+
with (
|
|
2277
|
+
patch.object(logger, "handle") as audit_mock,
|
|
2278
|
+
patch.object(_notifications_logger, "handle"),
|
|
2279
|
+
):
|
|
2280
|
+
send_notification("should not reach audit logger")
|
|
2281
|
+
(
|
|
2282
|
+
audit_mock.assert_not_called(),
|
|
2283
|
+
("Notification should not fire audit logger"),
|
|
2284
|
+
)
|
|
2285
|
+
|
|
2286
|
+
# ── async variant delegates to sync variant with captured source_file ──
|
|
2287
|
+
calls: list[tuple[str, str | None, str | None]] = []
|
|
2288
|
+
with patch(
|
|
2289
|
+
"JSONL_LOGGER.send_notification",
|
|
2290
|
+
side_effect=lambda m, logfile_name=None, module_name=None, source_file=None: (
|
|
2291
|
+
calls.append((m, logfile_name, module_name))
|
|
2292
|
+
),
|
|
2293
|
+
):
|
|
2294
|
+
asyncio.run(send_notification_async("async alert"))
|
|
2295
|
+
assert len(calls) == 1, f"Expected 1 call, got {len(calls)}"
|
|
2296
|
+
assert calls[0][0] == "async alert", (
|
|
2297
|
+
f"Expected 'async alert', got {calls[0][0]!r}"
|
|
2298
|
+
)
|
|
2299
|
+
|
|
2300
|
+
# ── async variant is a coroutine and completes without blocking ────────
|
|
2301
|
+
assert _inspect.iscoroutinefunction(send_notification_async), (
|
|
2302
|
+
"send_notification_async should be a coroutine function"
|
|
2303
|
+
)
|
|
2304
|
+
with patch("JSONL_LOGGER.send_notification"):
|
|
2305
|
+
asyncio.run(send_notification_async("non-blocking check"))
|
|
2306
|
+
|
|
2307
|
+
|
|
2308
|
+
# ── SECTION 12 · PERFORMANCE TEST ─────────────────────────────────────────────
|
|
2309
|
+
# Goal: Provide live performance benchmarks for all public API operations
|
|
2310
|
+
|
|
2311
|
+
|
|
2312
|
+
def run_performance_test(
|
|
2313
|
+
single_thread_count: int = 5000,
|
|
2314
|
+
multi_thread_count: int = 10000,
|
|
2315
|
+
num_threads: int = 10,
|
|
2316
|
+
show_logs: bool = True,
|
|
2317
|
+
) -> dict:
|
|
2318
|
+
"""Run live performance benchmarks for JSONL_LOGGER on current hardware."""
|
|
2319
|
+
import threading
|
|
2320
|
+
import time
|
|
2321
|
+
from datetime import datetime
|
|
2322
|
+
import JSONL_LOGGER as mod
|
|
2323
|
+
|
|
2324
|
+
original_console = mod.CONSOLE_LOGGING_ENABLED
|
|
2325
|
+
dropped_logs = 0
|
|
2326
|
+
|
|
2327
|
+
try:
|
|
2328
|
+
if not show_logs:
|
|
2329
|
+
mod.CONSOLE_LOGGING_ENABLED = False
|
|
2330
|
+
assert logger is not None # Initialized at import time
|
|
2331
|
+
for handler in logger.handlers[:]:
|
|
2332
|
+
if isinstance(handler, logging.StreamHandler) and not hasattr(
|
|
2333
|
+
handler, "queue"
|
|
2334
|
+
):
|
|
2335
|
+
logger.removeHandler(handler)
|
|
2336
|
+
|
|
2337
|
+
print("\n" + "=" * 70)
|
|
2338
|
+
print("JSONL_LOGGER Live Performance Test")
|
|
2339
|
+
print(f"Started: {datetime.now().isoformat()}")
|
|
2340
|
+
print("=" * 70)
|
|
2341
|
+
|
|
2342
|
+
results = {}
|
|
2343
|
+
|
|
2344
|
+
if mod._log_queue is None or mod._writer_thread is None:
|
|
2345
|
+
print("⚠️ Logger not initialized, re-initializing...")
|
|
2346
|
+
mod._init_logger()
|
|
2347
|
+
|
|
2348
|
+
# ── Single-Thread Test ──────────────────────────────────────────────
|
|
2349
|
+
print(f"\n📊 Single-Thread Test ({single_thread_count:,} logs):")
|
|
2350
|
+
print("-" * 40)
|
|
2351
|
+
|
|
2352
|
+
start = time.time()
|
|
2353
|
+
for i in range(single_thread_count):
|
|
2354
|
+
try:
|
|
2355
|
+
log_info(f"perf_test_single_{i}", test_type="single", index=i)
|
|
2356
|
+
except Exception as e:
|
|
2357
|
+
dropped_logs += 1
|
|
2358
|
+
print(f"⚠️ Log dropped in single-thread test: {e}")
|
|
2359
|
+
|
|
2360
|
+
elapsed = time.time() - start
|
|
2361
|
+
_flush_logs()
|
|
2362
|
+
|
|
2363
|
+
throughput = single_thread_count / elapsed
|
|
2364
|
+
results["single_thread"] = {
|
|
2365
|
+
"logs": single_thread_count,
|
|
2366
|
+
"time_seconds": round(elapsed, 2),
|
|
2367
|
+
"throughput": round(throughput, 0),
|
|
2368
|
+
"target": 10000,
|
|
2369
|
+
"dropped_logs": dropped_logs,
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
print(f" Logs: {single_thread_count:,}")
|
|
2373
|
+
print(f" Time: {elapsed:.2f} seconds")
|
|
2374
|
+
print(f" Throughput: {throughput:.0f} logs/sec")
|
|
2375
|
+
print(f" Dropped: {dropped_logs}")
|
|
2376
|
+
print(" Target: ≥8,000 logs/sec → ", end="")
|
|
2377
|
+
print("✅ PASS" if throughput >= 8000 else "⚠️ Below target")
|
|
2378
|
+
|
|
2379
|
+
# ── Multi-Thread Test ───────────────────────────────────────────────
|
|
2380
|
+
total_logs = num_threads * multi_thread_count
|
|
2381
|
+
print(
|
|
2382
|
+
f"\n📊 Multi-Thread Test ({num_threads} threads × {multi_thread_count:,} logs = {total_logs:,} total):"
|
|
2383
|
+
)
|
|
2384
|
+
print("-" * 40)
|
|
2385
|
+
|
|
2386
|
+
dropped_logs = 0
|
|
2387
|
+
thread_errors = []
|
|
2388
|
+
|
|
2389
|
+
def worker(worker_id: int, count: int) -> None:
|
|
2390
|
+
nonlocal dropped_logs
|
|
2391
|
+
for i in range(count):
|
|
2392
|
+
try:
|
|
2393
|
+
log_info(
|
|
2394
|
+
f"perf_test_multi_{worker_id}_{i}",
|
|
2395
|
+
test_type="multi",
|
|
2396
|
+
worker_id=worker_id,
|
|
2397
|
+
index=i,
|
|
2398
|
+
)
|
|
2399
|
+
except Exception as e:
|
|
2400
|
+
dropped_logs += 1
|
|
2401
|
+
if len(thread_errors) < 10:
|
|
2402
|
+
thread_errors.append(f"Thread {worker_id}: {e}")
|
|
2403
|
+
|
|
2404
|
+
threads = []
|
|
2405
|
+
start = time.time()
|
|
2406
|
+
for i in range(num_threads):
|
|
2407
|
+
t = threading.Thread(target=worker, args=(i, multi_thread_count))
|
|
2408
|
+
threads.append(t)
|
|
2409
|
+
t.start()
|
|
2410
|
+
|
|
2411
|
+
for t in threads:
|
|
2412
|
+
t.join()
|
|
2413
|
+
|
|
2414
|
+
elapsed_multi = time.time() - start
|
|
2415
|
+
_flush_logs()
|
|
2416
|
+
elapsed_multi = time.time() - start
|
|
2417
|
+
throughput_multi = total_logs / elapsed_multi
|
|
2418
|
+
|
|
2419
|
+
results["multi_thread"] = {
|
|
2420
|
+
"threads": num_threads,
|
|
2421
|
+
"logs_per_thread": multi_thread_count,
|
|
2422
|
+
"total_logs": total_logs,
|
|
2423
|
+
"time_seconds": round(elapsed_multi, 2),
|
|
2424
|
+
"throughput": round(throughput_multi, 0),
|
|
2425
|
+
"target": 5000,
|
|
2426
|
+
"dropped_logs": dropped_logs,
|
|
2427
|
+
"thread_errors": thread_errors[:5],
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
print(f" Total logs: {total_logs:,}")
|
|
2431
|
+
print(f" Time: {elapsed_multi:.2f} seconds")
|
|
2432
|
+
print(f" Throughput: {throughput_multi:.0f} logs/sec")
|
|
2433
|
+
print(f" Dropped: {dropped_logs}")
|
|
2434
|
+
if thread_errors:
|
|
2435
|
+
print(f" Thread errors: {len(thread_errors)} occurrences")
|
|
2436
|
+
print(" Target: ≥5,000 logs/sec → ", end="")
|
|
2437
|
+
print("✅ PASS" if throughput_multi >= 5000 else "⚠️ Below target")
|
|
2438
|
+
|
|
2439
|
+
# ── Queue Health Check ──────────────────────────────────────────────
|
|
2440
|
+
if mod._log_queue:
|
|
2441
|
+
queue_size = mod._log_queue.qsize()
|
|
2442
|
+
max_size = getattr(mod._log_queue, "maxsize", 0)
|
|
2443
|
+
if queue_size > 0:
|
|
2444
|
+
print(
|
|
2445
|
+
f"\n⚠️ Queue still has {queue_size} pending entries (max: {max_size})"
|
|
2446
|
+
)
|
|
2447
|
+
_flush_logs()
|
|
2448
|
+
|
|
2449
|
+
# ── Summary ─────────────────────────────────────────────────────────
|
|
2450
|
+
print("\n" + "=" * 70)
|
|
2451
|
+
print("✅ Performance Test Complete")
|
|
2452
|
+
print("=" * 70)
|
|
2453
|
+
|
|
2454
|
+
return results
|
|
2455
|
+
|
|
2456
|
+
except Exception as e:
|
|
2457
|
+
print(f"\n❌ Performance test failed: {e}")
|
|
2458
|
+
import traceback
|
|
2459
|
+
|
|
2460
|
+
traceback.print_exc()
|
|
2461
|
+
return {"error": str(e)}
|
|
2462
|
+
|
|
2463
|
+
finally:
|
|
2464
|
+
mod.CONSOLE_LOGGING_ENABLED = original_console
|
|
2465
|
+
try:
|
|
2466
|
+
_flush_logs()
|
|
2467
|
+
except Exception:
|
|
2468
|
+
pass # WHY: best-effort flush on error; should not mask original exception
|
|
2469
|
+
|
|
2470
|
+
|
|
2471
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
2472
|
+
# TEST COVERAGE MATRIX
|
|
2473
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
2474
|
+
#
|
|
2475
|
+
# Tier 1 = Public API → must be exhaustive; called by application code
|
|
2476
|
+
# Tier 2 = Internal Logic → correctness + resilience; helpers & infrastructure
|
|
2477
|
+
# Tier 3 = Dev/Debug → lower priority; dev-time utilities
|
|
2478
|
+
#
|
|
2479
|
+
# Legend: ✅ covered ⚠️ partial ❌ not covered
|
|
2480
|
+
#
|
|
2481
|
+
# ┌─────────────────────────────────────┬──────┬───────────┬────────┬────────────────────────────────────────────────────────────────────────┐
|
|
2482
|
+
# │ Function / Component │ Tier │ Test Class│ Tests │ What is tested │
|
|
2483
|
+
# ├─────────────────────────────────────┼──────┼───────────┼────────┼────────────────────────────────────────────────────────────────────────┤
|
|
2484
|
+
# │ log_info() │ 1 │TestLogInfo│ 6 ✅ │ level, message, auto-detect, empty, special chars, 10k msg, dual-source│
|
|
2485
|
+
# │ log_warn() │ 1 │TestLogWarn│ 6 ✅ │ level, message, auto-detect, empty, special chars, 10k msg, dual-source│
|
|
2486
|
+
# │ log_error() │ 1 │TestLogErr │ 8 ✅ │ level, message, auto-detect, empty, special chars, 10k msg, │
|
|
2487
|
+
# │ │ │ │ dual-write handlers, info/warn isolation, dual-source │
|
|
2488
|
+
# │ log_metric() │ 1 │TestLogMet │ 10 ✅ │ METR level, float/int value, tags, auto-detect, unit default, │
|
|
2489
|
+
# │ │ │ │ zero value, negative value, metric_name field, audit isolation, │
|
|
2490
|
+
# │ │ │ │ dual-source │
|
|
2491
|
+
# │ send_notification() │ 1 │TestSendNot│ 10 ✅ │ INFO level, message, module_name routing, source_file, empty msg, │
|
|
2492
|
+
# │ │ │ │ special chars, audit isolation, source field absent │
|
|
2493
|
+
# │ send_notification_async() │ 1 │TestSendNotAsync| 3 ✅ │ delegates to sync, coroutine contract, non-blocking │
|
|
2494
|
+
# ├─────────────────────────────────────┼──────┼───────────┼────────┼────────────────────────────────────────────────────────────────────────┤
|
|
2495
|
+
# │ _get_logfile_name() │ 2 │TestGetLog │ 3 ✅ │ LOGGER_FILE_NAME priority, filename fallback, exception safety │
|
|
2496
|
+
# │ _get_actual_source_file() │ 2 │TestGetAct │ 2 ✅ │ bypasses LOGGER_FILE_NAME, exception safety │
|
|
2497
|
+
# │ _get_log_path() │ 2 │TestGetLogP│ 9 ✅ │ suffix routing, ValueError on missing dirs, path construction │
|
|
2498
|
+
# │ _with_file_retry() │ 2 │TestWithRet│ 5 ✅ │ success on attempt 1, success on attempt 3, exhaustion sentinel, │
|
|
2499
|
+
# │ │ │ │ non-IOError retried, exponential backoff │
|
|
2500
|
+
# │ _flush_buffer() │ 2 │TestFlushBuf│ 5 ✅ │ re-buffer on retry exhaustion, buffer cap drop, retry logic │
|
|
2501
|
+
# │ QueueHandler.emit() │ 2 │TestQHEmit │ 3 ✅ │ queue.put_nowait called with (path, formatted_msg) │
|
|
2502
|
+
# │ _flush_logs() │ 2 │TestFlushLog│ 2 ✅ │ mock _writer_thread.join and assert _shutdown=True │
|
|
2503
|
+
# ├─────────────────────────────────────┼──────┼───────────┼────────┼────────────────────────────────────────────────────────────────────────┤
|
|
2504
|
+
# │ _get_timestamp() │ 3 │TestGetTime│ 4 ✅ │ ISO-8601 format with Z suffix and ms precision │
|
|
2505
|
+
# │ _debug_print() │ 3 │TestDbgPrnt│ 2 ✅ │ stderr print guard, DEBUG_PRINT=False in prod │
|
|
2506
|
+
# │ _warn_non_primitive_fields() │ 3 │TestWarnNP │ 5 ✅ │ stderr output for list/dict/datetime values │
|
|
2507
|
+
# │ ColoredFormatter.format() │ 3 │TestColorFM│ 9 ✅ │ emoji present, ANSI codes in output, console formatting │
|
|
2508
|
+
# │ UniformLevelFormatter.format() │ 3 │TestUnifFM │ 10 ✅ │ exact JSONL keys, compact separators, JSONL shape │
|
|
2509
|
+
# ├─────────────────────────────────────┼──────┼───────────┼────────┼────────────────────────────────────────────────────────────────────────┤
|
|
2510
|
+
# │ TOTALS │ │ │ │
|
|
2511
|
+
# │ Tier 1 — Public API │ 8 functions │ 57 ✅ │ All public functions fully covered │
|
|
2512
|
+
# │ Tier 2 — Internal Logic │ 10 functions │ 29 ✅ │ 7 helpers unit-tested; 3 covered via integration or import-time verify │
|
|
2513
|
+
# │ Tier 3 — Dev/Debug │ 5 components │ 30 ✅ │ All low-risk components now covered │
|
|
2514
|
+
# └─────────────────────────────────────┴──────┴───────────┴────────┴────────────────────────────────────────────────────────────────────────┘
|
|
2515
|
+
|
|
2516
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
2517
|
+
# USAGE GUIDE
|
|
2518
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
2519
|
+
#
|
|
2520
|
+
# Quick Start:
|
|
2521
|
+
# from JSONL_LOGGER import log_info, log_warn, log_error, log_metric
|
|
2522
|
+
# from JSONL_LOGGER import send_notification, send_notification_async
|
|
2523
|
+
#
|
|
2524
|
+
# log_info("User logged in", user_id=123)
|
|
2525
|
+
# log_warn("Rate limit approaching", remaining=10)
|
|
2526
|
+
# log_error("Payment failed", error_code=500)
|
|
2527
|
+
# log_metric("api_latency_ms", 142.5, unit="ms", endpoint="/checkout")
|
|
2528
|
+
# send_notification("Deployment complete — v2.3.1 live")
|
|
2529
|
+
# await send_notification_async("Async worker finished batch")
|
|
2530
|
+
#
|
|
2531
|
+
# Environment Variables (.env):
|
|
2532
|
+
# PROJECT_DIRECTORY=/path/to/logs # Required: NO space before =
|
|
2533
|
+
# LOGS_LOCAL_TIMEZONE=Asia/Kolkata # Required: local timezone
|
|
2534
|
+
# LOGGER_FILE_NAME=LOGS # Optional: log file name (default: LOGS)
|
|
2535
|
+
# CONSOLE_LOGGING_ENABLED=false # Optional: console output (default: false)
|
|
2536
|
+
# LOGGER_REGISTER_SIGNALS=false # Optional: signal handlers (default: false)
|
|
2537
|
+
# LOGGER_DAEMON_THREAD=true # Optional: daemon thread (default: true)
|
|
2538
|
+
# LOGGER_MAX_BUFFER_SIZE=200000 # Optional: per-file buffer cap (default: 200000)
|
|
2539
|
+
#
|
|
2540
|
+
# Output Location:
|
|
2541
|
+
# {PROJECT_DIRECTORY}/_LOGS_DIRECTORY/{YYYY_MM_DD}/LOGS/{logfile_name}.jsonl ← all logs
|
|
2542
|
+
# {PROJECT_DIRECTORY}/_LOGS_DIRECTORY/{YYYY_MM_DD}/LOGS/{logfile_name}.errors.jsonl ← errors only
|
|
2543
|
+
# {PROJECT_DIRECTORY}/_LOGS_DIRECTORY/{YYYY_MM_DD}/LOGS/{logfile_name}.metrics.jsonl ← metrics
|
|
2544
|
+
#
|
|
2545
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2546
|
+
# GROUP MULTIPLE FILES UNDER ONE LOG FILE:
|
|
2547
|
+
#
|
|
2548
|
+
# # trading_symbols.py
|
|
2549
|
+
# LOGGER_FILE_NAME = "TRADING"
|
|
2550
|
+
# from JSONL_LOGGER import log_info
|
|
2551
|
+
# log_info("Order placed", symbol="AAPL")
|
|
2552
|
+
#
|
|
2553
|
+
# # historical_data.py
|
|
2554
|
+
# LOGGER_FILE_NAME = "TRADING"
|
|
2555
|
+
# from JSONL_LOGGER import log_info
|
|
2556
|
+
# log_info("Data fetched", records=1000)
|
|
2557
|
+
#
|
|
2558
|
+
# Both write to: {PROJECT_DIRECTORY}/_LOGS_DIRECTORY/{date}/LOGS/TRADING.jsonl
|
|
2559
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2560
|
+
# HIGH-THROUGHPUT CALLERS — bypass inspect overhead (~1-5µs saved per call):
|
|
2561
|
+
#
|
|
2562
|
+
# log_info("msg", logfile_name="MY_LOG", module_name="MY_MODULE")
|
|
2563
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2564
|
+
#
|
|
2565
|
+
# Run Tests:
|
|
2566
|
+
# python3 -m pytest JSONL_LOGGER.py -v
|
|
2567
|
+
#
|
|
2568
|
+
# Run Performance Test:
|
|
2569
|
+
# python3 JSONL_LOGGER.py
|
|
2570
|
+
#
|
|
2571
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
2572
|
+
|
|
2573
|
+
if __name__ == "__main__":
|
|
2574
|
+
import asyncio
|
|
2575
|
+
import tempfile
|
|
2576
|
+
|
|
2577
|
+
# Set a default for testing if not already set
|
|
2578
|
+
if not os.getenv("PROJECT_DIRECTORY"):
|
|
2579
|
+
os.environ["PROJECT_DIRECTORY"] = tempfile.mkdtemp()
|
|
2580
|
+
print(f"⚠️ Using temp directory for demo: {os.environ['PROJECT_DIRECTORY']}")
|
|
2581
|
+
print(" For production, set PROJECT_DIRECTORY in .env file\n")
|
|
2582
|
+
|
|
2583
|
+
print("=" * 60)
|
|
2584
|
+
print("Testing JSONL_LOGGER...")
|
|
2585
|
+
print("=" * 60)
|
|
2586
|
+
|
|
2587
|
+
# Regular logs
|
|
2588
|
+
print("\n📝 Basic Logging:")
|
|
2589
|
+
log_info("Info message")
|
|
2590
|
+
log_warn("Warning message")
|
|
2591
|
+
log_error("Error message")
|
|
2592
|
+
log_info("With extra fields", user_id=123, action="login", response_time_ms=150)
|
|
2593
|
+
|
|
2594
|
+
# Metrics
|
|
2595
|
+
print("\n📊 Metric Logging:")
|
|
2596
|
+
log_metric("api_latency_ms", 142.5, unit="ms", endpoint="/checkout", method="POST")
|
|
2597
|
+
log_metric("queue_depth", 83, service="payment_worker")
|
|
2598
|
+
log_metric("cache_hit_rate", 0.94, unit="%", region="us-east-1")
|
|
2599
|
+
|
|
2600
|
+
# Notifications
|
|
2601
|
+
print("\n🔔 Notifications:")
|
|
2602
|
+
send_notification("Deployment complete — v2.3.1 live")
|
|
2603
|
+
asyncio.run(send_notification_async("Async worker finished batch — v2.3.1"))
|
|
2604
|
+
|
|
2605
|
+
# Run full performance test
|
|
2606
|
+
print("\n" + "=" * 60)
|
|
2607
|
+
print("Running FULL Performance Test Suite")
|
|
2608
|
+
print("=" * 60)
|
|
2609
|
+
run_performance_test(
|
|
2610
|
+
single_thread_count=10000,
|
|
2611
|
+
multi_thread_count=10000,
|
|
2612
|
+
num_threads=4,
|
|
2613
|
+
show_logs=False,
|
|
2614
|
+
)
|
|
2615
|
+
|
|
2616
|
+
_flush_logs()
|
|
2617
|
+
|
|
2618
|
+
print("\n" + "=" * 60)
|
|
2619
|
+
print(
|
|
2620
|
+
f"✅ Done — check {os.environ['PROJECT_DIRECTORY']}/_LOGS_DIRECTORY/ for JSONL files"
|
|
2621
|
+
)
|
|
2622
|
+
print("=" * 60)
|