sibi-dst 2025.8.7__py3-none-any.whl → 2025.8.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sibi_dst/df_helper/_df_helper.py +105 -89
- sibi_dst/df_helper/_parquet_artifact.py +11 -10
- sibi_dst/df_helper/_parquet_reader.py +4 -0
- sibi_dst/df_helper/backends/parquet/_parquet_options.py +504 -214
- sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py +11 -10
- sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py +9 -8
- sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py +4 -76
- sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py +0 -104
- sibi_dst/utils/boilerplate/__init__.py +6 -0
- sibi_dst/utils/boilerplate/base_data_artifact.py +110 -0
- sibi_dst/utils/boilerplate/base_data_cube.py +79 -0
- sibi_dst/utils/data_wrapper.py +22 -263
- sibi_dst/utils/iceberg_saver.py +126 -0
- sibi_dst/utils/log_utils.py +108 -529
- sibi_dst/utils/parquet_saver.py +110 -9
- sibi_dst/utils/progress/__init__.py +5 -0
- sibi_dst/utils/progress/jobs.py +82 -0
- sibi_dst/utils/progress/sse_runner.py +82 -0
- sibi_dst/utils/storage_hive.py +38 -1
- sibi_dst/utils/update_planner.py +617 -116
- {sibi_dst-2025.8.7.dist-info → sibi_dst-2025.8.9.dist-info}/METADATA +3 -2
- {sibi_dst-2025.8.7.dist-info → sibi_dst-2025.8.9.dist-info}/RECORD +23 -16
- {sibi_dst-2025.8.7.dist-info → sibi_dst-2025.8.9.dist-info}/WHEEL +0 -0
sibi_dst/utils/log_utils.py
CHANGED
@@ -4,12 +4,12 @@ import logging
|
|
4
4
|
import os
|
5
5
|
import sys
|
6
6
|
import time
|
7
|
-
from contextlib import contextmanager, nullcontext
|
7
|
+
from contextlib import contextmanager, nullcontext, suppress
|
8
8
|
from logging import LoggerAdapter
|
9
9
|
from logging.handlers import RotatingFileHandler
|
10
|
-
from typing import Optional, Dict, Any
|
10
|
+
from typing import Optional, Dict, Any, Union
|
11
11
|
|
12
|
-
# OpenTelemetry (optional)
|
12
|
+
# --- OpenTelemetry (optional) ---
|
13
13
|
try:
|
14
14
|
from opentelemetry import trace
|
15
15
|
from opentelemetry._logs import set_logger_provider, get_logger_provider
|
@@ -20,25 +20,15 @@ try:
|
|
20
20
|
from opentelemetry.sdk.resources import Resource
|
21
21
|
from opentelemetry.sdk.trace import TracerProvider
|
22
22
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
23
|
-
from opentelemetry.trace import Tracer as OTelTracer
|
24
23
|
_OTEL_AVAILABLE = True
|
25
24
|
except Exception:
|
26
|
-
# OTel is optional; keep class working without it
|
27
|
-
LoggerProvider = TracerProvider = OTelTracer = object # type: ignore
|
28
|
-
LoggingHandler = object # type: ignore
|
29
25
|
_OTEL_AVAILABLE = False
|
30
26
|
|
31
27
|
|
32
28
|
class Logger:
|
33
29
|
"""
|
34
30
|
Process-safe logger with optional OpenTelemetry integration.
|
35
|
-
|
36
|
-
Backward-compatible surface:
|
37
|
-
- Logger.default_logger(...)
|
38
|
-
- .debug/.info/.warning/.error/.critical
|
39
|
-
- .set_level(level), .shutdown()
|
40
|
-
- .bind(**extra) -> LoggerAdapter, .bound(**extra) ctx manager
|
41
|
-
- .start_span(name, attributes=None), .trace_function(span_name=None)
|
31
|
+
Idempotent handler setup. No propagation to root.
|
42
32
|
"""
|
43
33
|
|
44
34
|
DEBUG = logging.DEBUG
|
@@ -47,8 +37,8 @@ class Logger:
|
|
47
37
|
ERROR = logging.ERROR
|
48
38
|
CRITICAL = logging.CRITICAL
|
49
39
|
|
50
|
-
#
|
51
|
-
|
40
|
+
# idempotency guards per process
|
41
|
+
_attached_keys: set[tuple[str, str]] = set() # (logger_name, sink_id)
|
52
42
|
_otel_initialized_names: set[str] = set()
|
53
43
|
|
54
44
|
def __init__(
|
@@ -56,7 +46,7 @@ class Logger:
|
|
56
46
|
log_dir: str,
|
57
47
|
logger_name: str,
|
58
48
|
log_file: str,
|
59
|
-
log_level: int = logging.
|
49
|
+
log_level: int = logging.INFO,
|
60
50
|
enable_otel: bool = False,
|
61
51
|
otel_service_name: Optional[str] = None,
|
62
52
|
otel_stream_name: Optional[str] = None,
|
@@ -69,38 +59,35 @@ class Logger:
|
|
69
59
|
self.log_level = log_level
|
70
60
|
|
71
61
|
self.enable_otel = bool(enable_otel and _OTEL_AVAILABLE)
|
72
|
-
self.otel_service_name = (otel_service_name or logger_name
|
62
|
+
self.otel_service_name = (otel_service_name or logger_name or "app").strip()
|
73
63
|
self.otel_stream_name = (otel_stream_name or "").strip() or None
|
74
64
|
self.otel_endpoint = otel_endpoint
|
75
65
|
self.otel_insecure = otel_insecure
|
76
66
|
|
77
|
-
self.logger_provider
|
78
|
-
self.tracer_provider
|
79
|
-
self.tracer
|
67
|
+
self.logger_provider = None
|
68
|
+
self.tracer_provider = None
|
69
|
+
self.tracer = None
|
80
70
|
|
81
|
-
self.
|
82
|
-
self.
|
83
|
-
self.
|
71
|
+
self._core: logging.Logger = logging.getLogger(self.logger_name)
|
72
|
+
self._core.setLevel(self.log_level)
|
73
|
+
self._core.propagate = False
|
84
74
|
|
85
|
-
# public handle (may be
|
86
|
-
self.logger: logging.Logger
|
75
|
+
# public handle (may be LoggerAdapter)
|
76
|
+
self.logger: Union[logging.Logger, LoggerAdapter] = self._core
|
87
77
|
|
88
|
-
self.
|
78
|
+
self._setup_handlers()
|
89
79
|
if self.enable_otel:
|
90
|
-
self.
|
80
|
+
self._setup_otel()
|
91
81
|
|
92
|
-
# expose adapter with default extras if OTel stream requested
|
93
82
|
if self.enable_otel and self.otel_stream_name:
|
94
|
-
|
83
|
+
attrs = {
|
95
84
|
"log_stream": self.otel_stream_name,
|
96
85
|
"log_service_name": self.otel_service_name,
|
97
86
|
"logger_name": self.logger_name,
|
98
87
|
}
|
99
|
-
self.logger = LoggerAdapter(self.
|
88
|
+
self.logger = LoggerAdapter(self._core, extra=attrs)
|
100
89
|
|
101
|
-
#
|
102
|
-
# Public API
|
103
|
-
# -------------------------
|
90
|
+
# ---------------- Public API ----------------
|
104
91
|
|
105
92
|
@classmethod
|
106
93
|
def default_logger(
|
@@ -116,14 +103,11 @@ class Logger:
|
|
116
103
|
otel_insecure: bool = False,
|
117
104
|
) -> "Logger":
|
118
105
|
try:
|
119
|
-
|
120
|
-
caller_name = frame.f_globals.get("__name__", "default_logger")
|
106
|
+
caller_name = sys._getframe(1).f_globals.get("__name__", "default_logger")
|
121
107
|
except Exception:
|
122
108
|
caller_name = "default_logger"
|
123
|
-
|
124
109
|
logger_name = logger_name or caller_name
|
125
110
|
log_file = log_file or logger_name
|
126
|
-
|
127
111
|
return cls(
|
128
112
|
log_dir=log_dir,
|
129
113
|
logger_name=logger_name,
|
@@ -136,55 +120,11 @@ class Logger:
|
|
136
120
|
otel_insecure=otel_insecure,
|
137
121
|
)
|
138
122
|
|
139
|
-
def
|
140
|
-
|
141
|
-
try:
|
142
|
-
if self.enable_otel:
|
143
|
-
if isinstance(self.logger_provider, LoggerProvider):
|
144
|
-
try:
|
145
|
-
self._core_logger.info("Flushing OpenTelemetry logs...")
|
146
|
-
self.logger_provider.force_flush()
|
147
|
-
except Exception:
|
148
|
-
pass
|
149
|
-
try:
|
150
|
-
self._core_logger.info("Shutting down OpenTelemetry logs...")
|
151
|
-
self.logger_provider.shutdown()
|
152
|
-
except Exception:
|
153
|
-
pass
|
123
|
+
def set_level(self, level: int) -> None:
|
124
|
+
self._core.setLevel(level)
|
154
125
|
|
155
|
-
|
156
|
-
|
157
|
-
self._core_logger.info("Flushing OpenTelemetry traces...")
|
158
|
-
self.tracer_provider.force_flush()
|
159
|
-
except Exception:
|
160
|
-
pass
|
161
|
-
try:
|
162
|
-
self._core_logger.info("Shutting down OpenTelemetry traces...")
|
163
|
-
self.tracer_provider.shutdown()
|
164
|
-
except Exception:
|
165
|
-
pass
|
166
|
-
finally:
|
167
|
-
# Close our handlers explicitly to release file descriptors
|
168
|
-
for h in list(self._core_logger.handlers):
|
169
|
-
try:
|
170
|
-
h.flush()
|
171
|
-
except Exception:
|
172
|
-
pass
|
173
|
-
try:
|
174
|
-
h.close()
|
175
|
-
except Exception:
|
176
|
-
pass
|
177
|
-
try:
|
178
|
-
self._core_logger.removeHandler(h)
|
179
|
-
except Exception:
|
180
|
-
pass
|
181
|
-
logging.shutdown()
|
182
|
-
|
183
|
-
def set_level(self, level: int):
|
184
|
-
self._core_logger.setLevel(level)
|
185
|
-
|
186
|
-
# passthrough convenience
|
187
|
-
def _log(self, level: int, msg: str, *args, **kwargs):
|
126
|
+
# passthrough
|
127
|
+
def _log(self, level: int, msg: str, *args, **kwargs) -> None:
|
188
128
|
extra = kwargs.pop("extra", None)
|
189
129
|
if extra is not None:
|
190
130
|
if isinstance(self.logger, LoggerAdapter):
|
@@ -195,517 +135,156 @@ class Logger:
|
|
195
135
|
else:
|
196
136
|
self.logger.log(level, msg, *args, **kwargs)
|
197
137
|
|
198
|
-
def debug(self, msg: str, *
|
199
|
-
def info(self, msg: str, *
|
200
|
-
def warning(self, msg: str, *
|
201
|
-
def error(self, msg: str, *
|
202
|
-
def critical(self, msg: str, *
|
138
|
+
def debug(self, msg: str, *a, **k): self._log(logging.DEBUG, msg, *a, **k)
|
139
|
+
def info(self, msg: str, *a, **k): self._log(logging.INFO, msg, *a, **k)
|
140
|
+
def warning(self, msg: str, *a, **k): self._log(logging.WARNING, msg, *a, **k)
|
141
|
+
def error(self, msg: str, *a, **k): self._log(logging.ERROR, msg, *a, **k)
|
142
|
+
def critical(self, msg: str, *a, **k): self._log(logging.CRITICAL, msg, *a, **k)
|
203
143
|
|
204
144
|
def bind(self, **extra: Any) -> LoggerAdapter:
|
205
145
|
if isinstance(self.logger, LoggerAdapter):
|
206
|
-
|
207
|
-
return LoggerAdapter(self.logger.logger, merged)
|
146
|
+
return LoggerAdapter(self.logger.logger, {**self.logger.extra, **extra})
|
208
147
|
return LoggerAdapter(self.logger, extra)
|
209
148
|
|
210
149
|
@contextmanager
|
211
150
|
def bound(self, **extra: Any):
|
212
|
-
|
213
|
-
yield adapter
|
151
|
+
yield self.bind(**extra)
|
214
152
|
|
215
153
|
def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
|
216
154
|
if not (self.enable_otel and _OTEL_AVAILABLE and self.tracer):
|
217
|
-
# keep API but no-op cleanly
|
218
|
-
self.warning("Tracing is disabled or not initialized. Cannot start span.")
|
219
155
|
return nullcontext()
|
220
|
-
|
221
156
|
cm = self.tracer.start_as_current_span(name)
|
222
|
-
|
223
157
|
class _SpanCtx:
|
224
158
|
def __enter__(_self):
|
225
159
|
span = cm.__enter__()
|
226
160
|
if attributes:
|
227
161
|
for k, v in attributes.items():
|
228
|
-
|
162
|
+
with suppress(Exception):
|
229
163
|
span.set_attribute(k, v)
|
230
|
-
except Exception:
|
231
|
-
pass
|
232
164
|
return span
|
233
|
-
|
234
|
-
|
235
|
-
return cm.__exit__(exc_type, exc, tb)
|
236
|
-
|
165
|
+
def __exit__(_self, et, ev, tb):
|
166
|
+
return cm.__exit__(et, ev, tb)
|
237
167
|
return _SpanCtx()
|
238
168
|
|
239
169
|
def trace_function(self, span_name: Optional[str] = None):
|
240
|
-
def
|
241
|
-
def wrapper(*
|
170
|
+
def deco(func):
|
171
|
+
def wrapper(*a, **k):
|
242
172
|
name = span_name or func.__name__
|
243
173
|
with self.start_span(name):
|
244
|
-
return func(*
|
174
|
+
return func(*a, **k)
|
245
175
|
return wrapper
|
246
|
-
return
|
176
|
+
return deco
|
247
177
|
|
248
|
-
|
249
|
-
|
250
|
-
|
178
|
+
def shutdown(self) -> None:
|
179
|
+
try:
|
180
|
+
if self.enable_otel and _OTEL_AVAILABLE:
|
181
|
+
if self.logger_provider:
|
182
|
+
with suppress(Exception):
|
183
|
+
self._core.info("Flushing OpenTelemetry logs...")
|
184
|
+
self.logger_provider.force_flush()
|
185
|
+
with suppress(Exception):
|
186
|
+
self._core.info("Shutting down OpenTelemetry logs...")
|
187
|
+
self.logger_provider.shutdown()
|
188
|
+
if self.tracer_provider:
|
189
|
+
with suppress(Exception):
|
190
|
+
self._core.info("Flushing OpenTelemetry traces...")
|
191
|
+
self.tracer_provider.force_flush()
|
192
|
+
with suppress(Exception):
|
193
|
+
self._core.info("Shutting down OpenTelemetry traces...")
|
194
|
+
self.tracer_provider.shutdown()
|
195
|
+
finally:
|
196
|
+
for h in list(self._core.handlers):
|
197
|
+
with suppress(Exception): h.flush()
|
198
|
+
with suppress(Exception): h.close()
|
199
|
+
with suppress(Exception): self._core.removeHandler(h)
|
200
|
+
logging.shutdown()
|
251
201
|
|
252
|
-
|
202
|
+
# ---------------- Internal ----------------
|
203
|
+
|
204
|
+
def _setup_handlers(self) -> None:
|
253
205
|
os.makedirs(self.log_dir, exist_ok=True)
|
254
206
|
calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
255
207
|
log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
|
256
|
-
|
208
|
+
file_key = (self.logger_name, os.path.abspath(log_file_path))
|
209
|
+
console_key = (self.logger_name, "__console__")
|
257
210
|
|
258
|
-
|
211
|
+
fmt = logging.Formatter(
|
259
212
|
"[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
|
260
213
|
datefmt="%Y-%m-%d %H:%M:%S",
|
261
214
|
)
|
262
|
-
|
263
|
-
|
264
|
-
# attach once per process for this logger/file combo
|
265
|
-
if key not in self._handler_keys_attached:
|
266
|
-
file_handler = RotatingFileHandler(
|
267
|
-
log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
|
268
|
-
)
|
269
|
-
file_handler.setFormatter(formatter)
|
270
|
-
self._core_logger.addHandler(file_handler)
|
215
|
+
fmt.converter = time.gmtime # UTC
|
271
216
|
|
272
|
-
|
273
|
-
|
274
|
-
|
217
|
+
if file_key not in self._attached_keys:
|
218
|
+
fh = RotatingFileHandler(log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True)
|
219
|
+
fh.setFormatter(fmt)
|
220
|
+
self._core.addHandler(fh)
|
221
|
+
self._attached_keys.add(file_key)
|
275
222
|
|
276
|
-
|
223
|
+
if console_key not in self._attached_keys:
|
224
|
+
ch = logging.StreamHandler(sys.stdout)
|
225
|
+
ch.setFormatter(fmt)
|
226
|
+
self._core.addHandler(ch)
|
227
|
+
self._attached_keys.add(console_key)
|
277
228
|
|
278
229
|
def _normalize_otlp_endpoint(self, ep: str) -> str:
|
279
230
|
if "://" not in ep:
|
280
231
|
ep = ("http://" if self.otel_insecure else "https://") + ep
|
281
232
|
return ep
|
282
233
|
|
283
|
-
def
|
284
|
-
"""
|
285
|
-
Initialize OTel once per logger_name within the process to avoid
|
286
|
-
clobbering providers (important under reloaders).
|
287
|
-
"""
|
234
|
+
def _setup_otel(self) -> None:
|
288
235
|
if not _OTEL_AVAILABLE:
|
289
|
-
self.
|
236
|
+
self._core.warning("OpenTelemetry not available — skipping OTel setup.")
|
290
237
|
return
|
291
238
|
if self.logger_name in self._otel_initialized_names:
|
292
|
-
|
293
|
-
|
239
|
+
with suppress(Exception):
|
240
|
+
self.tracer = trace.get_tracer(self.logger_name)
|
294
241
|
return
|
295
242
|
|
296
|
-
#
|
297
|
-
|
298
|
-
"service.name": self.otel_service_name,
|
299
|
-
"logger.name": self.logger_name,
|
300
|
-
}
|
243
|
+
# resources
|
244
|
+
attrs = {"service.name": self.otel_service_name, "logger.name": self.logger_name}
|
301
245
|
if self.otel_stream_name:
|
302
|
-
|
303
|
-
resource = Resource.create(
|
246
|
+
attrs["log.stream"] = self.otel_stream_name
|
247
|
+
resource = Resource.create(attrs)
|
304
248
|
|
305
|
-
#
|
249
|
+
# providers (reuse if already set globally)
|
306
250
|
existing_lp = None
|
307
|
-
|
251
|
+
with suppress(Exception):
|
308
252
|
existing_lp = get_logger_provider()
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
if not isinstance(existing_lp, LoggerProvider):
|
253
|
+
if getattr(existing_lp, "add_log_record_processor", None):
|
254
|
+
self.logger_provider = existing_lp
|
255
|
+
else:
|
313
256
|
self.logger_provider = LoggerProvider(resource=resource)
|
314
257
|
set_logger_provider(self.logger_provider)
|
315
|
-
else:
|
316
|
-
# reuse existing; don’t overwrite global provider
|
317
|
-
self.logger_provider = existing_lp # type: ignore
|
318
258
|
|
319
259
|
existing_tp = None
|
320
|
-
|
260
|
+
with suppress(Exception):
|
321
261
|
existing_tp = trace.get_tracer_provider()
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
if not isinstance(existing_tp, TracerProvider):
|
262
|
+
if getattr(existing_tp, "add_span_processor", None):
|
263
|
+
self.tracer_provider = existing_tp
|
264
|
+
else:
|
326
265
|
self.tracer_provider = TracerProvider(resource=resource)
|
327
266
|
trace.set_tracer_provider(self.tracer_provider)
|
328
|
-
else:
|
329
|
-
self.tracer_provider = existing_tp # type: ignore
|
330
267
|
|
331
268
|
endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
|
332
269
|
|
333
|
-
#
|
270
|
+
# exporters/processors (only if we own the providers we created above)
|
334
271
|
if isinstance(self.logger_provider, LoggerProvider):
|
335
|
-
|
272
|
+
with suppress(Exception):
|
336
273
|
log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
337
274
|
self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
|
338
|
-
except Exception as e:
|
339
|
-
self._core_logger.warning(f"Failed to attach OTel log exporter: {e}")
|
340
275
|
|
341
|
-
# Traces exporter + processor (only if we created our own provider)
|
342
276
|
if isinstance(self.tracer_provider, TracerProvider):
|
343
|
-
|
277
|
+
with suppress(Exception):
|
344
278
|
span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
345
279
|
self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
self._core_logger.addHandler(otel_handler)
|
354
|
-
except Exception as e:
|
355
|
-
self._core_logger.warning(f"Failed to attach OTel logging handler: {e}")
|
356
|
-
|
357
|
-
# Tracer handle
|
358
|
-
try:
|
280
|
+
|
281
|
+
# attach OTel log handler once
|
282
|
+
if not any(type(h).__name__ == "LoggingHandler" for h in self._core.handlers):
|
283
|
+
with suppress(Exception):
|
284
|
+
self._core.addHandler(LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)) # type: ignore
|
285
|
+
|
286
|
+
with suppress(Exception):
|
359
287
|
self.tracer = trace.get_tracer(self.logger_name)
|
360
|
-
except Exception:
|
361
|
-
self.tracer = None
|
362
288
|
|
363
289
|
self._otel_initialized_names.add(self.logger_name)
|
364
|
-
self.
|
365
|
-
|
366
|
-
# from __future__ import annotations
|
367
|
-
#
|
368
|
-
# import logging
|
369
|
-
# import os
|
370
|
-
# import sys
|
371
|
-
# import time
|
372
|
-
# from contextlib import contextmanager
|
373
|
-
# from logging import LoggerAdapter
|
374
|
-
# from logging.handlers import RotatingFileHandler
|
375
|
-
# from typing import Optional, Dict, Any
|
376
|
-
#
|
377
|
-
# # OpenTelemetry imports
|
378
|
-
# from opentelemetry import trace
|
379
|
-
# from opentelemetry._logs import set_logger_provider
|
380
|
-
# from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
381
|
-
# from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
382
|
-
# from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
383
|
-
# from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
384
|
-
# from opentelemetry.sdk.resources import Resource
|
385
|
-
# from opentelemetry.sdk.trace import TracerProvider
|
386
|
-
# from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
387
|
-
# from opentelemetry.trace import Tracer as OTelTracer
|
388
|
-
#
|
389
|
-
#
|
390
|
-
# class Logger:
|
391
|
-
# """
|
392
|
-
# Handles the creation and management of logging, with optional OpenTelemetry integration.
|
393
|
-
# """
|
394
|
-
#
|
395
|
-
# DEBUG = logging.DEBUG
|
396
|
-
# INFO = logging.INFO
|
397
|
-
# WARNING = logging.WARNING
|
398
|
-
# ERROR = logging.ERROR
|
399
|
-
# CRITICAL = logging.CRITICAL
|
400
|
-
#
|
401
|
-
# def __init__(
|
402
|
-
# self,
|
403
|
-
# log_dir: str,
|
404
|
-
# logger_name: str,
|
405
|
-
# log_file: str,
|
406
|
-
# log_level: int = logging.DEBUG,
|
407
|
-
# enable_otel: bool = False,
|
408
|
-
# otel_service_name: Optional[str] = None,
|
409
|
-
# otel_stream_name: Optional[str] = None,
|
410
|
-
# otel_endpoint: str = "0.0.0.0:4317",
|
411
|
-
# otel_insecure: bool = False,
|
412
|
-
# ):
|
413
|
-
# self.log_dir = log_dir
|
414
|
-
# self.logger_name = logger_name
|
415
|
-
# self.log_file = log_file
|
416
|
-
# self.log_level = log_level
|
417
|
-
#
|
418
|
-
# self.enable_otel = enable_otel
|
419
|
-
# self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
|
420
|
-
# self.otel_stream_name = (otel_stream_name or "").strip() or None
|
421
|
-
# self.otel_endpoint = otel_endpoint
|
422
|
-
# self.otel_insecure = otel_insecure
|
423
|
-
#
|
424
|
-
# self.logger_provider: Optional[LoggerProvider] = None
|
425
|
-
# self.tracer_provider: Optional[TracerProvider] = None
|
426
|
-
# self.tracer: Optional[OTelTracer] = None
|
427
|
-
#
|
428
|
-
# # Internal logger vs public (adapter) logger
|
429
|
-
# self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
|
430
|
-
# self.logger: logging.Logger | LoggerAdapter = self._core_logger
|
431
|
-
#
|
432
|
-
# self._setup()
|
433
|
-
#
|
434
|
-
# # -------------------------
|
435
|
-
# # Public API
|
436
|
-
# # -------------------------
|
437
|
-
#
|
438
|
-
# @classmethod
|
439
|
-
# def default_logger(
|
440
|
-
# cls,
|
441
|
-
# log_dir: str = "./logs/",
|
442
|
-
# logger_name: Optional[str] = None,
|
443
|
-
# log_file: Optional[str] = None,
|
444
|
-
# log_level: int = logging.INFO,
|
445
|
-
# enable_otel: bool = False,
|
446
|
-
# otel_service_name: Optional[str] = None,
|
447
|
-
# otel_stream_name: Optional[str] = None,
|
448
|
-
# otel_endpoint: str = "0.0.0.0:4317",
|
449
|
-
# otel_insecure: bool = False,
|
450
|
-
# ) -> "Logger":
|
451
|
-
# try:
|
452
|
-
# frame = sys._getframe(1)
|
453
|
-
# caller_name = frame.f_globals.get("__name__", "default_logger")
|
454
|
-
# except (AttributeError, ValueError):
|
455
|
-
# caller_name = "default_logger"
|
456
|
-
#
|
457
|
-
# logger_name = logger_name or caller_name
|
458
|
-
# log_file = log_file or logger_name
|
459
|
-
#
|
460
|
-
# return cls(
|
461
|
-
# log_dir=log_dir,
|
462
|
-
# logger_name=logger_name,
|
463
|
-
# log_file=log_file,
|
464
|
-
# log_level=log_level,
|
465
|
-
# enable_otel=enable_otel,
|
466
|
-
# otel_service_name=otel_service_name,
|
467
|
-
# otel_stream_name=otel_stream_name,
|
468
|
-
# otel_endpoint=otel_endpoint,
|
469
|
-
# otel_insecure=otel_insecure,
|
470
|
-
# )
|
471
|
-
#
|
472
|
-
# def shutdown(self):
|
473
|
-
# """Flush and shut down logging and tracing providers, then Python logging."""
|
474
|
-
# try:
|
475
|
-
# if self.enable_otel:
|
476
|
-
# if self.logger_provider:
|
477
|
-
# try:
|
478
|
-
# self._core_logger.info("Flushing OpenTelemetry logs...")
|
479
|
-
# self.logger_provider.force_flush()
|
480
|
-
# except Exception:
|
481
|
-
# pass
|
482
|
-
# try:
|
483
|
-
# self._core_logger.info("Shutting down OpenTelemetry logs...")
|
484
|
-
# self.logger_provider.shutdown()
|
485
|
-
# except Exception:
|
486
|
-
# pass
|
487
|
-
#
|
488
|
-
# if self.tracer_provider:
|
489
|
-
# try:
|
490
|
-
# self._core_logger.info("Flushing OpenTelemetry traces...")
|
491
|
-
# self.tracer_provider.force_flush()
|
492
|
-
# except Exception:
|
493
|
-
# pass
|
494
|
-
# try:
|
495
|
-
# self._core_logger.info("Shutting down OpenTelemetry traces...")
|
496
|
-
# self.tracer_provider.shutdown()
|
497
|
-
# except Exception:
|
498
|
-
# pass
|
499
|
-
# finally:
|
500
|
-
# logging.shutdown()
|
501
|
-
#
|
502
|
-
# def set_level(self, level: int):
|
503
|
-
# """Set the logging level for the logger."""
|
504
|
-
# self._core_logger.setLevel(level)
|
505
|
-
#
|
506
|
-
# # passthrough convenience methods
|
507
|
-
# def _log(self, level: int, msg: str, *args, **kwargs):
|
508
|
-
# extra = kwargs.pop("extra", None)
|
509
|
-
# if extra is not None:
|
510
|
-
# # Always emit via an adapter so extras survive to OTel attributes
|
511
|
-
# if isinstance(self.logger, LoggerAdapter):
|
512
|
-
# merged = {**self.logger.extra, **extra}
|
513
|
-
# LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
|
514
|
-
# else:
|
515
|
-
# LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
|
516
|
-
# else:
|
517
|
-
# self.logger.log(level, msg, *args, **kwargs)
|
518
|
-
#
|
519
|
-
# def debug(self, msg: str, *args, **kwargs):
|
520
|
-
# self._log(logging.DEBUG, msg, *args, **kwargs)
|
521
|
-
#
|
522
|
-
# def info(self, msg: str, *args, **kwargs):
|
523
|
-
# self._log(logging.INFO, msg, *args, **kwargs)
|
524
|
-
#
|
525
|
-
# def warning(self, msg: str, *args, **kwargs):
|
526
|
-
# self._log(logging.WARNING, msg, *args, **kwargs)
|
527
|
-
#
|
528
|
-
# def error(self, msg: str, *args, **kwargs):
|
529
|
-
# self._log(logging.ERROR, msg, *args, **kwargs)
|
530
|
-
#
|
531
|
-
# def critical(self, msg: str, *args, **kwargs):
|
532
|
-
# self._log(logging.CRITICAL, msg, *args, **kwargs)
|
533
|
-
#
|
534
|
-
#
|
535
|
-
# def bind(self, **extra: Any) -> LoggerAdapter:
|
536
|
-
# """
|
537
|
-
# Return a new LoggerAdapter bound with extra context, merging with existing extras if present.
|
538
|
-
# Example:
|
539
|
-
# api_log = logger.bind(component="api", request_id=req.id)
|
540
|
-
# api_log.info("processing")
|
541
|
-
# """
|
542
|
-
# if isinstance(self.logger, LoggerAdapter):
|
543
|
-
# merged = {**self.logger.extra, **extra}
|
544
|
-
# return LoggerAdapter(self.logger.logger, merged)
|
545
|
-
# return LoggerAdapter(self.logger, extra)
|
546
|
-
#
|
547
|
-
# @contextmanager
|
548
|
-
# def bound(self, **extra: Any):
|
549
|
-
# """
|
550
|
-
# Context manager that yields a bound adapter for temporary context.
|
551
|
-
# Example:
|
552
|
-
# with logger.bound(order_id=oid) as log:
|
553
|
-
# log.info("starting")
|
554
|
-
# ...
|
555
|
-
# """
|
556
|
-
# adapter = self.bind(**extra)
|
557
|
-
# try:
|
558
|
-
# yield adapter
|
559
|
-
# finally:
|
560
|
-
# # nothing to clean up; adapter is ephemeral
|
561
|
-
# pass
|
562
|
-
#
|
563
|
-
# def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
|
564
|
-
# """
|
565
|
-
# Start a span as a context manager.
|
566
|
-
#
|
567
|
-
# Usage:
|
568
|
-
# with logger.start_span("my-task", {"key": "value"}) as span:
|
569
|
-
# ...
|
570
|
-
# """
|
571
|
-
# if not self.enable_otel or not self.tracer:
|
572
|
-
# self.warning("Tracing is disabled or not initialized. Cannot start span.")
|
573
|
-
# from contextlib import nullcontext
|
574
|
-
# return nullcontext()
|
575
|
-
#
|
576
|
-
# cm = self.tracer.start_as_current_span(name)
|
577
|
-
#
|
578
|
-
# class _SpanCtx:
|
579
|
-
# def __enter__(_self):
|
580
|
-
# span = cm.__enter__()
|
581
|
-
# if attributes:
|
582
|
-
# for k, v in attributes.items():
|
583
|
-
# try:
|
584
|
-
# span.set_attribute(k, v)
|
585
|
-
# except Exception:
|
586
|
-
# pass
|
587
|
-
# return span
|
588
|
-
#
|
589
|
-
# def __exit__(_self, exc_type, exc, tb):
|
590
|
-
# return cm.__exit__(exc_type, exc, tb)
|
591
|
-
#
|
592
|
-
# return _SpanCtx()
|
593
|
-
#
|
594
|
-
# def trace_function(self, span_name: Optional[str] = None):
|
595
|
-
# """Decorator to trace a function with an optional custom span name."""
|
596
|
-
# def decorator(func):
|
597
|
-
# def wrapper(*args, **kwargs):
|
598
|
-
# name = span_name or func.__name__
|
599
|
-
# with self.start_span(name):
|
600
|
-
# return func(*args, **kwargs)
|
601
|
-
# return wrapper
|
602
|
-
# return decorator
|
603
|
-
#
|
604
|
-
# # -------------------------
|
605
|
-
# # Internal setup
|
606
|
-
# # -------------------------
|
607
|
-
#
|
608
|
-
# def _setup(self):
|
609
|
-
# """Set up core logger, handlers, and optional OTel."""
|
610
|
-
# # Configure base logger
|
611
|
-
# self._core_logger = logging.getLogger(self.logger_name)
|
612
|
-
# self._core_logger.setLevel(self.log_level)
|
613
|
-
# self._core_logger.propagate = False
|
614
|
-
#
|
615
|
-
# # Standard (file + console) handlers
|
616
|
-
# self._setup_standard_handlers()
|
617
|
-
#
|
618
|
-
# # OTel handlers (logs + traces)
|
619
|
-
# if self.enable_otel:
|
620
|
-
# self._setup_otel_handler()
|
621
|
-
#
|
622
|
-
# # Public-facing logger (optionally wrapped with adapter extras)
|
623
|
-
# if self.enable_otel and self.otel_stream_name:
|
624
|
-
# attributes = {
|
625
|
-
# "log_stream": self.otel_stream_name,
|
626
|
-
# "log_service_name": self.otel_service_name,
|
627
|
-
# "logger_name": self.logger_name,
|
628
|
-
# }
|
629
|
-
# self.logger = LoggerAdapter(self._core_logger, extra=attributes)
|
630
|
-
# else:
|
631
|
-
# self.logger = self._core_logger
|
632
|
-
#
|
633
|
-
# def _setup_standard_handlers(self):
|
634
|
-
# """Sets up file and console handlers with deduping."""
|
635
|
-
# os.makedirs(self.log_dir, exist_ok=True)
|
636
|
-
# calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
637
|
-
# log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
|
638
|
-
#
|
639
|
-
# formatter = logging.Formatter(
|
640
|
-
# "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
|
641
|
-
# datefmt="%Y-%m-%d %H:%M:%S",
|
642
|
-
# )
|
643
|
-
# formatter.converter = time.gmtime # UTC timestamps
|
644
|
-
#
|
645
|
-
# # File handler (dedupe by filename)
|
646
|
-
# if not any(
|
647
|
-
# isinstance(h, RotatingFileHandler) and getattr(h, "baseFilename", "") == os.path.abspath(log_file_path)
|
648
|
-
# for h in self._core_logger.handlers
|
649
|
-
# ):
|
650
|
-
# file_handler = RotatingFileHandler(
|
651
|
-
# log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
|
652
|
-
# )
|
653
|
-
# file_handler.setFormatter(formatter)
|
654
|
-
# self._core_logger.addHandler(file_handler)
|
655
|
-
#
|
656
|
-
# # Console handler (dedupe by stream)
|
657
|
-
# if not any(
|
658
|
-
# isinstance(h, logging.StreamHandler) and getattr(h, "stream", None) is sys.stdout
|
659
|
-
# for h in self._core_logger.handlers
|
660
|
-
# ):
|
661
|
-
# console_handler = logging.StreamHandler(sys.stdout)
|
662
|
-
# console_handler.setFormatter(formatter)
|
663
|
-
# self._core_logger.addHandler(console_handler)
|
664
|
-
#
|
665
|
-
# def _normalize_otlp_endpoint(self, ep: str) -> str:
|
666
|
-
# """Ensure OTLP gRPC endpoint has a scheme."""
|
667
|
-
# if "://" not in ep:
|
668
|
-
# ep = ("http://" if self.otel_insecure else "https://") + ep
|
669
|
-
# return ep
|
670
|
-
#
|
671
|
-
# def _setup_otel_handler(self):
|
672
|
-
# """
|
673
|
-
# Configure OpenTelemetry providers, exporters, and attach a LoggingHandler.
|
674
|
-
# - service.name: used by most backends (incl. OpenObserve) to group streams/services.
|
675
|
-
# - log.stream: extra attribute you can filter on in the backend.
|
676
|
-
# """
|
677
|
-
# resource_attrs = {
|
678
|
-
# "service.name": self.otel_service_name,
|
679
|
-
# "logger.name": self.logger_name,
|
680
|
-
# }
|
681
|
-
# if self.otel_stream_name:
|
682
|
-
# resource_attrs["log.stream"] = self.otel_stream_name
|
683
|
-
#
|
684
|
-
# resource = Resource.create(resource_attrs)
|
685
|
-
#
|
686
|
-
# # Logs provider
|
687
|
-
# self.logger_provider = LoggerProvider(resource=resource)
|
688
|
-
# set_logger_provider(self.logger_provider)
|
689
|
-
#
|
690
|
-
# # Traces provider
|
691
|
-
# self.tracer_provider = TracerProvider(resource=resource)
|
692
|
-
# trace.set_tracer_provider(self.tracer_provider)
|
693
|
-
#
|
694
|
-
# endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
|
695
|
-
#
|
696
|
-
# # Logs exporter + processor
|
697
|
-
# log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
698
|
-
# self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
|
699
|
-
#
|
700
|
-
# # Traces exporter + processor
|
701
|
-
# span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
702
|
-
# self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
|
703
|
-
# self.tracer = trace.get_tracer(self.logger_name, tracer_provider=self.tracer_provider)
|
704
|
-
#
|
705
|
-
# # Attach OTel LoggingHandler once
|
706
|
-
# if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
|
707
|
-
# otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)
|
708
|
-
# self._core_logger.addHandler(otel_handler)
|
709
|
-
#
|
710
|
-
# self._core_logger.info("OpenTelemetry logging and tracing enabled and attached.")
|
711
|
-
#
|
290
|
+
self._core.info("OpenTelemetry logging/tracing initialized.")
|