sibi-dst 2025.1.13__py3-none-any.whl → 2025.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sibi_dst/__init__.py +7 -1
- sibi_dst/df_helper/_artifact_updater_multi_wrapper.py +235 -342
- sibi_dst/df_helper/_df_helper.py +417 -117
- sibi_dst/df_helper/_parquet_artifact.py +255 -283
- sibi_dst/df_helper/backends/parquet/_parquet_options.py +8 -4
- sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py +68 -107
- sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py +15 -0
- sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py +105 -255
- sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py +90 -42
- sibi_dst/df_helper/backends/sqlalchemy/_model_registry.py +192 -0
- sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py +122 -72
- sibi_dst/osmnx_helper/route_path_builder.py +45 -46
- sibi_dst/utils/base.py +302 -96
- sibi_dst/utils/clickhouse_writer.py +472 -206
- sibi_dst/utils/data_utils.py +139 -186
- sibi_dst/utils/data_wrapper.py +317 -73
- sibi_dst/utils/date_utils.py +1 -0
- sibi_dst/utils/df_utils.py +193 -213
- sibi_dst/utils/file_utils.py +3 -2
- sibi_dst/utils/filepath_generator.py +314 -152
- sibi_dst/utils/log_utils.py +581 -242
- sibi_dst/utils/manifest_manager.py +60 -76
- sibi_dst/utils/parquet_saver.py +33 -27
- sibi_dst/utils/phone_formatter.py +88 -95
- sibi_dst/utils/update_planner.py +180 -178
- sibi_dst/utils/webdav_client.py +116 -166
- {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/METADATA +1 -1
- {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/RECORD +29 -27
- {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/WHEEL +0 -0
sibi_dst/utils/log_utils.py
CHANGED
@@ -1,27 +1,44 @@
|
|
1
1
|
from __future__ import annotations
|
2
|
+
|
2
3
|
import logging
|
3
4
|
import os
|
4
5
|
import sys
|
5
6
|
import time
|
7
|
+
from contextlib import contextmanager, nullcontext
|
6
8
|
from logging import LoggerAdapter
|
7
|
-
from typing import Optional
|
8
9
|
from logging.handlers import RotatingFileHandler
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
from opentelemetry
|
14
|
-
from opentelemetry.
|
15
|
-
from opentelemetry.
|
16
|
-
from opentelemetry.
|
17
|
-
from opentelemetry.sdk.
|
18
|
-
from opentelemetry.sdk.
|
19
|
-
from opentelemetry.sdk.
|
10
|
+
from typing import Optional, Dict, Any
|
11
|
+
|
12
|
+
# OpenTelemetry (optional)
|
13
|
+
try:
|
14
|
+
from opentelemetry import trace
|
15
|
+
from opentelemetry._logs import set_logger_provider, get_logger_provider
|
16
|
+
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
17
|
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
18
|
+
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
19
|
+
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
20
|
+
from opentelemetry.sdk.resources import Resource
|
21
|
+
from opentelemetry.sdk.trace import TracerProvider
|
22
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
23
|
+
from opentelemetry.trace import Tracer as OTelTracer
|
24
|
+
_OTEL_AVAILABLE = True
|
25
|
+
except Exception:
|
26
|
+
# OTel is optional; keep class working without it
|
27
|
+
LoggerProvider = TracerProvider = OTelTracer = object # type: ignore
|
28
|
+
LoggingHandler = object # type: ignore
|
29
|
+
_OTEL_AVAILABLE = False
|
20
30
|
|
21
31
|
|
22
32
|
class Logger:
|
23
33
|
"""
|
24
|
-
|
34
|
+
Process-safe logger with optional OpenTelemetry integration.
|
35
|
+
|
36
|
+
Backward-compatible surface:
|
37
|
+
- Logger.default_logger(...)
|
38
|
+
- .debug/.info/.warning/.error/.critical
|
39
|
+
- .set_level(level), .shutdown()
|
40
|
+
- .bind(**extra) -> LoggerAdapter, .bound(**extra) ctx manager
|
41
|
+
- .start_span(name, attributes=None), .trace_function(span_name=None)
|
25
42
|
"""
|
26
43
|
|
27
44
|
DEBUG = logging.DEBUG
|
@@ -30,133 +47,79 @@ class Logger:
|
|
30
47
|
ERROR = logging.ERROR
|
31
48
|
CRITICAL = logging.CRITICAL
|
32
49
|
|
50
|
+
# prevent attaching duplicate handlers per (logger_name, file_path) in process
|
51
|
+
_handler_keys_attached: set[tuple[str, str]] = set()
|
52
|
+
_otel_initialized_names: set[str] = set()
|
53
|
+
|
33
54
|
def __init__(
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
55
|
+
self,
|
56
|
+
log_dir: str,
|
57
|
+
logger_name: str,
|
58
|
+
log_file: str,
|
59
|
+
log_level: int = logging.DEBUG,
|
60
|
+
enable_otel: bool = False,
|
61
|
+
otel_service_name: Optional[str] = None,
|
62
|
+
otel_stream_name: Optional[str] = None,
|
63
|
+
otel_endpoint: str = "0.0.0.0:4317",
|
64
|
+
otel_insecure: bool = False,
|
44
65
|
):
|
45
66
|
self.log_dir = log_dir
|
46
67
|
self.logger_name = logger_name
|
47
68
|
self.log_file = log_file
|
48
69
|
self.log_level = log_level
|
49
|
-
|
50
|
-
self.
|
51
|
-
self.
|
70
|
+
|
71
|
+
self.enable_otel = bool(enable_otel and _OTEL_AVAILABLE)
|
72
|
+
self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
|
73
|
+
self.otel_stream_name = (otel_stream_name or "").strip() or None
|
52
74
|
self.otel_endpoint = otel_endpoint
|
53
75
|
self.otel_insecure = otel_insecure
|
76
|
+
|
54
77
|
self.logger_provider: Optional[LoggerProvider] = None
|
55
78
|
self.tracer_provider: Optional[TracerProvider] = None
|
56
|
-
self.tracer: Optional[
|
57
|
-
|
58
|
-
# Internal logger for configuration vs. public logger for use
|
59
|
-
self._core_logger: Optional[logging.Logger] = logging.getLogger(self.logger_name)
|
60
|
-
self.logger: Optional[logging.Logger | LoggerAdapter] = self._core_logger
|
79
|
+
self.tracer: Optional[OTelTracer] = None
|
61
80
|
|
62
|
-
self.
|
63
|
-
|
64
|
-
def _setup(self):
|
65
|
-
"""Set up the logger and then wrap it in an adapter if needed."""
|
66
|
-
# 1. Create and configure the actual logger instance
|
67
|
-
self._core_logger = logging.getLogger(self.logger_name)
|
81
|
+
self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
|
68
82
|
self._core_logger.setLevel(self.log_level)
|
69
83
|
self._core_logger.propagate = False
|
70
84
|
|
71
|
-
#
|
85
|
+
# public handle (may be adapter)
|
86
|
+
self.logger: logging.Logger | LoggerAdapter = self._core_logger
|
87
|
+
|
72
88
|
self._setup_standard_handlers()
|
73
89
|
if self.enable_otel:
|
74
|
-
self.
|
90
|
+
self._setup_otel_if_needed()
|
75
91
|
|
76
|
-
#
|
92
|
+
# expose adapter with default extras if OTel stream requested
|
77
93
|
if self.enable_otel and self.otel_stream_name:
|
78
|
-
|
79
|
-
attributes = {"attributes": {
|
94
|
+
attributes = {
|
80
95
|
"log_stream": self.otel_stream_name,
|
81
96
|
"log_service_name": self.otel_service_name,
|
82
97
|
"logger_name": self.logger_name,
|
83
|
-
}
|
84
98
|
}
|
85
99
|
self.logger = LoggerAdapter(self._core_logger, extra=attributes)
|
86
|
-
else:
|
87
|
-
# Otherwise, just use the logger directly
|
88
|
-
self.logger = self._core_logger
|
89
100
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
94
|
-
log_file_path = os.path.join(
|
95
|
-
self.log_dir, f"{self.log_file}_{calling_script}.log"
|
96
|
-
)
|
97
|
-
|
98
|
-
formatter = logging.Formatter(
|
99
|
-
'[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
|
100
|
-
datefmt='%Y-%m-%d %H:%M:%S',
|
101
|
-
)
|
102
|
-
#formatter.converter = time.localtime
|
103
|
-
formatter.converter = time.gmtime # Use GMT for consistency in logs
|
104
|
-
|
105
|
-
#file_handler = logging.FileHandler(log_file_path, delay=True)
|
106
|
-
file_handler = RotatingFileHandler(log_file_path, maxBytes=5*1024*1024, backupCount=5, delay=True)
|
107
|
-
file_handler.setFormatter(formatter)
|
108
|
-
self._core_logger.addHandler(file_handler)
|
109
|
-
|
110
|
-
console_handler = logging.StreamHandler(sys.stdout)
|
111
|
-
console_handler.setFormatter(formatter)
|
112
|
-
self._core_logger.addHandler(console_handler)
|
113
|
-
|
114
|
-
def _setup_otel_handler(self):
|
115
|
-
"""Sets up the OpenTelemetry logging handler."""
|
116
|
-
resource = Resource.create({"service.name": self.otel_stream_name or self.otel_service_name})
|
117
|
-
self.logger_provider = LoggerProvider(resource=resource)
|
118
|
-
set_logger_provider(self.logger_provider)
|
119
|
-
self.tracer_provider = TracerProvider(resource=resource)
|
120
|
-
trace.set_tracer_provider(self.tracer_provider)
|
121
|
-
|
122
|
-
exporter = OTLPLogExporter(
|
123
|
-
endpoint=self.otel_endpoint, insecure=self.otel_insecure
|
124
|
-
)
|
125
|
-
log_processor = BatchLogRecordProcessor(exporter)
|
126
|
-
self.logger_provider.add_log_record_processor(log_processor)
|
127
|
-
|
128
|
-
span_exporter = OTLPSpanExporter(
|
129
|
-
endpoint=self.otel_endpoint, insecure=self.otel_insecure
|
130
|
-
)
|
131
|
-
span_processor = BatchSpanProcessor(span_exporter)
|
132
|
-
self.tracer_provider.add_span_processor(span_processor)
|
133
|
-
self.tracer = trace.get_tracer(
|
134
|
-
self.logger_name, tracer_provider=self.tracer_provider
|
135
|
-
)
|
136
|
-
otel_handler = LoggingHandler(
|
137
|
-
level=logging.NOTSET, logger_provider=self.logger_provider
|
138
|
-
)
|
139
|
-
self._core_logger.addHandler(otel_handler)
|
140
|
-
self._core_logger.info("OpenTelemetry logging and tracing enabled and attached.")
|
101
|
+
# -------------------------
|
102
|
+
# Public API
|
103
|
+
# -------------------------
|
141
104
|
|
142
105
|
@classmethod
|
143
106
|
def default_logger(
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
) ->
|
107
|
+
cls,
|
108
|
+
log_dir: str = "./logs/",
|
109
|
+
logger_name: Optional[str] = None,
|
110
|
+
log_file: Optional[str] = None,
|
111
|
+
log_level: int = logging.INFO,
|
112
|
+
enable_otel: bool = False,
|
113
|
+
otel_service_name: Optional[str] = None,
|
114
|
+
otel_stream_name: Optional[str] = None,
|
115
|
+
otel_endpoint: str = "0.0.0.0:4317",
|
116
|
+
otel_insecure: bool = False,
|
117
|
+
) -> "Logger":
|
155
118
|
try:
|
156
119
|
frame = sys._getframe(1)
|
157
|
-
caller_name = frame.f_globals.get(
|
158
|
-
except
|
159
|
-
caller_name =
|
120
|
+
caller_name = frame.f_globals.get("__name__", "default_logger")
|
121
|
+
except Exception:
|
122
|
+
caller_name = "default_logger"
|
160
123
|
|
161
124
|
logger_name = logger_name or caller_name
|
162
125
|
log_file = log_file or logger_name
|
@@ -174,88 +137,259 @@ class Logger:
|
|
174
137
|
)
|
175
138
|
|
176
139
|
def shutdown(self):
|
177
|
-
"""
|
178
|
-
|
179
|
-
self.
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
140
|
+
"""Flush/close OTel providers and Python logging handlers."""
|
141
|
+
try:
|
142
|
+
if self.enable_otel:
|
143
|
+
if isinstance(self.logger_provider, LoggerProvider):
|
144
|
+
try:
|
145
|
+
self._core_logger.info("Flushing OpenTelemetry logs...")
|
146
|
+
self.logger_provider.force_flush()
|
147
|
+
except Exception:
|
148
|
+
pass
|
149
|
+
try:
|
150
|
+
self._core_logger.info("Shutting down OpenTelemetry logs...")
|
151
|
+
self.logger_provider.shutdown()
|
152
|
+
except Exception:
|
153
|
+
pass
|
154
|
+
|
155
|
+
if isinstance(self.tracer_provider, TracerProvider):
|
156
|
+
try:
|
157
|
+
self._core_logger.info("Flushing OpenTelemetry traces...")
|
158
|
+
self.tracer_provider.force_flush()
|
159
|
+
except Exception:
|
160
|
+
pass
|
161
|
+
try:
|
162
|
+
self._core_logger.info("Shutting down OpenTelemetry traces...")
|
163
|
+
self.tracer_provider.shutdown()
|
164
|
+
except Exception:
|
165
|
+
pass
|
166
|
+
finally:
|
167
|
+
# Close our handlers explicitly to release file descriptors
|
168
|
+
for h in list(self._core_logger.handlers):
|
169
|
+
try:
|
170
|
+
h.flush()
|
171
|
+
except Exception:
|
172
|
+
pass
|
173
|
+
try:
|
174
|
+
h.close()
|
175
|
+
except Exception:
|
176
|
+
pass
|
177
|
+
try:
|
178
|
+
self._core_logger.removeHandler(h)
|
179
|
+
except Exception:
|
180
|
+
pass
|
181
|
+
logging.shutdown()
|
185
182
|
|
186
183
|
def set_level(self, level: int):
|
187
|
-
"""Set the logging level for the logger."""
|
188
184
|
self._core_logger.setLevel(level)
|
189
185
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
186
|
+
# passthrough convenience
|
187
|
+
def _log(self, level: int, msg: str, *args, **kwargs):
|
188
|
+
extra = kwargs.pop("extra", None)
|
189
|
+
if extra is not None:
|
190
|
+
if isinstance(self.logger, LoggerAdapter):
|
191
|
+
merged = {**self.logger.extra, **extra}
|
192
|
+
LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
|
193
|
+
else:
|
194
|
+
LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
|
195
|
+
else:
|
196
|
+
self.logger.log(level, msg, *args, **kwargs)
|
197
|
+
|
198
|
+
def debug(self, msg: str, *args, **kwargs): self._log(logging.DEBUG, msg, *args, **kwargs)
|
199
|
+
def info(self, msg: str, *args, **kwargs): self._log(logging.INFO, msg, *args, **kwargs)
|
200
|
+
def warning(self, msg: str, *args, **kwargs): self._log(logging.WARNING, msg, *args, **kwargs)
|
201
|
+
def error(self, msg: str, *args, **kwargs): self._log(logging.ERROR, msg, *args, **kwargs)
|
202
|
+
def critical(self, msg: str, *args, **kwargs): self._log(logging.CRITICAL, msg, *args, **kwargs)
|
203
|
+
|
204
|
+
def bind(self, **extra: Any) -> LoggerAdapter:
|
205
|
+
if isinstance(self.logger, LoggerAdapter):
|
206
|
+
merged = {**self.logger.extra, **extra}
|
207
|
+
return LoggerAdapter(self.logger.logger, merged)
|
208
|
+
return LoggerAdapter(self.logger, extra)
|
209
|
+
|
210
|
+
@contextmanager
|
211
|
+
def bound(self, **extra: Any):
|
212
|
+
adapter = self.bind(**extra)
|
213
|
+
yield adapter
|
214
|
+
|
215
|
+
def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
|
216
|
+
if not (self.enable_otel and _OTEL_AVAILABLE and self.tracer):
|
217
|
+
# keep API but no-op cleanly
|
218
|
+
self.warning("Tracing is disabled or not initialized. Cannot start span.")
|
219
|
+
return nullcontext()
|
198
220
|
|
199
|
-
|
200
|
-
self.logger.error(msg, *args, **kwargs)
|
221
|
+
cm = self.tracer.start_as_current_span(name)
|
201
222
|
|
202
|
-
|
203
|
-
|
223
|
+
class _SpanCtx:
|
224
|
+
def __enter__(_self):
|
225
|
+
span = cm.__enter__()
|
226
|
+
if attributes:
|
227
|
+
for k, v in attributes.items():
|
228
|
+
try:
|
229
|
+
span.set_attribute(k, v)
|
230
|
+
except Exception:
|
231
|
+
pass
|
232
|
+
return span
|
204
233
|
|
205
|
-
|
206
|
-
|
207
|
-
Starts a span using the configured tracer.
|
208
|
-
Usage:
|
209
|
-
with logger.start_span("my-task") as span:
|
210
|
-
...
|
211
|
-
"""
|
212
|
-
if not self.enable_otel or not self.tracer:
|
213
|
-
self.warning("Tracing is disabled or not initialized. Cannot start span.")
|
214
|
-
# return dummy context manager
|
215
|
-
from contextlib import nullcontext
|
216
|
-
return nullcontext()
|
234
|
+
def __exit__(_self, exc_type, exc, tb):
|
235
|
+
return cm.__exit__(exc_type, exc, tb)
|
217
236
|
|
218
|
-
return
|
237
|
+
return _SpanCtx()
|
219
238
|
|
220
|
-
# Use this decorator to trace a function
|
221
239
|
def trace_function(self, span_name: Optional[str] = None):
|
222
240
|
def decorator(func):
|
223
241
|
def wrapper(*args, **kwargs):
|
224
242
|
name = span_name or func.__name__
|
225
243
|
with self.start_span(name):
|
226
244
|
return func(*args, **kwargs)
|
227
|
-
|
228
245
|
return wrapper
|
229
|
-
|
230
246
|
return decorator
|
231
247
|
|
248
|
+
# -------------------------
|
249
|
+
# Internal setup
|
250
|
+
# -------------------------
|
251
|
+
|
252
|
+
def _setup_standard_handlers(self):
|
253
|
+
os.makedirs(self.log_dir, exist_ok=True)
|
254
|
+
calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
255
|
+
log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
|
256
|
+
key = (self.logger_name, os.path.abspath(log_file_path))
|
257
|
+
|
258
|
+
formatter = logging.Formatter(
|
259
|
+
"[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
|
260
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
261
|
+
)
|
262
|
+
formatter.converter = time.gmtime # UTC timestamps
|
263
|
+
|
264
|
+
# attach once per process for this logger/file combo
|
265
|
+
if key not in self._handler_keys_attached:
|
266
|
+
file_handler = RotatingFileHandler(
|
267
|
+
log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
|
268
|
+
)
|
269
|
+
file_handler.setFormatter(formatter)
|
270
|
+
self._core_logger.addHandler(file_handler)
|
271
|
+
|
272
|
+
console_handler = logging.StreamHandler(sys.stdout)
|
273
|
+
console_handler.setFormatter(formatter)
|
274
|
+
self._core_logger.addHandler(console_handler)
|
275
|
+
|
276
|
+
self._handler_keys_attached.add(key)
|
232
277
|
|
278
|
+
def _normalize_otlp_endpoint(self, ep: str) -> str:
|
279
|
+
if "://" not in ep:
|
280
|
+
ep = ("http://" if self.otel_insecure else "https://") + ep
|
281
|
+
return ep
|
282
|
+
|
283
|
+
def _setup_otel_if_needed(self):
|
284
|
+
"""
|
285
|
+
Initialize OTel once per logger_name within the process to avoid
|
286
|
+
clobbering providers (important under reloaders).
|
287
|
+
"""
|
288
|
+
if not _OTEL_AVAILABLE:
|
289
|
+
self._core_logger.warning("OpenTelemetry not available — skipping OTel setup.")
|
290
|
+
return
|
291
|
+
if self.logger_name in self._otel_initialized_names:
|
292
|
+
# already initialized for this logger name in this process
|
293
|
+
self.tracer = trace.get_tracer(self.logger_name)
|
294
|
+
return
|
295
|
+
|
296
|
+
# Create resources
|
297
|
+
resource_attrs = {
|
298
|
+
"service.name": self.otel_service_name,
|
299
|
+
"logger.name": self.logger_name,
|
300
|
+
}
|
301
|
+
if self.otel_stream_name:
|
302
|
+
resource_attrs["log.stream"] = self.otel_stream_name
|
303
|
+
resource = Resource.create(resource_attrs)
|
304
|
+
|
305
|
+
# Respect any existing providers to avoid breaking apps that configured OTel elsewhere
|
306
|
+
existing_lp = None
|
307
|
+
try:
|
308
|
+
existing_lp = get_logger_provider()
|
309
|
+
except Exception:
|
310
|
+
pass
|
311
|
+
|
312
|
+
if not isinstance(existing_lp, LoggerProvider):
|
313
|
+
self.logger_provider = LoggerProvider(resource=resource)
|
314
|
+
set_logger_provider(self.logger_provider)
|
315
|
+
else:
|
316
|
+
# reuse existing; don’t overwrite global provider
|
317
|
+
self.logger_provider = existing_lp # type: ignore
|
318
|
+
|
319
|
+
existing_tp = None
|
320
|
+
try:
|
321
|
+
existing_tp = trace.get_tracer_provider()
|
322
|
+
except Exception:
|
323
|
+
pass
|
324
|
+
|
325
|
+
if not isinstance(existing_tp, TracerProvider):
|
326
|
+
self.tracer_provider = TracerProvider(resource=resource)
|
327
|
+
trace.set_tracer_provider(self.tracer_provider)
|
328
|
+
else:
|
329
|
+
self.tracer_provider = existing_tp # type: ignore
|
330
|
+
|
331
|
+
endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
|
332
|
+
|
333
|
+
# Logs exporter + processor (only if we created our own provider)
|
334
|
+
if isinstance(self.logger_provider, LoggerProvider):
|
335
|
+
try:
|
336
|
+
log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
337
|
+
self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
|
338
|
+
except Exception as e:
|
339
|
+
self._core_logger.warning(f"Failed to attach OTel log exporter: {e}")
|
340
|
+
|
341
|
+
# Traces exporter + processor (only if we created our own provider)
|
342
|
+
if isinstance(self.tracer_provider, TracerProvider):
|
343
|
+
try:
|
344
|
+
span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
345
|
+
self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
|
346
|
+
except Exception as e:
|
347
|
+
self._core_logger.warning(f"Failed to attach OTel span exporter: {e}")
|
348
|
+
|
349
|
+
# Attach OTel LoggingHandler once
|
350
|
+
if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
|
351
|
+
try:
|
352
|
+
otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider) # type: ignore
|
353
|
+
self._core_logger.addHandler(otel_handler)
|
354
|
+
except Exception as e:
|
355
|
+
self._core_logger.warning(f"Failed to attach OTel logging handler: {e}")
|
356
|
+
|
357
|
+
# Tracer handle
|
358
|
+
try:
|
359
|
+
self.tracer = trace.get_tracer(self.logger_name)
|
360
|
+
except Exception:
|
361
|
+
self.tracer = None
|
362
|
+
|
363
|
+
self._otel_initialized_names.add(self.logger_name)
|
364
|
+
self._core_logger.info("OpenTelemetry logging/tracing initialized.")
|
365
|
+
|
366
|
+
# from __future__ import annotations
|
367
|
+
#
|
233
368
|
# import logging
|
234
369
|
# import os
|
235
370
|
# import sys
|
236
371
|
# import time
|
237
|
-
# from
|
372
|
+
# from contextlib import contextmanager
|
373
|
+
# from logging import LoggerAdapter
|
374
|
+
# from logging.handlers import RotatingFileHandler
|
375
|
+
# from typing import Optional, Dict, Any
|
376
|
+
#
|
377
|
+
# # OpenTelemetry imports
|
378
|
+
# from opentelemetry import trace
|
379
|
+
# from opentelemetry._logs import set_logger_provider
|
380
|
+
# from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
381
|
+
# from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
382
|
+
# from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
383
|
+
# from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
384
|
+
# from opentelemetry.sdk.resources import Resource
|
385
|
+
# from opentelemetry.sdk.trace import TracerProvider
|
386
|
+
# from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
387
|
+
# from opentelemetry.trace import Tracer as OTelTracer
|
238
388
|
#
|
239
389
|
#
|
240
390
|
# class Logger:
|
241
391
|
# """
|
242
|
-
# Handles the creation
|
243
|
-
#
|
244
|
-
# This class facilitates logging by creating and managing a logger instance with
|
245
|
-
# customizable logging directory, name, and file. It ensures logs from a script
|
246
|
-
# are stored in a well-defined directory and file, and provides various logging
|
247
|
-
# methods for different log levels. The logger automatically formats and handles
|
248
|
-
# log messages. Additionally, this class provides a class method to initialize a
|
249
|
-
# logger with default behaviors.
|
250
|
-
#
|
251
|
-
# :ivar log_dir: Path to the directory where log files are stored.
|
252
|
-
# :type log_dir: str
|
253
|
-
# :ivar logger_name: Name of the logger instance.
|
254
|
-
# :type logger_name: str
|
255
|
-
# :ivar log_file: Base name of the log file.
|
256
|
-
# :type log_file: str
|
257
|
-
# :ivar logger: The initialized logger instance used for logging messages.
|
258
|
-
# :type logger: logging.Logger
|
392
|
+
# Handles the creation and management of logging, with optional OpenTelemetry integration.
|
259
393
|
# """
|
260
394
|
#
|
261
395
|
# DEBUG = logging.DEBUG
|
@@ -264,109 +398,314 @@ class Logger:
|
|
264
398
|
# ERROR = logging.ERROR
|
265
399
|
# CRITICAL = logging.CRITICAL
|
266
400
|
#
|
267
|
-
# def __init__(
|
268
|
-
#
|
269
|
-
#
|
270
|
-
#
|
271
|
-
# :
|
272
|
-
# :
|
273
|
-
# :
|
274
|
-
# :
|
275
|
-
#
|
401
|
+
# def __init__(
|
402
|
+
# self,
|
403
|
+
# log_dir: str,
|
404
|
+
# logger_name: str,
|
405
|
+
# log_file: str,
|
406
|
+
# log_level: int = logging.DEBUG,
|
407
|
+
# enable_otel: bool = False,
|
408
|
+
# otel_service_name: Optional[str] = None,
|
409
|
+
# otel_stream_name: Optional[str] = None,
|
410
|
+
# otel_endpoint: str = "0.0.0.0:4317",
|
411
|
+
# otel_insecure: bool = False,
|
412
|
+
# ):
|
276
413
|
# self.log_dir = log_dir
|
277
414
|
# self.logger_name = logger_name
|
278
415
|
# self.log_file = log_file
|
279
416
|
# self.log_level = log_level
|
280
|
-
# self.logger = None
|
281
417
|
#
|
282
|
-
# self.
|
418
|
+
# self.enable_otel = enable_otel
|
419
|
+
# self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
|
420
|
+
# self.otel_stream_name = (otel_stream_name or "").strip() or None
|
421
|
+
# self.otel_endpoint = otel_endpoint
|
422
|
+
# self.otel_insecure = otel_insecure
|
283
423
|
#
|
284
|
-
#
|
285
|
-
#
|
286
|
-
#
|
287
|
-
# os.makedirs(self.log_dir, exist_ok=True)
|
424
|
+
# self.logger_provider: Optional[LoggerProvider] = None
|
425
|
+
# self.tracer_provider: Optional[TracerProvider] = None
|
426
|
+
# self.tracer: Optional[OTelTracer] = None
|
288
427
|
#
|
289
|
-
# #
|
290
|
-
#
|
428
|
+
# # Internal logger vs public (adapter) logger
|
429
|
+
# self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
|
430
|
+
# self.logger: logging.Logger | LoggerAdapter = self._core_logger
|
291
431
|
#
|
292
|
-
#
|
293
|
-
# log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
|
432
|
+
# self._setup()
|
294
433
|
#
|
295
|
-
#
|
296
|
-
#
|
297
|
-
#
|
434
|
+
# # -------------------------
|
435
|
+
# # Public API
|
436
|
+
# # -------------------------
|
298
437
|
#
|
299
|
-
#
|
300
|
-
#
|
301
|
-
#
|
438
|
+
# @classmethod
|
439
|
+
# def default_logger(
|
440
|
+
# cls,
|
441
|
+
# log_dir: str = "./logs/",
|
442
|
+
# logger_name: Optional[str] = None,
|
443
|
+
# log_file: Optional[str] = None,
|
444
|
+
# log_level: int = logging.INFO,
|
445
|
+
# enable_otel: bool = False,
|
446
|
+
# otel_service_name: Optional[str] = None,
|
447
|
+
# otel_stream_name: Optional[str] = None,
|
448
|
+
# otel_endpoint: str = "0.0.0.0:4317",
|
449
|
+
# otel_insecure: bool = False,
|
450
|
+
# ) -> "Logger":
|
451
|
+
# try:
|
452
|
+
# frame = sys._getframe(1)
|
453
|
+
# caller_name = frame.f_globals.get("__name__", "default_logger")
|
454
|
+
# except (AttributeError, ValueError):
|
455
|
+
# caller_name = "default_logger"
|
302
456
|
#
|
303
|
-
#
|
304
|
-
#
|
305
|
-
#
|
306
|
-
#
|
457
|
+
# logger_name = logger_name or caller_name
|
458
|
+
# log_file = log_file or logger_name
|
459
|
+
#
|
460
|
+
# return cls(
|
461
|
+
# log_dir=log_dir,
|
462
|
+
# logger_name=logger_name,
|
463
|
+
# log_file=log_file,
|
464
|
+
# log_level=log_level,
|
465
|
+
# enable_otel=enable_otel,
|
466
|
+
# otel_service_name=otel_service_name,
|
467
|
+
# otel_stream_name=otel_stream_name,
|
468
|
+
# otel_endpoint=otel_endpoint,
|
469
|
+
# otel_insecure=otel_insecure,
|
307
470
|
# )
|
308
471
|
#
|
309
|
-
#
|
472
|
+
# def shutdown(self):
|
473
|
+
# """Flush and shut down logging and tracing providers, then Python logging."""
|
474
|
+
# try:
|
475
|
+
# if self.enable_otel:
|
476
|
+
# if self.logger_provider:
|
477
|
+
# try:
|
478
|
+
# self._core_logger.info("Flushing OpenTelemetry logs...")
|
479
|
+
# self.logger_provider.force_flush()
|
480
|
+
# except Exception:
|
481
|
+
# pass
|
482
|
+
# try:
|
483
|
+
# self._core_logger.info("Shutting down OpenTelemetry logs...")
|
484
|
+
# self.logger_provider.shutdown()
|
485
|
+
# except Exception:
|
486
|
+
# pass
|
310
487
|
#
|
311
|
-
#
|
312
|
-
#
|
313
|
-
#
|
314
|
-
#
|
488
|
+
# if self.tracer_provider:
|
489
|
+
# try:
|
490
|
+
# self._core_logger.info("Flushing OpenTelemetry traces...")
|
491
|
+
# self.tracer_provider.force_flush()
|
492
|
+
# except Exception:
|
493
|
+
# pass
|
494
|
+
# try:
|
495
|
+
# self._core_logger.info("Shutting down OpenTelemetry traces...")
|
496
|
+
# self.tracer_provider.shutdown()
|
497
|
+
# except Exception:
|
498
|
+
# pass
|
499
|
+
# finally:
|
500
|
+
# logging.shutdown()
|
315
501
|
#
|
316
|
-
#
|
317
|
-
#
|
318
|
-
#
|
319
|
-
# self.logger.addHandler(console_handler)
|
502
|
+
# def set_level(self, level: int):
|
503
|
+
# """Set the logging level for the logger."""
|
504
|
+
# self._core_logger.setLevel(level)
|
320
505
|
#
|
321
|
-
#
|
322
|
-
# def
|
323
|
-
#
|
324
|
-
#
|
325
|
-
#
|
326
|
-
#
|
327
|
-
#
|
328
|
-
#
|
506
|
+
# # passthrough convenience methods
|
507
|
+
# def _log(self, level: int, msg: str, *args, **kwargs):
|
508
|
+
# extra = kwargs.pop("extra", None)
|
509
|
+
# if extra is not None:
|
510
|
+
# # Always emit via an adapter so extras survive to OTel attributes
|
511
|
+
# if isinstance(self.logger, LoggerAdapter):
|
512
|
+
# merged = {**self.logger.extra, **extra}
|
513
|
+
# LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
|
514
|
+
# else:
|
515
|
+
# LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
|
516
|
+
# else:
|
517
|
+
# self.logger.log(level, msg, *args, **kwargs)
|
518
|
+
#
|
519
|
+
# def debug(self, msg: str, *args, **kwargs):
|
520
|
+
# self._log(logging.DEBUG, msg, *args, **kwargs)
|
521
|
+
#
|
522
|
+
# def info(self, msg: str, *args, **kwargs):
|
523
|
+
# self._log(logging.INFO, msg, *args, **kwargs)
|
524
|
+
#
|
525
|
+
# def warning(self, msg: str, *args, **kwargs):
|
526
|
+
# self._log(logging.WARNING, msg, *args, **kwargs)
|
527
|
+
#
|
528
|
+
# def error(self, msg: str, *args, **kwargs):
|
529
|
+
# self._log(logging.ERROR, msg, *args, **kwargs)
|
530
|
+
#
|
531
|
+
# def critical(self, msg: str, *args, **kwargs):
|
532
|
+
# self._log(logging.CRITICAL, msg, *args, **kwargs)
|
533
|
+
#
|
534
|
+
#
|
535
|
+
# def bind(self, **extra: Any) -> LoggerAdapter:
|
536
|
+
# """
|
537
|
+
# Return a new LoggerAdapter bound with extra context, merging with existing extras if present.
|
538
|
+
# Example:
|
539
|
+
# api_log = logger.bind(component="api", request_id=req.id)
|
540
|
+
# api_log.info("processing")
|
329
541
|
# """
|
330
|
-
#
|
542
|
+
# if isinstance(self.logger, LoggerAdapter):
|
543
|
+
# merged = {**self.logger.extra, **extra}
|
544
|
+
# return LoggerAdapter(self.logger.logger, merged)
|
545
|
+
# return LoggerAdapter(self.logger, extra)
|
331
546
|
#
|
332
|
-
#
|
333
|
-
#
|
334
|
-
# :param log_file: Name of the log file (defaults to logger_name).
|
335
|
-
# :param log_level: Logging level (defaults to INFO).
|
336
|
-
# :return: Instance of Logger.
|
547
|
+
# @contextmanager
|
548
|
+
# def bound(self, **extra: Any):
|
337
549
|
# """
|
338
|
-
#
|
339
|
-
#
|
340
|
-
#
|
550
|
+
# Context manager that yields a bound adapter for temporary context.
|
551
|
+
# Example:
|
552
|
+
# with logger.bound(order_id=oid) as log:
|
553
|
+
# log.info("starting")
|
554
|
+
# ...
|
555
|
+
# """
|
556
|
+
# adapter = self.bind(**extra)
|
557
|
+
# try:
|
558
|
+
# yield adapter
|
559
|
+
# finally:
|
560
|
+
# # nothing to clean up; adapter is ephemeral
|
561
|
+
# pass
|
341
562
|
#
|
342
|
-
# def
|
563
|
+
# def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
|
343
564
|
# """
|
344
|
-
#
|
565
|
+
# Start a span as a context manager.
|
345
566
|
#
|
346
|
-
# :
|
567
|
+
# Usage:
|
568
|
+
# with logger.start_span("my-task", {"key": "value"}) as span:
|
569
|
+
# ...
|
347
570
|
# """
|
348
|
-
# self.
|
571
|
+
# if not self.enable_otel or not self.tracer:
|
572
|
+
# self.warning("Tracing is disabled or not initialized. Cannot start span.")
|
573
|
+
# from contextlib import nullcontext
|
574
|
+
# return nullcontext()
|
349
575
|
#
|
350
|
-
#
|
351
|
-
# """Log a debug message."""
|
352
|
-
# self.logger.debug(msg, *args, **kwargs)
|
576
|
+
# cm = self.tracer.start_as_current_span(name)
|
353
577
|
#
|
354
|
-
#
|
355
|
-
#
|
356
|
-
#
|
578
|
+
# class _SpanCtx:
|
579
|
+
# def __enter__(_self):
|
580
|
+
# span = cm.__enter__()
|
581
|
+
# if attributes:
|
582
|
+
# for k, v in attributes.items():
|
583
|
+
# try:
|
584
|
+
# span.set_attribute(k, v)
|
585
|
+
# except Exception:
|
586
|
+
# pass
|
587
|
+
# return span
|
357
588
|
#
|
358
|
-
#
|
359
|
-
#
|
360
|
-
# self.logger.warning(msg, *args, **kwargs)
|
589
|
+
# def __exit__(_self, exc_type, exc, tb):
|
590
|
+
# return cm.__exit__(exc_type, exc, tb)
|
361
591
|
#
|
362
|
-
#
|
363
|
-
#
|
364
|
-
#
|
592
|
+
# return _SpanCtx()
|
593
|
+
#
|
594
|
+
# def trace_function(self, span_name: Optional[str] = None):
|
595
|
+
# """Decorator to trace a function with an optional custom span name."""
|
596
|
+
# def decorator(func):
|
597
|
+
# def wrapper(*args, **kwargs):
|
598
|
+
# name = span_name or func.__name__
|
599
|
+
# with self.start_span(name):
|
600
|
+
# return func(*args, **kwargs)
|
601
|
+
# return wrapper
|
602
|
+
# return decorator
|
603
|
+
#
|
604
|
+
# # -------------------------
|
605
|
+
# # Internal setup
|
606
|
+
# # -------------------------
|
607
|
+
#
|
608
|
+
# def _setup(self):
|
609
|
+
# """Set up core logger, handlers, and optional OTel."""
|
610
|
+
# # Configure base logger
|
611
|
+
# self._core_logger = logging.getLogger(self.logger_name)
|
612
|
+
# self._core_logger.setLevel(self.log_level)
|
613
|
+
# self._core_logger.propagate = False
|
614
|
+
#
|
615
|
+
# # Standard (file + console) handlers
|
616
|
+
# self._setup_standard_handlers()
|
617
|
+
#
|
618
|
+
# # OTel handlers (logs + traces)
|
619
|
+
# if self.enable_otel:
|
620
|
+
# self._setup_otel_handler()
|
621
|
+
#
|
622
|
+
# # Public-facing logger (optionally wrapped with adapter extras)
|
623
|
+
# if self.enable_otel and self.otel_stream_name:
|
624
|
+
# attributes = {
|
625
|
+
# "log_stream": self.otel_stream_name,
|
626
|
+
# "log_service_name": self.otel_service_name,
|
627
|
+
# "logger_name": self.logger_name,
|
628
|
+
# }
|
629
|
+
# self.logger = LoggerAdapter(self._core_logger, extra=attributes)
|
630
|
+
# else:
|
631
|
+
# self.logger = self._core_logger
|
632
|
+
#
|
633
|
+
# def _setup_standard_handlers(self):
|
634
|
+
# """Sets up file and console handlers with deduping."""
|
635
|
+
# os.makedirs(self.log_dir, exist_ok=True)
|
636
|
+
# calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
637
|
+
# log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
|
638
|
+
#
|
639
|
+
# formatter = logging.Formatter(
|
640
|
+
# "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
|
641
|
+
# datefmt="%Y-%m-%d %H:%M:%S",
|
642
|
+
# )
|
643
|
+
# formatter.converter = time.gmtime # UTC timestamps
|
644
|
+
#
|
645
|
+
# # File handler (dedupe by filename)
|
646
|
+
# if not any(
|
647
|
+
# isinstance(h, RotatingFileHandler) and getattr(h, "baseFilename", "") == os.path.abspath(log_file_path)
|
648
|
+
# for h in self._core_logger.handlers
|
649
|
+
# ):
|
650
|
+
# file_handler = RotatingFileHandler(
|
651
|
+
# log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
|
652
|
+
# )
|
653
|
+
# file_handler.setFormatter(formatter)
|
654
|
+
# self._core_logger.addHandler(file_handler)
|
655
|
+
#
|
656
|
+
# # Console handler (dedupe by stream)
|
657
|
+
# if not any(
|
658
|
+
# isinstance(h, logging.StreamHandler) and getattr(h, "stream", None) is sys.stdout
|
659
|
+
# for h in self._core_logger.handlers
|
660
|
+
# ):
|
661
|
+
# console_handler = logging.StreamHandler(sys.stdout)
|
662
|
+
# console_handler.setFormatter(formatter)
|
663
|
+
# self._core_logger.addHandler(console_handler)
|
365
664
|
#
|
366
|
-
#
|
665
|
+
# def _normalize_otlp_endpoint(self, ep: str) -> str:
|
666
|
+
# """Ensure OTLP gRPC endpoint has a scheme."""
|
667
|
+
# if "://" not in ep:
|
668
|
+
# ep = ("http://" if self.otel_insecure else "https://") + ep
|
669
|
+
# return ep
|
670
|
+
#
|
671
|
+
# def _setup_otel_handler(self):
|
672
|
+
# """
|
673
|
+
# Configure OpenTelemetry providers, exporters, and attach a LoggingHandler.
|
674
|
+
# - service.name: used by most backends (incl. OpenObserve) to group streams/services.
|
675
|
+
# - log.stream: extra attribute you can filter on in the backend.
|
367
676
|
# """
|
368
|
-
#
|
677
|
+
# resource_attrs = {
|
678
|
+
# "service.name": self.otel_service_name,
|
679
|
+
# "logger.name": self.logger_name,
|
680
|
+
# }
|
681
|
+
# if self.otel_stream_name:
|
682
|
+
# resource_attrs["log.stream"] = self.otel_stream_name
|
683
|
+
#
|
684
|
+
# resource = Resource.create(resource_attrs)
|
685
|
+
#
|
686
|
+
# # Logs provider
|
687
|
+
# self.logger_provider = LoggerProvider(resource=resource)
|
688
|
+
# set_logger_provider(self.logger_provider)
|
689
|
+
#
|
690
|
+
# # Traces provider
|
691
|
+
# self.tracer_provider = TracerProvider(resource=resource)
|
692
|
+
# trace.set_tracer_provider(self.tracer_provider)
|
693
|
+
#
|
694
|
+
# endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
|
695
|
+
#
|
696
|
+
# # Logs exporter + processor
|
697
|
+
# log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
698
|
+
# self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
|
699
|
+
#
|
700
|
+
# # Traces exporter + processor
|
701
|
+
# span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
|
702
|
+
# self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
|
703
|
+
# self.tracer = trace.get_tracer(self.logger_name, tracer_provider=self.tracer_provider)
|
704
|
+
#
|
705
|
+
# # Attach OTel LoggingHandler once
|
706
|
+
# if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
|
707
|
+
# otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)
|
708
|
+
# self._core_logger.addHandler(otel_handler)
|
709
|
+
#
|
710
|
+
# self._core_logger.info("OpenTelemetry logging and tracing enabled and attached.")
|
369
711
|
#
|
370
|
-
# def critical(self, msg: str, *args, **kwargs):
|
371
|
-
# """Log a critical message."""
|
372
|
-
# self.logger.critical(msg, *args, **kwargs)
|