sibi-dst 2025.1.13__py3-none-any.whl → 2025.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. sibi_dst/__init__.py +7 -1
  2. sibi_dst/df_helper/_artifact_updater_multi_wrapper.py +235 -342
  3. sibi_dst/df_helper/_df_helper.py +417 -117
  4. sibi_dst/df_helper/_parquet_artifact.py +255 -283
  5. sibi_dst/df_helper/backends/parquet/_parquet_options.py +8 -4
  6. sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py +68 -107
  7. sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py +15 -0
  8. sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py +105 -255
  9. sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py +90 -42
  10. sibi_dst/df_helper/backends/sqlalchemy/_model_registry.py +192 -0
  11. sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py +122 -72
  12. sibi_dst/osmnx_helper/route_path_builder.py +45 -46
  13. sibi_dst/utils/base.py +302 -96
  14. sibi_dst/utils/clickhouse_writer.py +472 -206
  15. sibi_dst/utils/data_utils.py +139 -186
  16. sibi_dst/utils/data_wrapper.py +317 -73
  17. sibi_dst/utils/date_utils.py +1 -0
  18. sibi_dst/utils/df_utils.py +193 -213
  19. sibi_dst/utils/file_utils.py +3 -2
  20. sibi_dst/utils/filepath_generator.py +314 -152
  21. sibi_dst/utils/log_utils.py +581 -242
  22. sibi_dst/utils/manifest_manager.py +60 -76
  23. sibi_dst/utils/parquet_saver.py +33 -27
  24. sibi_dst/utils/phone_formatter.py +88 -95
  25. sibi_dst/utils/update_planner.py +180 -178
  26. sibi_dst/utils/webdav_client.py +116 -166
  27. {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/METADATA +1 -1
  28. {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/RECORD +29 -27
  29. {sibi_dst-2025.1.13.dist-info → sibi_dst-2025.8.1.dist-info}/WHEEL +0 -0
@@ -1,27 +1,44 @@
1
1
  from __future__ import annotations
2
+
2
3
  import logging
3
4
  import os
4
5
  import sys
5
6
  import time
7
+ from contextlib import contextmanager, nullcontext
6
8
  from logging import LoggerAdapter
7
- from typing import Optional
8
9
  from logging.handlers import RotatingFileHandler
9
-
10
- # OpenTelemetry imports
11
- from opentelemetry import trace
12
- from opentelemetry._logs import set_logger_provider
13
- from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
14
- from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
15
- from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
16
- from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
17
- from opentelemetry.sdk.resources import Resource
18
- from opentelemetry.sdk.trace import Tracer, TracerProvider
19
- from opentelemetry.sdk.trace.export import BatchSpanProcessor
10
+ from typing import Optional, Dict, Any
11
+
12
+ # OpenTelemetry (optional)
13
+ try:
14
+ from opentelemetry import trace
15
+ from opentelemetry._logs import set_logger_provider, get_logger_provider
16
+ from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
17
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
18
+ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
19
+ from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
20
+ from opentelemetry.sdk.resources import Resource
21
+ from opentelemetry.sdk.trace import TracerProvider
22
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
23
+ from opentelemetry.trace import Tracer as OTelTracer
24
+ _OTEL_AVAILABLE = True
25
+ except Exception:
26
+ # OTel is optional; keep class working without it
27
+ LoggerProvider = TracerProvider = OTelTracer = object # type: ignore
28
+ LoggingHandler = object # type: ignore
29
+ _OTEL_AVAILABLE = False
20
30
 
21
31
 
22
32
  class Logger:
23
33
  """
24
- Handles the creation and management of logging, with optional OpenTelemetry integration.
34
+ Process-safe logger with optional OpenTelemetry integration.
35
+
36
+ Backward-compatible surface:
37
+ - Logger.default_logger(...)
38
+ - .debug/.info/.warning/.error/.critical
39
+ - .set_level(level), .shutdown()
40
+ - .bind(**extra) -> LoggerAdapter, .bound(**extra) ctx manager
41
+ - .start_span(name, attributes=None), .trace_function(span_name=None)
25
42
  """
26
43
 
27
44
  DEBUG = logging.DEBUG
@@ -30,133 +47,79 @@ class Logger:
30
47
  ERROR = logging.ERROR
31
48
  CRITICAL = logging.CRITICAL
32
49
 
50
+ # prevent attaching duplicate handlers per (logger_name, file_path) in process
51
+ _handler_keys_attached: set[tuple[str, str]] = set()
52
+ _otel_initialized_names: set[str] = set()
53
+
33
54
  def __init__(
34
- self,
35
- log_dir: str,
36
- logger_name: str,
37
- log_file: str,
38
- log_level: int = logging.DEBUG,
39
- enable_otel: bool = False,
40
- otel_service_name: Optional[str] = None,
41
- otel_stream_name: Optional[str] = None,
42
- otel_endpoint: str = "0.0.0.0:4317",
43
- otel_insecure: bool = False,
55
+ self,
56
+ log_dir: str,
57
+ logger_name: str,
58
+ log_file: str,
59
+ log_level: int = logging.DEBUG,
60
+ enable_otel: bool = False,
61
+ otel_service_name: Optional[str] = None,
62
+ otel_stream_name: Optional[str] = None,
63
+ otel_endpoint: str = "0.0.0.0:4317",
64
+ otel_insecure: bool = False,
44
65
  ):
45
66
  self.log_dir = log_dir
46
67
  self.logger_name = logger_name
47
68
  self.log_file = log_file
48
69
  self.log_level = log_level
49
- self.enable_otel = enable_otel
50
- self.otel_service_name = otel_service_name or self.logger_name
51
- self.otel_stream_name = otel_stream_name
70
+
71
+ self.enable_otel = bool(enable_otel and _OTEL_AVAILABLE)
72
+ self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
73
+ self.otel_stream_name = (otel_stream_name or "").strip() or None
52
74
  self.otel_endpoint = otel_endpoint
53
75
  self.otel_insecure = otel_insecure
76
+
54
77
  self.logger_provider: Optional[LoggerProvider] = None
55
78
  self.tracer_provider: Optional[TracerProvider] = None
56
- self.tracer: Optional[Tracer] = None
57
-
58
- # Internal logger for configuration vs. public logger for use
59
- self._core_logger: Optional[logging.Logger] = logging.getLogger(self.logger_name)
60
- self.logger: Optional[logging.Logger | LoggerAdapter] = self._core_logger
79
+ self.tracer: Optional[OTelTracer] = None
61
80
 
62
- self._setup()
63
-
64
- def _setup(self):
65
- """Set up the logger and then wrap it in an adapter if needed."""
66
- # 1. Create and configure the actual logger instance
67
- self._core_logger = logging.getLogger(self.logger_name)
81
+ self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
68
82
  self._core_logger.setLevel(self.log_level)
69
83
  self._core_logger.propagate = False
70
84
 
71
- # 2. Add all handlers to the actual logger instance
85
+ # public handle (may be adapter)
86
+ self.logger: logging.Logger | LoggerAdapter = self._core_logger
87
+
72
88
  self._setup_standard_handlers()
73
89
  if self.enable_otel:
74
- self._setup_otel_handler()
90
+ self._setup_otel_if_needed()
75
91
 
76
- # 3. Create the final, public-facing logger object
92
+ # expose adapter with default extras if OTel stream requested
77
93
  if self.enable_otel and self.otel_stream_name:
78
- # If stream name is used, wrap the configured logger in an adapter
79
- attributes = {"attributes": {
94
+ attributes = {
80
95
  "log_stream": self.otel_stream_name,
81
96
  "log_service_name": self.otel_service_name,
82
97
  "logger_name": self.logger_name,
83
- }
84
98
  }
85
99
  self.logger = LoggerAdapter(self._core_logger, extra=attributes)
86
- else:
87
- # Otherwise, just use the logger directly
88
- self.logger = self._core_logger
89
100
 
90
- def _setup_standard_handlers(self):
91
- """Sets up the file and console logging handlers."""
92
- os.makedirs(self.log_dir, exist_ok=True)
93
- calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
94
- log_file_path = os.path.join(
95
- self.log_dir, f"{self.log_file}_{calling_script}.log"
96
- )
97
-
98
- formatter = logging.Formatter(
99
- '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
100
- datefmt='%Y-%m-%d %H:%M:%S',
101
- )
102
- #formatter.converter = time.localtime
103
- formatter.converter = time.gmtime # Use GMT for consistency in logs
104
-
105
- #file_handler = logging.FileHandler(log_file_path, delay=True)
106
- file_handler = RotatingFileHandler(log_file_path, maxBytes=5*1024*1024, backupCount=5, delay=True)
107
- file_handler.setFormatter(formatter)
108
- self._core_logger.addHandler(file_handler)
109
-
110
- console_handler = logging.StreamHandler(sys.stdout)
111
- console_handler.setFormatter(formatter)
112
- self._core_logger.addHandler(console_handler)
113
-
114
- def _setup_otel_handler(self):
115
- """Sets up the OpenTelemetry logging handler."""
116
- resource = Resource.create({"service.name": self.otel_stream_name or self.otel_service_name})
117
- self.logger_provider = LoggerProvider(resource=resource)
118
- set_logger_provider(self.logger_provider)
119
- self.tracer_provider = TracerProvider(resource=resource)
120
- trace.set_tracer_provider(self.tracer_provider)
121
-
122
- exporter = OTLPLogExporter(
123
- endpoint=self.otel_endpoint, insecure=self.otel_insecure
124
- )
125
- log_processor = BatchLogRecordProcessor(exporter)
126
- self.logger_provider.add_log_record_processor(log_processor)
127
-
128
- span_exporter = OTLPSpanExporter(
129
- endpoint=self.otel_endpoint, insecure=self.otel_insecure
130
- )
131
- span_processor = BatchSpanProcessor(span_exporter)
132
- self.tracer_provider.add_span_processor(span_processor)
133
- self.tracer = trace.get_tracer(
134
- self.logger_name, tracer_provider=self.tracer_provider
135
- )
136
- otel_handler = LoggingHandler(
137
- level=logging.NOTSET, logger_provider=self.logger_provider
138
- )
139
- self._core_logger.addHandler(otel_handler)
140
- self._core_logger.info("OpenTelemetry logging and tracing enabled and attached.")
101
+ # -------------------------
102
+ # Public API
103
+ # -------------------------
141
104
 
142
105
  @classmethod
143
106
  def default_logger(
144
- cls,
145
- log_dir: str = './logs/',
146
- logger_name: Optional[str] = None,
147
- log_file: Optional[str] = None,
148
- log_level: int = logging.INFO,
149
- enable_otel: bool = False,
150
- otel_service_name: Optional[str] = None,
151
- otel_stream_name: Optional[str] = None,
152
- otel_endpoint: str = "0.0.0.0:4317",
153
- otel_insecure: bool = False,
154
- ) -> 'Logger':
107
+ cls,
108
+ log_dir: str = "./logs/",
109
+ logger_name: Optional[str] = None,
110
+ log_file: Optional[str] = None,
111
+ log_level: int = logging.INFO,
112
+ enable_otel: bool = False,
113
+ otel_service_name: Optional[str] = None,
114
+ otel_stream_name: Optional[str] = None,
115
+ otel_endpoint: str = "0.0.0.0:4317",
116
+ otel_insecure: bool = False,
117
+ ) -> "Logger":
155
118
  try:
156
119
  frame = sys._getframe(1)
157
- caller_name = frame.f_globals.get('__name__', 'default_logger')
158
- except (AttributeError, ValueError):
159
- caller_name = 'default_logger'
120
+ caller_name = frame.f_globals.get("__name__", "default_logger")
121
+ except Exception:
122
+ caller_name = "default_logger"
160
123
 
161
124
  logger_name = logger_name or caller_name
162
125
  log_file = log_file or logger_name
@@ -174,88 +137,259 @@ class Logger:
174
137
  )
175
138
 
176
139
  def shutdown(self):
177
- """Gracefully shuts down the logger and the OpenTelemetry provider."""
178
- if self.enable_otel and self.logger_provider:
179
- self.logger.info("Shutting down OpenTelemetry logger provider...")
180
- if self.otel_stream_name:
181
- self._core_logger.info(f"OpenObserve stream configured as: '{self.otel_stream_name}'")
182
- self.logger_provider.shutdown()
183
- print("Logger provider shut down.")
184
- logging.shutdown()
140
+ """Flush/close OTel providers and Python logging handlers."""
141
+ try:
142
+ if self.enable_otel:
143
+ if isinstance(self.logger_provider, LoggerProvider):
144
+ try:
145
+ self._core_logger.info("Flushing OpenTelemetry logs...")
146
+ self.logger_provider.force_flush()
147
+ except Exception:
148
+ pass
149
+ try:
150
+ self._core_logger.info("Shutting down OpenTelemetry logs...")
151
+ self.logger_provider.shutdown()
152
+ except Exception:
153
+ pass
154
+
155
+ if isinstance(self.tracer_provider, TracerProvider):
156
+ try:
157
+ self._core_logger.info("Flushing OpenTelemetry traces...")
158
+ self.tracer_provider.force_flush()
159
+ except Exception:
160
+ pass
161
+ try:
162
+ self._core_logger.info("Shutting down OpenTelemetry traces...")
163
+ self.tracer_provider.shutdown()
164
+ except Exception:
165
+ pass
166
+ finally:
167
+ # Close our handlers explicitly to release file descriptors
168
+ for h in list(self._core_logger.handlers):
169
+ try:
170
+ h.flush()
171
+ except Exception:
172
+ pass
173
+ try:
174
+ h.close()
175
+ except Exception:
176
+ pass
177
+ try:
178
+ self._core_logger.removeHandler(h)
179
+ except Exception:
180
+ pass
181
+ logging.shutdown()
185
182
 
186
183
  def set_level(self, level: int):
187
- """Set the logging level for the logger."""
188
184
  self._core_logger.setLevel(level)
189
185
 
190
- def debug(self, msg: str, *args, **kwargs):
191
- self.logger.debug(msg, *args, **kwargs)
192
-
193
- def info(self, msg: str, *args, **kwargs):
194
- self.logger.info(msg, *args, **kwargs)
195
-
196
- def warning(self, msg: str, *args, **kwargs):
197
- self.logger.warning(msg, *args, **kwargs)
186
+ # passthrough convenience
187
+ def _log(self, level: int, msg: str, *args, **kwargs):
188
+ extra = kwargs.pop("extra", None)
189
+ if extra is not None:
190
+ if isinstance(self.logger, LoggerAdapter):
191
+ merged = {**self.logger.extra, **extra}
192
+ LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
193
+ else:
194
+ LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
195
+ else:
196
+ self.logger.log(level, msg, *args, **kwargs)
197
+
198
+ def debug(self, msg: str, *args, **kwargs): self._log(logging.DEBUG, msg, *args, **kwargs)
199
+ def info(self, msg: str, *args, **kwargs): self._log(logging.INFO, msg, *args, **kwargs)
200
+ def warning(self, msg: str, *args, **kwargs): self._log(logging.WARNING, msg, *args, **kwargs)
201
+ def error(self, msg: str, *args, **kwargs): self._log(logging.ERROR, msg, *args, **kwargs)
202
+ def critical(self, msg: str, *args, **kwargs): self._log(logging.CRITICAL, msg, *args, **kwargs)
203
+
204
+ def bind(self, **extra: Any) -> LoggerAdapter:
205
+ if isinstance(self.logger, LoggerAdapter):
206
+ merged = {**self.logger.extra, **extra}
207
+ return LoggerAdapter(self.logger.logger, merged)
208
+ return LoggerAdapter(self.logger, extra)
209
+
210
+ @contextmanager
211
+ def bound(self, **extra: Any):
212
+ adapter = self.bind(**extra)
213
+ yield adapter
214
+
215
+ def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
216
+ if not (self.enable_otel and _OTEL_AVAILABLE and self.tracer):
217
+ # keep API but no-op cleanly
218
+ self.warning("Tracing is disabled or not initialized. Cannot start span.")
219
+ return nullcontext()
198
220
 
199
- def error(self, msg: str, *args, **kwargs):
200
- self.logger.error(msg, *args, **kwargs)
221
+ cm = self.tracer.start_as_current_span(name)
201
222
 
202
- def critical(self, msg: str, *args, **kwargs):
203
- self.logger.critical(msg, *args, **kwargs)
223
+ class _SpanCtx:
224
+ def __enter__(_self):
225
+ span = cm.__enter__()
226
+ if attributes:
227
+ for k, v in attributes.items():
228
+ try:
229
+ span.set_attribute(k, v)
230
+ except Exception:
231
+ pass
232
+ return span
204
233
 
205
- def start_span(self, name: str, attributes: Optional[dict] = None):
206
- """
207
- Starts a span using the configured tracer.
208
- Usage:
209
- with logger.start_span("my-task") as span:
210
- ...
211
- """
212
- if not self.enable_otel or not self.tracer:
213
- self.warning("Tracing is disabled or not initialized. Cannot start span.")
214
- # return dummy context manager
215
- from contextlib import nullcontext
216
- return nullcontext()
234
+ def __exit__(_self, exc_type, exc, tb):
235
+ return cm.__exit__(exc_type, exc, tb)
217
236
 
218
- return self.tracer.start_as_current_span(name)
237
+ return _SpanCtx()
219
238
 
220
- # Use this decorator to trace a function
221
239
  def trace_function(self, span_name: Optional[str] = None):
222
240
  def decorator(func):
223
241
  def wrapper(*args, **kwargs):
224
242
  name = span_name or func.__name__
225
243
  with self.start_span(name):
226
244
  return func(*args, **kwargs)
227
-
228
245
  return wrapper
229
-
230
246
  return decorator
231
247
 
248
+ # -------------------------
249
+ # Internal setup
250
+ # -------------------------
251
+
252
+ def _setup_standard_handlers(self):
253
+ os.makedirs(self.log_dir, exist_ok=True)
254
+ calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
255
+ log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
256
+ key = (self.logger_name, os.path.abspath(log_file_path))
257
+
258
+ formatter = logging.Formatter(
259
+ "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
260
+ datefmt="%Y-%m-%d %H:%M:%S",
261
+ )
262
+ formatter.converter = time.gmtime # UTC timestamps
263
+
264
+ # attach once per process for this logger/file combo
265
+ if key not in self._handler_keys_attached:
266
+ file_handler = RotatingFileHandler(
267
+ log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
268
+ )
269
+ file_handler.setFormatter(formatter)
270
+ self._core_logger.addHandler(file_handler)
271
+
272
+ console_handler = logging.StreamHandler(sys.stdout)
273
+ console_handler.setFormatter(formatter)
274
+ self._core_logger.addHandler(console_handler)
275
+
276
+ self._handler_keys_attached.add(key)
232
277
 
278
+ def _normalize_otlp_endpoint(self, ep: str) -> str:
279
+ if "://" not in ep:
280
+ ep = ("http://" if self.otel_insecure else "https://") + ep
281
+ return ep
282
+
283
+ def _setup_otel_if_needed(self):
284
+ """
285
+ Initialize OTel once per logger_name within the process to avoid
286
+ clobbering providers (important under reloaders).
287
+ """
288
+ if not _OTEL_AVAILABLE:
289
+ self._core_logger.warning("OpenTelemetry not available — skipping OTel setup.")
290
+ return
291
+ if self.logger_name in self._otel_initialized_names:
292
+ # already initialized for this logger name in this process
293
+ self.tracer = trace.get_tracer(self.logger_name)
294
+ return
295
+
296
+ # Create resources
297
+ resource_attrs = {
298
+ "service.name": self.otel_service_name,
299
+ "logger.name": self.logger_name,
300
+ }
301
+ if self.otel_stream_name:
302
+ resource_attrs["log.stream"] = self.otel_stream_name
303
+ resource = Resource.create(resource_attrs)
304
+
305
+ # Respect any existing providers to avoid breaking apps that configured OTel elsewhere
306
+ existing_lp = None
307
+ try:
308
+ existing_lp = get_logger_provider()
309
+ except Exception:
310
+ pass
311
+
312
+ if not isinstance(existing_lp, LoggerProvider):
313
+ self.logger_provider = LoggerProvider(resource=resource)
314
+ set_logger_provider(self.logger_provider)
315
+ else:
316
+ # reuse existing; don’t overwrite global provider
317
+ self.logger_provider = existing_lp # type: ignore
318
+
319
+ existing_tp = None
320
+ try:
321
+ existing_tp = trace.get_tracer_provider()
322
+ except Exception:
323
+ pass
324
+
325
+ if not isinstance(existing_tp, TracerProvider):
326
+ self.tracer_provider = TracerProvider(resource=resource)
327
+ trace.set_tracer_provider(self.tracer_provider)
328
+ else:
329
+ self.tracer_provider = existing_tp # type: ignore
330
+
331
+ endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
332
+
333
+ # Logs exporter + processor (only if we created our own provider)
334
+ if isinstance(self.logger_provider, LoggerProvider):
335
+ try:
336
+ log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
337
+ self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
338
+ except Exception as e:
339
+ self._core_logger.warning(f"Failed to attach OTel log exporter: {e}")
340
+
341
+ # Traces exporter + processor (only if we created our own provider)
342
+ if isinstance(self.tracer_provider, TracerProvider):
343
+ try:
344
+ span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
345
+ self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
346
+ except Exception as e:
347
+ self._core_logger.warning(f"Failed to attach OTel span exporter: {e}")
348
+
349
+ # Attach OTel LoggingHandler once
350
+ if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
351
+ try:
352
+ otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider) # type: ignore
353
+ self._core_logger.addHandler(otel_handler)
354
+ except Exception as e:
355
+ self._core_logger.warning(f"Failed to attach OTel logging handler: {e}")
356
+
357
+ # Tracer handle
358
+ try:
359
+ self.tracer = trace.get_tracer(self.logger_name)
360
+ except Exception:
361
+ self.tracer = None
362
+
363
+ self._otel_initialized_names.add(self.logger_name)
364
+ self._core_logger.info("OpenTelemetry logging/tracing initialized.")
365
+
366
+ # from __future__ import annotations
367
+ #
233
368
  # import logging
234
369
  # import os
235
370
  # import sys
236
371
  # import time
237
- # from typing import Optional
372
+ # from contextlib import contextmanager
373
+ # from logging import LoggerAdapter
374
+ # from logging.handlers import RotatingFileHandler
375
+ # from typing import Optional, Dict, Any
376
+ #
377
+ # # OpenTelemetry imports
378
+ # from opentelemetry import trace
379
+ # from opentelemetry._logs import set_logger_provider
380
+ # from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
381
+ # from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
382
+ # from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
383
+ # from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
384
+ # from opentelemetry.sdk.resources import Resource
385
+ # from opentelemetry.sdk.trace import TracerProvider
386
+ # from opentelemetry.sdk.trace.export import BatchSpanProcessor
387
+ # from opentelemetry.trace import Tracer as OTelTracer
238
388
  #
239
389
  #
240
390
  # class Logger:
241
391
  # """
242
- # Handles the creation, setup, and management of logging functionalities.
243
- #
244
- # This class facilitates logging by creating and managing a logger instance with
245
- # customizable logging directory, name, and file. It ensures logs from a script
246
- # are stored in a well-defined directory and file, and provides various logging
247
- # methods for different log levels. The logger automatically formats and handles
248
- # log messages. Additionally, this class provides a class method to initialize a
249
- # logger with default behaviors.
250
- #
251
- # :ivar log_dir: Path to the directory where log files are stored.
252
- # :type log_dir: str
253
- # :ivar logger_name: Name of the logger instance.
254
- # :type logger_name: str
255
- # :ivar log_file: Base name of the log file.
256
- # :type log_file: str
257
- # :ivar logger: The initialized logger instance used for logging messages.
258
- # :type logger: logging.Logger
392
+ # Handles the creation and management of logging, with optional OpenTelemetry integration.
259
393
  # """
260
394
  #
261
395
  # DEBUG = logging.DEBUG
@@ -264,109 +398,314 @@ class Logger:
264
398
  # ERROR = logging.ERROR
265
399
  # CRITICAL = logging.CRITICAL
266
400
  #
267
- # def __init__(self, log_dir: str, logger_name: str, log_file: str, log_level: int = logging.DEBUG):
268
- # """
269
- # Initialize the Logger instance.
270
- #
271
- # :param log_dir: Directory where logs are stored.
272
- # :param logger_name: Name of the logger instance.
273
- # :param log_file: Base name of the log file.
274
- # :param log_level: Logging level (defaults to DEBUG).
275
- # """
401
+ # def __init__(
402
+ # self,
403
+ # log_dir: str,
404
+ # logger_name: str,
405
+ # log_file: str,
406
+ # log_level: int = logging.DEBUG,
407
+ # enable_otel: bool = False,
408
+ # otel_service_name: Optional[str] = None,
409
+ # otel_stream_name: Optional[str] = None,
410
+ # otel_endpoint: str = "0.0.0.0:4317",
411
+ # otel_insecure: bool = False,
412
+ # ):
276
413
  # self.log_dir = log_dir
277
414
  # self.logger_name = logger_name
278
415
  # self.log_file = log_file
279
416
  # self.log_level = log_level
280
- # self.logger = None
281
417
  #
282
- # self._setup()
418
+ # self.enable_otel = enable_otel
419
+ # self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
420
+ # self.otel_stream_name = (otel_stream_name or "").strip() or None
421
+ # self.otel_endpoint = otel_endpoint
422
+ # self.otel_insecure = otel_insecure
283
423
  #
284
- # def _setup(self):
285
- # """Set up the logger with file and console handlers."""
286
- # # Ensure the log directory exists
287
- # os.makedirs(self.log_dir, exist_ok=True)
424
+ # self.logger_provider: Optional[LoggerProvider] = None
425
+ # self.tracer_provider: Optional[TracerProvider] = None
426
+ # self.tracer: Optional[OTelTracer] = None
288
427
  #
289
- # # Get the name of the calling script
290
- # calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
428
+ # # Internal logger vs public (adapter) logger
429
+ # self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
430
+ # self.logger: logging.Logger | LoggerAdapter = self._core_logger
291
431
  #
292
- # # Create a log file path
293
- # log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
432
+ # self._setup()
294
433
  #
295
- # # Delete the existing log file if it exists
296
- # if os.path.exists(log_file_path):
297
- # os.remove(log_file_path)
434
+ # # -------------------------
435
+ # # Public API
436
+ # # -------------------------
298
437
  #
299
- # # Create a logger
300
- # self.logger = logging.getLogger(self.logger_name)
301
- # self.logger.setLevel(self.log_level)
438
+ # @classmethod
439
+ # def default_logger(
440
+ # cls,
441
+ # log_dir: str = "./logs/",
442
+ # logger_name: Optional[str] = None,
443
+ # log_file: Optional[str] = None,
444
+ # log_level: int = logging.INFO,
445
+ # enable_otel: bool = False,
446
+ # otel_service_name: Optional[str] = None,
447
+ # otel_stream_name: Optional[str] = None,
448
+ # otel_endpoint: str = "0.0.0.0:4317",
449
+ # otel_insecure: bool = False,
450
+ # ) -> "Logger":
451
+ # try:
452
+ # frame = sys._getframe(1)
453
+ # caller_name = frame.f_globals.get("__name__", "default_logger")
454
+ # except (AttributeError, ValueError):
455
+ # caller_name = "default_logger"
302
456
  #
303
- # # Create a formatter
304
- # formatter = logging.Formatter(
305
- # '[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',
306
- # datefmt='%Y-%m-%d %H:%M:%S'
457
+ # logger_name = logger_name or caller_name
458
+ # log_file = log_file or logger_name
459
+ #
460
+ # return cls(
461
+ # log_dir=log_dir,
462
+ # logger_name=logger_name,
463
+ # log_file=log_file,
464
+ # log_level=log_level,
465
+ # enable_otel=enable_otel,
466
+ # otel_service_name=otel_service_name,
467
+ # otel_stream_name=otel_stream_name,
468
+ # otel_endpoint=otel_endpoint,
469
+ # otel_insecure=otel_insecure,
307
470
  # )
308
471
  #
309
- # formatter.converter = time.localtime # << Set local time explicitly
472
+ # def shutdown(self):
473
+ # """Flush and shut down logging and tracing providers, then Python logging."""
474
+ # try:
475
+ # if self.enable_otel:
476
+ # if self.logger_provider:
477
+ # try:
478
+ # self._core_logger.info("Flushing OpenTelemetry logs...")
479
+ # self.logger_provider.force_flush()
480
+ # except Exception:
481
+ # pass
482
+ # try:
483
+ # self._core_logger.info("Shutting down OpenTelemetry logs...")
484
+ # self.logger_provider.shutdown()
485
+ # except Exception:
486
+ # pass
310
487
  #
311
- # # Create a file handler
312
- # file_handler = logging.FileHandler(log_file_path, delay=True)
313
- # file_handler.setFormatter(formatter)
314
- # self.logger.addHandler(file_handler)
488
+ # if self.tracer_provider:
489
+ # try:
490
+ # self._core_logger.info("Flushing OpenTelemetry traces...")
491
+ # self.tracer_provider.force_flush()
492
+ # except Exception:
493
+ # pass
494
+ # try:
495
+ # self._core_logger.info("Shutting down OpenTelemetry traces...")
496
+ # self.tracer_provider.shutdown()
497
+ # except Exception:
498
+ # pass
499
+ # finally:
500
+ # logging.shutdown()
315
501
  #
316
- # # Create a console handler (optional)
317
- # console_handler = logging.StreamHandler()
318
- # console_handler.setFormatter(formatter)
319
- # self.logger.addHandler(console_handler)
502
+ # def set_level(self, level: int):
503
+ # """Set the logging level for the logger."""
504
+ # self._core_logger.setLevel(level)
320
505
  #
321
- # @classmethod
322
- # def default_logger(
323
- # cls,
324
- # log_dir: str = './logs/',
325
- # logger_name: Optional[str] = None,
326
- # log_file: Optional[str] = None,
327
- # log_level: int = logging.INFO
328
- # ) -> 'Logger':
506
+ # # passthrough convenience methods
507
+ # def _log(self, level: int, msg: str, *args, **kwargs):
508
+ # extra = kwargs.pop("extra", None)
509
+ # if extra is not None:
510
+ # # Always emit via an adapter so extras survive to OTel attributes
511
+ # if isinstance(self.logger, LoggerAdapter):
512
+ # merged = {**self.logger.extra, **extra}
513
+ # LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
514
+ # else:
515
+ # LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
516
+ # else:
517
+ # self.logger.log(level, msg, *args, **kwargs)
518
+ #
519
+ # def debug(self, msg: str, *args, **kwargs):
520
+ # self._log(logging.DEBUG, msg, *args, **kwargs)
521
+ #
522
+ # def info(self, msg: str, *args, **kwargs):
523
+ # self._log(logging.INFO, msg, *args, **kwargs)
524
+ #
525
+ # def warning(self, msg: str, *args, **kwargs):
526
+ # self._log(logging.WARNING, msg, *args, **kwargs)
527
+ #
528
+ # def error(self, msg: str, *args, **kwargs):
529
+ # self._log(logging.ERROR, msg, *args, **kwargs)
530
+ #
531
+ # def critical(self, msg: str, *args, **kwargs):
532
+ # self._log(logging.CRITICAL, msg, *args, **kwargs)
533
+ #
534
+ #
535
+ # def bind(self, **extra: Any) -> LoggerAdapter:
536
+ # """
537
+ # Return a new LoggerAdapter bound with extra context, merging with existing extras if present.
538
+ # Example:
539
+ # api_log = logger.bind(component="api", request_id=req.id)
540
+ # api_log.info("processing")
329
541
  # """
330
- # Class-level method to create a default logger with generic parameters.
542
+ # if isinstance(self.logger, LoggerAdapter):
543
+ # merged = {**self.logger.extra, **extra}
544
+ # return LoggerAdapter(self.logger.logger, merged)
545
+ # return LoggerAdapter(self.logger, extra)
331
546
  #
332
- # :param log_dir: Directory where logs are stored (defaults to './logs/').
333
- # :param logger_name: Name of the logger (defaults to __name__).
334
- # :param log_file: Name of the log file (defaults to logger_name).
335
- # :param log_level: Logging level (defaults to INFO).
336
- # :return: Instance of Logger.
547
+ # @contextmanager
548
+ # def bound(self, **extra: Any):
337
549
  # """
338
- # logger_name = logger_name or __name__
339
- # log_file = log_file or logger_name
340
- # return cls(log_dir=log_dir, logger_name=logger_name, log_file=log_file, log_level=log_level)
550
+ # Context manager that yields a bound adapter for temporary context.
551
+ # Example:
552
+ # with logger.bound(order_id=oid) as log:
553
+ # log.info("starting")
554
+ # ...
555
+ # """
556
+ # adapter = self.bind(**extra)
557
+ # try:
558
+ # yield adapter
559
+ # finally:
560
+ # # nothing to clean up; adapter is ephemeral
561
+ # pass
341
562
  #
342
- # def set_level(self, level: int):
563
+ # def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
343
564
  # """
344
- # Set the logging level for the logger.
565
+ # Start a span as a context manager.
345
566
  #
346
- # :param level: Logging level (e.g., logging.DEBUG, logging.INFO).
567
+ # Usage:
568
+ # with logger.start_span("my-task", {"key": "value"}) as span:
569
+ # ...
347
570
  # """
348
- # self.logger.setLevel(level)
571
+ # if not self.enable_otel or not self.tracer:
572
+ # self.warning("Tracing is disabled or not initialized. Cannot start span.")
573
+ # from contextlib import nullcontext
574
+ # return nullcontext()
349
575
  #
350
- # def debug(self, msg: str, *args, **kwargs):
351
- # """Log a debug message."""
352
- # self.logger.debug(msg, *args, **kwargs)
576
+ # cm = self.tracer.start_as_current_span(name)
353
577
  #
354
- # def info(self, msg: str, *args, **kwargs):
355
- # """Log an info message."""
356
- # self.logger.info(msg, *args, **kwargs)
578
+ # class _SpanCtx:
579
+ # def __enter__(_self):
580
+ # span = cm.__enter__()
581
+ # if attributes:
582
+ # for k, v in attributes.items():
583
+ # try:
584
+ # span.set_attribute(k, v)
585
+ # except Exception:
586
+ # pass
587
+ # return span
357
588
  #
358
- # def warning(self, msg: str, *args, **kwargs):
359
- # """Log a warning message."""
360
- # self.logger.warning(msg, *args, **kwargs)
589
+ # def __exit__(_self, exc_type, exc, tb):
590
+ # return cm.__exit__(exc_type, exc, tb)
361
591
  #
362
- # def error(self, msg: str, *args, **kwargs):
363
- # """
364
- # Log an error message.
592
+ # return _SpanCtx()
593
+ #
594
+ # def trace_function(self, span_name: Optional[str] = None):
595
+ # """Decorator to trace a function with an optional custom span name."""
596
+ # def decorator(func):
597
+ # def wrapper(*args, **kwargs):
598
+ # name = span_name or func.__name__
599
+ # with self.start_span(name):
600
+ # return func(*args, **kwargs)
601
+ # return wrapper
602
+ # return decorator
603
+ #
604
+ # # -------------------------
605
+ # # Internal setup
606
+ # # -------------------------
607
+ #
608
+ # def _setup(self):
609
+ # """Set up core logger, handlers, and optional OTel."""
610
+ # # Configure base logger
611
+ # self._core_logger = logging.getLogger(self.logger_name)
612
+ # self._core_logger.setLevel(self.log_level)
613
+ # self._core_logger.propagate = False
614
+ #
615
+ # # Standard (file + console) handlers
616
+ # self._setup_standard_handlers()
617
+ #
618
+ # # OTel handlers (logs + traces)
619
+ # if self.enable_otel:
620
+ # self._setup_otel_handler()
621
+ #
622
+ # # Public-facing logger (optionally wrapped with adapter extras)
623
+ # if self.enable_otel and self.otel_stream_name:
624
+ # attributes = {
625
+ # "log_stream": self.otel_stream_name,
626
+ # "log_service_name": self.otel_service_name,
627
+ # "logger_name": self.logger_name,
628
+ # }
629
+ # self.logger = LoggerAdapter(self._core_logger, extra=attributes)
630
+ # else:
631
+ # self.logger = self._core_logger
632
+ #
633
+ # def _setup_standard_handlers(self):
634
+ # """Sets up file and console handlers with deduping."""
635
+ # os.makedirs(self.log_dir, exist_ok=True)
636
+ # calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
637
+ # log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
638
+ #
639
+ # formatter = logging.Formatter(
640
+ # "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
641
+ # datefmt="%Y-%m-%d %H:%M:%S",
642
+ # )
643
+ # formatter.converter = time.gmtime # UTC timestamps
644
+ #
645
+ # # File handler (dedupe by filename)
646
+ # if not any(
647
+ # isinstance(h, RotatingFileHandler) and getattr(h, "baseFilename", "") == os.path.abspath(log_file_path)
648
+ # for h in self._core_logger.handlers
649
+ # ):
650
+ # file_handler = RotatingFileHandler(
651
+ # log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
652
+ # )
653
+ # file_handler.setFormatter(formatter)
654
+ # self._core_logger.addHandler(file_handler)
655
+ #
656
+ # # Console handler (dedupe by stream)
657
+ # if not any(
658
+ # isinstance(h, logging.StreamHandler) and getattr(h, "stream", None) is sys.stdout
659
+ # for h in self._core_logger.handlers
660
+ # ):
661
+ # console_handler = logging.StreamHandler(sys.stdout)
662
+ # console_handler.setFormatter(formatter)
663
+ # self._core_logger.addHandler(console_handler)
365
664
  #
366
- # To log exception information, use the `exc_info=True` keyword argument.
665
+ # def _normalize_otlp_endpoint(self, ep: str) -> str:
666
+ # """Ensure OTLP gRPC endpoint has a scheme."""
667
+ # if "://" not in ep:
668
+ # ep = ("http://" if self.otel_insecure else "https://") + ep
669
+ # return ep
670
+ #
671
+ # def _setup_otel_handler(self):
672
+ # """
673
+ # Configure OpenTelemetry providers, exporters, and attach a LoggingHandler.
674
+ # - service.name: used by most backends (incl. OpenObserve) to group streams/services.
675
+ # - log.stream: extra attribute you can filter on in the backend.
367
676
  # """
368
- # self.logger.error(msg, *args, **kwargs)
677
+ # resource_attrs = {
678
+ # "service.name": self.otel_service_name,
679
+ # "logger.name": self.logger_name,
680
+ # }
681
+ # if self.otel_stream_name:
682
+ # resource_attrs["log.stream"] = self.otel_stream_name
683
+ #
684
+ # resource = Resource.create(resource_attrs)
685
+ #
686
+ # # Logs provider
687
+ # self.logger_provider = LoggerProvider(resource=resource)
688
+ # set_logger_provider(self.logger_provider)
689
+ #
690
+ # # Traces provider
691
+ # self.tracer_provider = TracerProvider(resource=resource)
692
+ # trace.set_tracer_provider(self.tracer_provider)
693
+ #
694
+ # endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
695
+ #
696
+ # # Logs exporter + processor
697
+ # log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
698
+ # self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
699
+ #
700
+ # # Traces exporter + processor
701
+ # span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
702
+ # self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
703
+ # self.tracer = trace.get_tracer(self.logger_name, tracer_provider=self.tracer_provider)
704
+ #
705
+ # # Attach OTel LoggingHandler once
706
+ # if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
707
+ # otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)
708
+ # self._core_logger.addHandler(otel_handler)
709
+ #
710
+ # self._core_logger.info("OpenTelemetry logging and tracing enabled and attached.")
369
711
  #
370
- # def critical(self, msg: str, *args, **kwargs):
371
- # """Log a critical message."""
372
- # self.logger.critical(msg, *args, **kwargs)