sibi-dst 2025.8.8__tar.gz → 2025.8.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/PKG-INFO +1 -1
  2. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/pyproject.toml +1 -1
  3. sibi_dst-2025.8.9/sibi_dst/utils/log_utils.py +290 -0
  4. sibi_dst-2025.8.8/sibi_dst/utils/log_utils.py +0 -365
  5. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/README.md +0 -0
  6. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/__init__.py +0 -0
  7. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/__init__.py +0 -0
  8. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/_artifact_updater_async.py +0 -0
  9. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/_artifact_updater_threaded.py +0 -0
  10. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/_df_helper.py +0 -0
  11. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/_parquet_artifact.py +0 -0
  12. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/_parquet_reader.py +0 -0
  13. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/__init__.py +0 -0
  14. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/http/__init__.py +0 -0
  15. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/http/_http_config.py +0 -0
  16. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/parquet/__init__.py +0 -0
  17. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/parquet/_parquet_options.py +0 -0
  18. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/__init__.py +0 -0
  19. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py +0 -0
  20. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py +0 -0
  21. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py +0 -0
  22. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py +0 -0
  23. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_model_registry.py +0 -0
  24. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py +0 -0
  25. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/core/__init__.py +0 -0
  26. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/core/_defaults.py +0 -0
  27. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/core/_filter_handler.py +0 -0
  28. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/core/_params_config.py +0 -0
  29. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/core/_query_config.py +0 -0
  30. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/df_helper/data_cleaner.py +0 -0
  31. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/geopy_helper/__init__.py +0 -0
  32. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/geopy_helper/geo_location_service.py +0 -0
  33. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/geopy_helper/utils.py +0 -0
  34. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/__init__.py +0 -0
  35. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/base_osm_map.py +0 -0
  36. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/basemaps/__init__.py +0 -0
  37. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/basemaps/calendar_html.py +0 -0
  38. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/basemaps/route_map_plotter.py +0 -0
  39. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/basemaps/router_plotter.py +0 -0
  40. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/route_path_builder.py +0 -0
  41. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/osmnx_helper/utils.py +0 -0
  42. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/tests/__init__.py +0 -0
  43. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/tests/test_data_wrapper_class.py +0 -0
  44. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/__init__.py +0 -0
  45. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/async_utils.py +0 -0
  46. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/base.py +0 -0
  47. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/boilerplate/__init__.py +0 -0
  48. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/boilerplate/base_data_artifact.py +0 -0
  49. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/boilerplate/base_data_cube.py +0 -0
  50. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/business_days.py +0 -0
  51. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/clickhouse_writer.py +0 -0
  52. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/credentials.py +0 -0
  53. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/data_from_http_source.py +0 -0
  54. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/data_utils.py +0 -0
  55. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/data_wrapper.py +0 -0
  56. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/date_utils.py +0 -0
  57. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/df_utils.py +0 -0
  58. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/file_age_checker.py +0 -0
  59. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/file_utils.py +0 -0
  60. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/filepath_generator.py +0 -0
  61. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/iceberg_saver.py +0 -0
  62. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/manifest_manager.py +0 -0
  63. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/parquet_saver.py +0 -0
  64. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/periods.py +0 -0
  65. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/phone_formatter.py +0 -0
  66. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/progress/__init__.py +0 -0
  67. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/progress/jobs.py +0 -0
  68. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/progress/sse_runner.py +0 -0
  69. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/storage_config.py +0 -0
  70. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/storage_hive.py +0 -0
  71. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/storage_manager.py +0 -0
  72. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/update_planner.py +0 -0
  73. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/utils/webdav_client.py +0 -0
  74. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/__init__.py +0 -0
  75. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/__init__.py +0 -0
  76. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/_df_helper.py +0 -0
  77. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/__init__.py +0 -0
  78. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlalchemy/__init__.py +0 -0
  79. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlalchemy/_db_connection.py +0 -0
  80. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlalchemy/_io_dask.py +0 -0
  81. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlalchemy/_load_from_db.py +0 -0
  82. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlalchemy/_model_builder.py +0 -0
  83. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlmodel/__init__.py +0 -0
  84. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlmodel/_db_connection.py +0 -0
  85. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlmodel/_io_dask.py +0 -0
  86. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlmodel/_load_from_db.py +0 -0
  87. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/backends/sqlmodel/_model_builder.py +0 -0
  88. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/core/__init__.py +0 -0
  89. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/core/_filter_handler.py +0 -0
  90. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/core/_params_config.py +0 -0
  91. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/df_helper/core/_query_config.py +0 -0
  92. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/utils/__init__.py +0 -0
  93. {sibi_dst-2025.8.8 → sibi_dst-2025.8.9}/sibi_dst/v2/utils/log_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sibi-dst
3
- Version: 2025.8.8
3
+ Version: 2025.8.9
4
4
  Summary: Data Science Toolkit
5
5
  Author: Luis Valverde
6
6
  Author-email: lvalverdeb@gmail.com
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "sibi-dst"
3
- version = "2025.8.8"
3
+ version = "2025.8.9"
4
4
  description = "Data Science Toolkit"
5
5
  authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
6
6
  readme = "README.md"
@@ -0,0 +1,290 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import os
5
+ import sys
6
+ import time
7
+ from contextlib import contextmanager, nullcontext, suppress
8
+ from logging import LoggerAdapter
9
+ from logging.handlers import RotatingFileHandler
10
+ from typing import Optional, Dict, Any, Union
11
+
12
+ # --- OpenTelemetry (optional) ---
13
+ try:
14
+ from opentelemetry import trace
15
+ from opentelemetry._logs import set_logger_provider, get_logger_provider
16
+ from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
17
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
18
+ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
19
+ from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
20
+ from opentelemetry.sdk.resources import Resource
21
+ from opentelemetry.sdk.trace import TracerProvider
22
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
23
+ _OTEL_AVAILABLE = True
24
+ except Exception:
25
+ _OTEL_AVAILABLE = False
26
+
27
+
28
+ class Logger:
29
+ """
30
+ Process-safe logger with optional OpenTelemetry integration.
31
+ Idempotent handler setup. No propagation to root.
32
+ """
33
+
34
+ DEBUG = logging.DEBUG
35
+ INFO = logging.INFO
36
+ WARNING = logging.WARNING
37
+ ERROR = logging.ERROR
38
+ CRITICAL = logging.CRITICAL
39
+
40
+ # idempotency guards per process
41
+ _attached_keys: set[tuple[str, str]] = set() # (logger_name, sink_id)
42
+ _otel_initialized_names: set[str] = set()
43
+
44
+ def __init__(
45
+ self,
46
+ log_dir: str,
47
+ logger_name: str,
48
+ log_file: str,
49
+ log_level: int = logging.INFO,
50
+ enable_otel: bool = False,
51
+ otel_service_name: Optional[str] = None,
52
+ otel_stream_name: Optional[str] = None,
53
+ otel_endpoint: str = "0.0.0.0:4317",
54
+ otel_insecure: bool = False,
55
+ ):
56
+ self.log_dir = log_dir
57
+ self.logger_name = logger_name
58
+ self.log_file = log_file
59
+ self.log_level = log_level
60
+
61
+ self.enable_otel = bool(enable_otel and _OTEL_AVAILABLE)
62
+ self.otel_service_name = (otel_service_name or logger_name or "app").strip()
63
+ self.otel_stream_name = (otel_stream_name or "").strip() or None
64
+ self.otel_endpoint = otel_endpoint
65
+ self.otel_insecure = otel_insecure
66
+
67
+ self.logger_provider = None
68
+ self.tracer_provider = None
69
+ self.tracer = None
70
+
71
+ self._core: logging.Logger = logging.getLogger(self.logger_name)
72
+ self._core.setLevel(self.log_level)
73
+ self._core.propagate = False
74
+
75
+ # public handle (may be LoggerAdapter)
76
+ self.logger: Union[logging.Logger, LoggerAdapter] = self._core
77
+
78
+ self._setup_handlers()
79
+ if self.enable_otel:
80
+ self._setup_otel()
81
+
82
+ if self.enable_otel and self.otel_stream_name:
83
+ attrs = {
84
+ "log_stream": self.otel_stream_name,
85
+ "log_service_name": self.otel_service_name,
86
+ "logger_name": self.logger_name,
87
+ }
88
+ self.logger = LoggerAdapter(self._core, extra=attrs)
89
+
90
+ # ---------------- Public API ----------------
91
+
92
+ @classmethod
93
+ def default_logger(
94
+ cls,
95
+ log_dir: str = "./logs/",
96
+ logger_name: Optional[str] = None,
97
+ log_file: Optional[str] = None,
98
+ log_level: int = logging.INFO,
99
+ enable_otel: bool = False,
100
+ otel_service_name: Optional[str] = None,
101
+ otel_stream_name: Optional[str] = None,
102
+ otel_endpoint: str = "0.0.0.0:4317",
103
+ otel_insecure: bool = False,
104
+ ) -> "Logger":
105
+ try:
106
+ caller_name = sys._getframe(1).f_globals.get("__name__", "default_logger")
107
+ except Exception:
108
+ caller_name = "default_logger"
109
+ logger_name = logger_name or caller_name
110
+ log_file = log_file or logger_name
111
+ return cls(
112
+ log_dir=log_dir,
113
+ logger_name=logger_name,
114
+ log_file=log_file,
115
+ log_level=log_level,
116
+ enable_otel=enable_otel,
117
+ otel_service_name=otel_service_name,
118
+ otel_stream_name=otel_stream_name,
119
+ otel_endpoint=otel_endpoint,
120
+ otel_insecure=otel_insecure,
121
+ )
122
+
123
+ def set_level(self, level: int) -> None:
124
+ self._core.setLevel(level)
125
+
126
+ # passthrough
127
+ def _log(self, level: int, msg: str, *args, **kwargs) -> None:
128
+ extra = kwargs.pop("extra", None)
129
+ if extra is not None:
130
+ if isinstance(self.logger, LoggerAdapter):
131
+ merged = {**self.logger.extra, **extra}
132
+ LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
133
+ else:
134
+ LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
135
+ else:
136
+ self.logger.log(level, msg, *args, **kwargs)
137
+
138
+ def debug(self, msg: str, *a, **k): self._log(logging.DEBUG, msg, *a, **k)
139
+ def info(self, msg: str, *a, **k): self._log(logging.INFO, msg, *a, **k)
140
+ def warning(self, msg: str, *a, **k): self._log(logging.WARNING, msg, *a, **k)
141
+ def error(self, msg: str, *a, **k): self._log(logging.ERROR, msg, *a, **k)
142
+ def critical(self, msg: str, *a, **k): self._log(logging.CRITICAL, msg, *a, **k)
143
+
144
+ def bind(self, **extra: Any) -> LoggerAdapter:
145
+ if isinstance(self.logger, LoggerAdapter):
146
+ return LoggerAdapter(self.logger.logger, {**self.logger.extra, **extra})
147
+ return LoggerAdapter(self.logger, extra)
148
+
149
+ @contextmanager
150
+ def bound(self, **extra: Any):
151
+ yield self.bind(**extra)
152
+
153
+ def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
154
+ if not (self.enable_otel and _OTEL_AVAILABLE and self.tracer):
155
+ return nullcontext()
156
+ cm = self.tracer.start_as_current_span(name)
157
+ class _SpanCtx:
158
+ def __enter__(_self):
159
+ span = cm.__enter__()
160
+ if attributes:
161
+ for k, v in attributes.items():
162
+ with suppress(Exception):
163
+ span.set_attribute(k, v)
164
+ return span
165
+ def __exit__(_self, et, ev, tb):
166
+ return cm.__exit__(et, ev, tb)
167
+ return _SpanCtx()
168
+
169
+ def trace_function(self, span_name: Optional[str] = None):
170
+ def deco(func):
171
+ def wrapper(*a, **k):
172
+ name = span_name or func.__name__
173
+ with self.start_span(name):
174
+ return func(*a, **k)
175
+ return wrapper
176
+ return deco
177
+
178
+ def shutdown(self) -> None:
179
+ try:
180
+ if self.enable_otel and _OTEL_AVAILABLE:
181
+ if self.logger_provider:
182
+ with suppress(Exception):
183
+ self._core.info("Flushing OpenTelemetry logs...")
184
+ self.logger_provider.force_flush()
185
+ with suppress(Exception):
186
+ self._core.info("Shutting down OpenTelemetry logs...")
187
+ self.logger_provider.shutdown()
188
+ if self.tracer_provider:
189
+ with suppress(Exception):
190
+ self._core.info("Flushing OpenTelemetry traces...")
191
+ self.tracer_provider.force_flush()
192
+ with suppress(Exception):
193
+ self._core.info("Shutting down OpenTelemetry traces...")
194
+ self.tracer_provider.shutdown()
195
+ finally:
196
+ for h in list(self._core.handlers):
197
+ with suppress(Exception): h.flush()
198
+ with suppress(Exception): h.close()
199
+ with suppress(Exception): self._core.removeHandler(h)
200
+ logging.shutdown()
201
+
202
+ # ---------------- Internal ----------------
203
+
204
+ def _setup_handlers(self) -> None:
205
+ os.makedirs(self.log_dir, exist_ok=True)
206
+ calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
207
+ log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
208
+ file_key = (self.logger_name, os.path.abspath(log_file_path))
209
+ console_key = (self.logger_name, "__console__")
210
+
211
+ fmt = logging.Formatter(
212
+ "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
213
+ datefmt="%Y-%m-%d %H:%M:%S",
214
+ )
215
+ fmt.converter = time.gmtime # UTC
216
+
217
+ if file_key not in self._attached_keys:
218
+ fh = RotatingFileHandler(log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True)
219
+ fh.setFormatter(fmt)
220
+ self._core.addHandler(fh)
221
+ self._attached_keys.add(file_key)
222
+
223
+ if console_key not in self._attached_keys:
224
+ ch = logging.StreamHandler(sys.stdout)
225
+ ch.setFormatter(fmt)
226
+ self._core.addHandler(ch)
227
+ self._attached_keys.add(console_key)
228
+
229
+ def _normalize_otlp_endpoint(self, ep: str) -> str:
230
+ if "://" not in ep:
231
+ ep = ("http://" if self.otel_insecure else "https://") + ep
232
+ return ep
233
+
234
+ def _setup_otel(self) -> None:
235
+ if not _OTEL_AVAILABLE:
236
+ self._core.warning("OpenTelemetry not available — skipping OTel setup.")
237
+ return
238
+ if self.logger_name in self._otel_initialized_names:
239
+ with suppress(Exception):
240
+ self.tracer = trace.get_tracer(self.logger_name)
241
+ return
242
+
243
+ # resources
244
+ attrs = {"service.name": self.otel_service_name, "logger.name": self.logger_name}
245
+ if self.otel_stream_name:
246
+ attrs["log.stream"] = self.otel_stream_name
247
+ resource = Resource.create(attrs)
248
+
249
+ # providers (reuse if already set globally)
250
+ existing_lp = None
251
+ with suppress(Exception):
252
+ existing_lp = get_logger_provider()
253
+ if getattr(existing_lp, "add_log_record_processor", None):
254
+ self.logger_provider = existing_lp
255
+ else:
256
+ self.logger_provider = LoggerProvider(resource=resource)
257
+ set_logger_provider(self.logger_provider)
258
+
259
+ existing_tp = None
260
+ with suppress(Exception):
261
+ existing_tp = trace.get_tracer_provider()
262
+ if getattr(existing_tp, "add_span_processor", None):
263
+ self.tracer_provider = existing_tp
264
+ else:
265
+ self.tracer_provider = TracerProvider(resource=resource)
266
+ trace.set_tracer_provider(self.tracer_provider)
267
+
268
+ endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
269
+
270
+ # exporters/processors (only if we own the providers we created above)
271
+ if isinstance(self.logger_provider, LoggerProvider):
272
+ with suppress(Exception):
273
+ log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
274
+ self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
275
+
276
+ if isinstance(self.tracer_provider, TracerProvider):
277
+ with suppress(Exception):
278
+ span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
279
+ self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
280
+
281
+ # attach OTel log handler once
282
+ if not any(type(h).__name__ == "LoggingHandler" for h in self._core.handlers):
283
+ with suppress(Exception):
284
+ self._core.addHandler(LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)) # type: ignore
285
+
286
+ with suppress(Exception):
287
+ self.tracer = trace.get_tracer(self.logger_name)
288
+
289
+ self._otel_initialized_names.add(self.logger_name)
290
+ self._core.info("OpenTelemetry logging/tracing initialized.")
@@ -1,365 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import logging
4
- import os
5
- import sys
6
- import time
7
- from contextlib import contextmanager, nullcontext
8
- from logging import LoggerAdapter
9
- from logging.handlers import RotatingFileHandler
10
- from typing import Optional, Dict, Any
11
-
12
- # OpenTelemetry (optional)
13
- try:
14
- from opentelemetry import trace
15
- from opentelemetry._logs import set_logger_provider, get_logger_provider
16
- from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
17
- from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
18
- from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
19
- from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
20
- from opentelemetry.sdk.resources import Resource
21
- from opentelemetry.sdk.trace import TracerProvider
22
- from opentelemetry.sdk.trace.export import BatchSpanProcessor
23
- from opentelemetry.trace import Tracer as OTelTracer
24
- _OTEL_AVAILABLE = True
25
- except Exception:
26
- # OTel is optional; keep class working without it
27
- LoggerProvider = TracerProvider = OTelTracer = object # type: ignore
28
- LoggingHandler = object # type: ignore
29
- _OTEL_AVAILABLE = False
30
-
31
-
32
- class Logger:
33
- """
34
- Process-safe logger with optional OpenTelemetry integration.
35
-
36
- Backward-compatible surface:
37
- - Logger.default_logger(...)
38
- - .debug/.info/.warning/.error/.critical
39
- - .set_level(level), .shutdown()
40
- - .bind(**extra) -> LoggerAdapter, .bound(**extra) ctx manager
41
- - .start_span(name, attributes=None), .trace_function(span_name=None)
42
- """
43
-
44
- DEBUG = logging.DEBUG
45
- INFO = logging.INFO
46
- WARNING = logging.WARNING
47
- ERROR = logging.ERROR
48
- CRITICAL = logging.CRITICAL
49
-
50
- # prevent attaching duplicate handlers per (logger_name, file_path) in process
51
- _handler_keys_attached: set[tuple[str, str]] = set()
52
- _otel_initialized_names: set[str] = set()
53
-
54
- def __init__(
55
- self,
56
- log_dir: str,
57
- logger_name: str,
58
- log_file: str,
59
- log_level: int = logging.DEBUG,
60
- enable_otel: bool = False,
61
- otel_service_name: Optional[str] = None,
62
- otel_stream_name: Optional[str] = None,
63
- otel_endpoint: str = "0.0.0.0:4317",
64
- otel_insecure: bool = False,
65
- ):
66
- self.log_dir = log_dir
67
- self.logger_name = logger_name
68
- self.log_file = log_file
69
- self.log_level = log_level
70
-
71
- self.enable_otel = bool(enable_otel and _OTEL_AVAILABLE)
72
- self.otel_service_name = (otel_service_name or logger_name).strip() or "app"
73
- self.otel_stream_name = (otel_stream_name or "").strip() or None
74
- self.otel_endpoint = otel_endpoint
75
- self.otel_insecure = otel_insecure
76
-
77
- self.logger_provider: Optional[LoggerProvider] = None
78
- self.tracer_provider: Optional[TracerProvider] = None
79
- self.tracer: Optional[OTelTracer] = None
80
-
81
- self._core_logger: logging.Logger = logging.getLogger(self.logger_name)
82
- self._core_logger.setLevel(self.log_level)
83
- self._core_logger.propagate = False
84
-
85
- # public handle (may be adapter)
86
- self.logger: logging.Logger | LoggerAdapter = self._core_logger
87
-
88
- self._setup_standard_handlers()
89
- if self.enable_otel:
90
- self._setup_otel_if_needed()
91
-
92
- # expose adapter with default extras if OTel stream requested
93
- if self.enable_otel and self.otel_stream_name:
94
- attributes = {
95
- "log_stream": self.otel_stream_name,
96
- "log_service_name": self.otel_service_name,
97
- "logger_name": self.logger_name,
98
- }
99
- self.logger = LoggerAdapter(self._core_logger, extra=attributes)
100
-
101
- # -------------------------
102
- # Public API
103
- # -------------------------
104
-
105
- @classmethod
106
- def default_logger(
107
- cls,
108
- log_dir: str = "./logs/",
109
- logger_name: Optional[str] = None,
110
- log_file: Optional[str] = None,
111
- log_level: int = logging.INFO,
112
- enable_otel: bool = False,
113
- otel_service_name: Optional[str] = None,
114
- otel_stream_name: Optional[str] = None,
115
- otel_endpoint: str = "0.0.0.0:4317",
116
- otel_insecure: bool = False,
117
- ) -> "Logger":
118
- try:
119
- frame = sys._getframe(1)
120
- caller_name = frame.f_globals.get("__name__", "default_logger")
121
- except Exception:
122
- caller_name = "default_logger"
123
-
124
- logger_name = logger_name or caller_name
125
- log_file = log_file or logger_name
126
-
127
- return cls(
128
- log_dir=log_dir,
129
- logger_name=logger_name,
130
- log_file=log_file,
131
- log_level=log_level,
132
- enable_otel=enable_otel,
133
- otel_service_name=otel_service_name,
134
- otel_stream_name=otel_stream_name,
135
- otel_endpoint=otel_endpoint,
136
- otel_insecure=otel_insecure,
137
- )
138
-
139
- def shutdown(self):
140
- """Flush/close OTel providers and Python logging handlers."""
141
- try:
142
- if self.enable_otel:
143
- if isinstance(self.logger_provider, LoggerProvider):
144
- try:
145
- self._core_logger.info("Flushing OpenTelemetry logs...")
146
- self.logger_provider.force_flush()
147
- except Exception:
148
- pass
149
- try:
150
- self._core_logger.info("Shutting down OpenTelemetry logs...")
151
- self.logger_provider.shutdown()
152
- except Exception:
153
- pass
154
-
155
- if isinstance(self.tracer_provider, TracerProvider):
156
- try:
157
- self._core_logger.info("Flushing OpenTelemetry traces...")
158
- self.tracer_provider.force_flush()
159
- except Exception:
160
- pass
161
- try:
162
- self._core_logger.info("Shutting down OpenTelemetry traces...")
163
- self.tracer_provider.shutdown()
164
- except Exception:
165
- pass
166
- finally:
167
- # Close our handlers explicitly to release file descriptors
168
- for h in list(self._core_logger.handlers):
169
- try:
170
- h.flush()
171
- except Exception:
172
- pass
173
- try:
174
- h.close()
175
- except Exception:
176
- pass
177
- try:
178
- self._core_logger.removeHandler(h)
179
- except Exception:
180
- pass
181
- logging.shutdown()
182
-
183
- def set_level(self, level: int):
184
- self._core_logger.setLevel(level)
185
-
186
- # passthrough convenience
187
- def _log(self, level: int, msg: str, *args, **kwargs):
188
- extra = kwargs.pop("extra", None)
189
- if extra is not None:
190
- if isinstance(self.logger, LoggerAdapter):
191
- merged = {**self.logger.extra, **extra}
192
- LoggerAdapter(self.logger.logger, merged).log(level, msg, *args, **kwargs)
193
- else:
194
- LoggerAdapter(self.logger, extra).log(level, msg, *args, **kwargs)
195
- else:
196
- self.logger.log(level, msg, *args, **kwargs)
197
-
198
- def debug(self, msg: str, *args, **kwargs): self._log(logging.DEBUG, msg, *args, **kwargs)
199
- def info(self, msg: str, *args, **kwargs): self._log(logging.INFO, msg, *args, **kwargs)
200
- def warning(self, msg: str, *args, **kwargs): self._log(logging.WARNING, msg, *args, **kwargs)
201
- def error(self, msg: str, *args, **kwargs): self._log(logging.ERROR, msg, *args, **kwargs)
202
- def critical(self, msg: str, *args, **kwargs): self._log(logging.CRITICAL, msg, *args, **kwargs)
203
-
204
- def bind(self, **extra: Any) -> LoggerAdapter:
205
- if isinstance(self.logger, LoggerAdapter):
206
- merged = {**self.logger.extra, **extra}
207
- return LoggerAdapter(self.logger.logger, merged)
208
- return LoggerAdapter(self.logger, extra)
209
-
210
- @contextmanager
211
- def bound(self, **extra: Any):
212
- adapter = self.bind(**extra)
213
- yield adapter
214
-
215
- def start_span(self, name: str, attributes: Optional[Dict[str, Any]] = None):
216
- if not (self.enable_otel and _OTEL_AVAILABLE and self.tracer):
217
- # keep API but no-op cleanly
218
- self.warning("Tracing is disabled or not initialized. Cannot start span.")
219
- return nullcontext()
220
-
221
- cm = self.tracer.start_as_current_span(name)
222
-
223
- class _SpanCtx:
224
- def __enter__(_self):
225
- span = cm.__enter__()
226
- if attributes:
227
- for k, v in attributes.items():
228
- try:
229
- span.set_attribute(k, v)
230
- except Exception:
231
- pass
232
- return span
233
-
234
- def __exit__(_self, exc_type, exc, tb):
235
- return cm.__exit__(exc_type, exc, tb)
236
-
237
- return _SpanCtx()
238
-
239
- def trace_function(self, span_name: Optional[str] = None):
240
- def decorator(func):
241
- def wrapper(*args, **kwargs):
242
- name = span_name or func.__name__
243
- with self.start_span(name):
244
- return func(*args, **kwargs)
245
- return wrapper
246
- return decorator
247
-
248
- # -------------------------
249
- # Internal setup
250
- # -------------------------
251
-
252
- def _setup_standard_handlers(self):
253
- os.makedirs(self.log_dir, exist_ok=True)
254
- calling_script = os.path.splitext(os.path.basename(sys.argv[0]))[0]
255
- log_file_path = os.path.join(self.log_dir, f"{self.log_file}_{calling_script}.log")
256
- key = (self.logger_name, os.path.abspath(log_file_path))
257
-
258
- formatter = logging.Formatter(
259
- "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s",
260
- datefmt="%Y-%m-%d %H:%M:%S",
261
- )
262
- formatter.converter = time.gmtime # UTC timestamps
263
-
264
- # attach once per process for this logger/file combo
265
- if key not in self._handler_keys_attached:
266
- file_handler = RotatingFileHandler(
267
- log_file_path, maxBytes=5 * 1024 * 1024, backupCount=5, delay=True
268
- )
269
- file_handler.setFormatter(formatter)
270
- self._core_logger.addHandler(file_handler)
271
-
272
- console_handler = logging.StreamHandler(sys.stdout)
273
- console_handler.setFormatter(formatter)
274
- self._core_logger.addHandler(console_handler)
275
-
276
- self._handler_keys_attached.add(key)
277
-
278
- def _normalize_otlp_endpoint(self, ep: str) -> str:
279
- if "://" not in ep:
280
- ep = ("http://" if self.otel_insecure else "https://") + ep
281
- return ep
282
-
283
- def _setup_otel_if_needed(self):
284
- """
285
- Initialize OTel once per logger_name within the process to avoid
286
- clobbering providers (important under reloaders).
287
- """
288
- if not _OTEL_AVAILABLE:
289
- self._core_logger.warning("OpenTelemetry not available — skipping OTel setup.")
290
- return
291
- if self.logger_name in self._otel_initialized_names:
292
- # already initialized for this logger name in this process
293
- self.tracer = trace.get_tracer(self.logger_name)
294
- return
295
-
296
- # Create resources
297
- resource_attrs = {
298
- "service.name": self.otel_service_name,
299
- "logger.name": self.logger_name,
300
- }
301
- if self.otel_stream_name:
302
- resource_attrs["log.stream"] = self.otel_stream_name
303
- resource = Resource.create(resource_attrs)
304
-
305
- # Respect any existing providers to avoid breaking apps that configured OTel elsewhere
306
- existing_lp = None
307
- try:
308
- existing_lp = get_logger_provider()
309
- except Exception:
310
- pass
311
-
312
- if not isinstance(existing_lp, LoggerProvider):
313
- self.logger_provider = LoggerProvider(resource=resource)
314
- set_logger_provider(self.logger_provider)
315
- else:
316
- # reuse existing; don’t overwrite global provider
317
- self.logger_provider = existing_lp # type: ignore
318
-
319
- existing_tp = None
320
- try:
321
- existing_tp = trace.get_tracer_provider()
322
- except Exception:
323
- pass
324
-
325
- if not isinstance(existing_tp, TracerProvider):
326
- self.tracer_provider = TracerProvider(resource=resource)
327
- trace.set_tracer_provider(self.tracer_provider)
328
- else:
329
- self.tracer_provider = existing_tp # type: ignore
330
-
331
- endpoint = self._normalize_otlp_endpoint(self.otel_endpoint)
332
-
333
- # Logs exporter + processor (only if we created our own provider)
334
- if isinstance(self.logger_provider, LoggerProvider):
335
- try:
336
- log_exporter = OTLPLogExporter(endpoint=endpoint, insecure=self.otel_insecure)
337
- self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
338
- except Exception as e:
339
- self._core_logger.warning(f"Failed to attach OTel log exporter: {e}")
340
-
341
- # Traces exporter + processor (only if we created our own provider)
342
- if isinstance(self.tracer_provider, TracerProvider):
343
- try:
344
- span_exporter = OTLPSpanExporter(endpoint=endpoint, insecure=self.otel_insecure)
345
- self.tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
346
- except Exception as e:
347
- self._core_logger.warning(f"Failed to attach OTel span exporter: {e}")
348
-
349
- # Attach OTel LoggingHandler once
350
- if not any(type(h).__name__ == "LoggingHandler" for h in self._core_logger.handlers):
351
- try:
352
- otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider) # type: ignore
353
- self._core_logger.addHandler(otel_handler)
354
- except Exception as e:
355
- self._core_logger.warning(f"Failed to attach OTel logging handler: {e}")
356
-
357
- # Tracer handle
358
- try:
359
- self.tracer = trace.get_tracer(self.logger_name)
360
- except Exception:
361
- self.tracer = None
362
-
363
- self._otel_initialized_names.add(self.logger_name)
364
- self._core_logger.info("OpenTelemetry logging/tracing initialized.")
365
-
File without changes