fluidattacks_core_logging 12.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fluidattacks_core/logging/__init__.py +59 -0
- fluidattacks_core/logging/filters.py +23 -0
- fluidattacks_core/logging/formatters.py +173 -0
- fluidattacks_core/logging/handlers.py +186 -0
- fluidattacks_core/logging/presets.py +40 -0
- fluidattacks_core/logging/py.typed +0 -0
- fluidattacks_core/logging/sources/__init__.py +180 -0
- fluidattacks_core/logging/sources/types.py +13 -0
- fluidattacks_core/logging/sources/utils.py +37 -0
- fluidattacks_core/logging/utils.py +37 -0
- fluidattacks_core_logging-12.0.0.dist-info/METADATA +15 -0
- fluidattacks_core_logging-12.0.0.dist-info/RECORD +13 -0
- fluidattacks_core_logging-12.0.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import logging.config
|
|
5
|
+
import sys
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from fluidattacks_core.logging.presets import DATE_FORMAT, PRODUCT_LOGGING
|
|
9
|
+
from fluidattacks_core.logging.sources.utils import (
|
|
10
|
+
set_commit_ref_name,
|
|
11
|
+
set_commit_sha,
|
|
12
|
+
set_product_environment,
|
|
13
|
+
set_product_id,
|
|
14
|
+
set_product_version,
|
|
15
|
+
)
|
|
16
|
+
from fluidattacks_core.logging.utils import set_telemetry_metadata
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from logging.config import _DictConfigArgs
|
|
20
|
+
from types import TracebackType
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def init_uncaught_exception_logging() -> None:
|
|
24
|
+
logger = logging.getLogger("unhandled")
|
|
25
|
+
|
|
26
|
+
def handle_uncaught_exception(
|
|
27
|
+
exception_type: type[BaseException],
|
|
28
|
+
msg: BaseException,
|
|
29
|
+
traceback: TracebackType | None,
|
|
30
|
+
) -> None:
|
|
31
|
+
if issubclass(exception_type, KeyboardInterrupt):
|
|
32
|
+
sys.__excepthook__(exception_type, msg, traceback)
|
|
33
|
+
return
|
|
34
|
+
|
|
35
|
+
logger.critical(
|
|
36
|
+
"Uncaught exception",
|
|
37
|
+
exc_info=(exception_type, msg, traceback),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
sys.excepthook = handle_uncaught_exception
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def init_logging(preset: _DictConfigArgs | None = None) -> None:
|
|
44
|
+
logging.config.dictConfig(preset or PRODUCT_LOGGING)
|
|
45
|
+
init_uncaught_exception_logging()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
__all__ = [
|
|
49
|
+
"DATE_FORMAT",
|
|
50
|
+
"PRODUCT_LOGGING",
|
|
51
|
+
"init_logging",
|
|
52
|
+
"init_uncaught_exception_logging",
|
|
53
|
+
"set_commit_ref_name",
|
|
54
|
+
"set_commit_sha",
|
|
55
|
+
"set_product_environment",
|
|
56
|
+
"set_product_id",
|
|
57
|
+
"set_product_version",
|
|
58
|
+
"set_telemetry_metadata",
|
|
59
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from fluidattacks_core.logging.sources.utils import get_env_var, get_environment
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class NoProductionFilter(logging.Filter):
|
|
7
|
+
def filter(self, _record: logging.LogRecord) -> bool:
|
|
8
|
+
return get_environment() != "production"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ProductionOnlyFilter(logging.Filter):
|
|
12
|
+
def filter(self, _record: logging.LogRecord) -> bool:
|
|
13
|
+
return get_environment() == "production"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ErrorOnlyFilter(logging.Filter):
|
|
17
|
+
def filter(self, _record: logging.LogRecord) -> bool:
|
|
18
|
+
return _record.levelno >= logging.ERROR
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class EnabledTelemetryFilter(logging.Filter):
|
|
22
|
+
def filter(self, _record: logging.LogRecord) -> bool:
|
|
23
|
+
return str(get_env_var("TELEMETRY_OPT_OUT")).lower() != "true"
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import traceback
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
import simplejson as json
|
|
8
|
+
from pythonjsonlogger.json import JsonFormatter
|
|
9
|
+
|
|
10
|
+
from fluidattacks_core.logging.sources import (
|
|
11
|
+
BatchSource,
|
|
12
|
+
ContainerSource,
|
|
13
|
+
DefaultSource,
|
|
14
|
+
LambdaSource,
|
|
15
|
+
PipelineSource,
|
|
16
|
+
)
|
|
17
|
+
from fluidattacks_core.logging.utils import get_telemetry_metadata
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from fluidattacks_core.logging.sources.types import SourceStrategy
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ColorfulFormatter(logging.Formatter):
|
|
24
|
+
grey: str = "\x1b[38;1m"
|
|
25
|
+
yellow: str = "\x1b[33;1m"
|
|
26
|
+
red: str = "\x1b[31;1m"
|
|
27
|
+
reset: str = "\x1b[0m"
|
|
28
|
+
msg_format: str = "{asctime} [{levelname}] [{name}] {message}"
|
|
29
|
+
|
|
30
|
+
FORMATS = { # noqa: RUF012
|
|
31
|
+
logging.DEBUG: grey + msg_format + reset,
|
|
32
|
+
logging.INFO: msg_format,
|
|
33
|
+
logging.WARNING: yellow + msg_format + reset,
|
|
34
|
+
logging.ERROR: red + msg_format + reset,
|
|
35
|
+
logging.CRITICAL: red + msg_format + reset,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
39
|
+
log_fmt = self.FORMATS.get(record.levelno)
|
|
40
|
+
formatter = logging.Formatter(
|
|
41
|
+
log_fmt,
|
|
42
|
+
datefmt=self.datefmt,
|
|
43
|
+
style="{",
|
|
44
|
+
)
|
|
45
|
+
return formatter.format(record)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class CustomJsonFormatter(JsonFormatter):
|
|
49
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None: # noqa: ANN401
|
|
50
|
+
def json_default(object_: object) -> Any: # noqa: ANN401
|
|
51
|
+
if isinstance(object_, set):
|
|
52
|
+
return list(object_)
|
|
53
|
+
if isinstance(object_, datetime):
|
|
54
|
+
return object_.astimezone(tz=UTC).isoformat()
|
|
55
|
+
if isinstance(object_, float):
|
|
56
|
+
return Decimal(str(object_))
|
|
57
|
+
|
|
58
|
+
if hasattr(object_, "__dict__"):
|
|
59
|
+
try:
|
|
60
|
+
return {k: v for k, v in object_.__dict__.items() if not k.startswith("_")}
|
|
61
|
+
except (TypeError, ValueError, RecursionError):
|
|
62
|
+
return f"<{type(object_).__name__} object>"
|
|
63
|
+
|
|
64
|
+
return object_
|
|
65
|
+
|
|
66
|
+
super().__init__(*args, **kwargs, json_serializer=json.dumps, json_default=json_default)
|
|
67
|
+
|
|
68
|
+
def add_fields(
|
|
69
|
+
self,
|
|
70
|
+
log_record: dict[str, Any],
|
|
71
|
+
record: logging.LogRecord,
|
|
72
|
+
message_dict: dict[str, Any],
|
|
73
|
+
) -> None:
|
|
74
|
+
super().add_fields(log_record, record, message_dict)
|
|
75
|
+
|
|
76
|
+
log_record["timestamp"] = self._get_timestamp(log_record)
|
|
77
|
+
log_record["level"] = log_record.get("level") or record.levelname
|
|
78
|
+
log_record["name"] = record.name
|
|
79
|
+
log_record["file_location"] = f"{record.filename}:{record.lineno}"
|
|
80
|
+
log_record["lineno"] = record.lineno
|
|
81
|
+
|
|
82
|
+
self._add_source_fields(log_record)
|
|
83
|
+
|
|
84
|
+
self._add_default_telemetry_fields(log_record)
|
|
85
|
+
self._add_error_fields(log_record, record)
|
|
86
|
+
self._add_tracing_fields(log_record)
|
|
87
|
+
self._add_extra_fields(log_record)
|
|
88
|
+
|
|
89
|
+
def _get_timestamp(self, log_record: dict[str, Any]) -> int:
|
|
90
|
+
if timestamp := log_record.get("timestamp"):
|
|
91
|
+
if isinstance(timestamp, int):
|
|
92
|
+
return timestamp
|
|
93
|
+
if isinstance(timestamp, str) and timestamp.isdigit():
|
|
94
|
+
return int(timestamp)
|
|
95
|
+
return round(datetime.now(tz=UTC).timestamp() * 1000)
|
|
96
|
+
|
|
97
|
+
def _add_source_fields(self, log_record: dict[str, Any]) -> None:
|
|
98
|
+
"""Add source information to the log record.
|
|
99
|
+
|
|
100
|
+
It includes:
|
|
101
|
+
- Source
|
|
102
|
+
- Service
|
|
103
|
+
- Version
|
|
104
|
+
- Deployment environment
|
|
105
|
+
- Other metadata according to the source
|
|
106
|
+
"""
|
|
107
|
+
supported_sources: list[type[SourceStrategy]] = [
|
|
108
|
+
LambdaSource,
|
|
109
|
+
BatchSource,
|
|
110
|
+
PipelineSource,
|
|
111
|
+
ContainerSource,
|
|
112
|
+
DefaultSource,
|
|
113
|
+
]
|
|
114
|
+
for source in supported_sources:
|
|
115
|
+
if source.detect():
|
|
116
|
+
log_record.update(source.log_metadata())
|
|
117
|
+
break
|
|
118
|
+
|
|
119
|
+
def _add_default_telemetry_fields(self, log_record: dict[str, Any]) -> None:
|
|
120
|
+
"""Add default metadata fields to the log record.
|
|
121
|
+
|
|
122
|
+
Added fields via `set_telemetry_metadata` are included in this step.
|
|
123
|
+
|
|
124
|
+
"""
|
|
125
|
+
for key, value in get_telemetry_metadata().items():
|
|
126
|
+
log_record[key] = value # noqa: PERF403
|
|
127
|
+
|
|
128
|
+
def _add_error_fields(self, log_record: dict[str, Any], record: logging.LogRecord) -> None:
|
|
129
|
+
"""Add error fields to the log record.
|
|
130
|
+
|
|
131
|
+
It includes:
|
|
132
|
+
- `error.kind`
|
|
133
|
+
- `error.message`
|
|
134
|
+
- `error.stack`
|
|
135
|
+
"""
|
|
136
|
+
if record.exc_info:
|
|
137
|
+
if exc_type := record.exc_info[0]:
|
|
138
|
+
log_record["error.kind"] = exc_type.__name__
|
|
139
|
+
if exc_value := record.exc_info[1]:
|
|
140
|
+
log_record["error.message"] = str(exc_value)
|
|
141
|
+
if exc_tb := record.exc_info[2]:
|
|
142
|
+
log_record["error.stack"] = "".join(traceback.format_tb(exc_tb))
|
|
143
|
+
|
|
144
|
+
# Remove duplicated info
|
|
145
|
+
log_record.pop("exc_info", None)
|
|
146
|
+
|
|
147
|
+
def _add_tracing_fields(self, log_record: dict[str, Any]) -> None:
|
|
148
|
+
"""Add tracing fields (by OpenTelemetry) to the log record.
|
|
149
|
+
|
|
150
|
+
It includes:
|
|
151
|
+
- `trace_id`
|
|
152
|
+
- `span_id`
|
|
153
|
+
- `trace_sampled`
|
|
154
|
+
"""
|
|
155
|
+
if log_record.get("otelTraceID") is not None:
|
|
156
|
+
log_record["trace_id"] = log_record.pop("otelTraceID")
|
|
157
|
+
|
|
158
|
+
if log_record.get("otelSpanID") is not None:
|
|
159
|
+
log_record["span_id"] = log_record.pop("otelSpanID")
|
|
160
|
+
|
|
161
|
+
if log_record.get("otelTraceSampled") is not None:
|
|
162
|
+
log_record["trace_sampled"] = log_record.pop("otelTraceSampled")
|
|
163
|
+
|
|
164
|
+
def _add_extra_fields(self, log_record: dict[str, Any]) -> None:
|
|
165
|
+
"""Add fields from `extra` to the log record.
|
|
166
|
+
|
|
167
|
+
It adds fields at the top level, allowing them to override existing fields.
|
|
168
|
+
"""
|
|
169
|
+
if log_record.get("extra") is None:
|
|
170
|
+
log_record.pop("extra", None)
|
|
171
|
+
elif isinstance(log_record.get("extra"), dict):
|
|
172
|
+
log_record.update(log_record["extra"])
|
|
173
|
+
log_record.pop("extra", None)
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import logging
|
|
3
|
+
import sys
|
|
4
|
+
from logging.handlers import HTTPHandler, QueueHandler, QueueListener
|
|
5
|
+
from queue import Queue
|
|
6
|
+
from typing import TextIO
|
|
7
|
+
|
|
8
|
+
import simplejson as json
|
|
9
|
+
|
|
10
|
+
from fluidattacks_core.logging.filters import (
|
|
11
|
+
EnabledTelemetryFilter,
|
|
12
|
+
ErrorOnlyFilter,
|
|
13
|
+
NoProductionFilter,
|
|
14
|
+
ProductionOnlyFilter,
|
|
15
|
+
)
|
|
16
|
+
from fluidattacks_core.logging.formatters import ColorfulFormatter, CustomJsonFormatter
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CustomQueueHandler(QueueHandler):
|
|
20
|
+
"""Class to fix QueueHandler missing exc_info field."""
|
|
21
|
+
|
|
22
|
+
# Defaults must exist before __init__ runs: Handler.__init__ already
|
|
23
|
+
# registers the instance, so a concurrent logging.shutdown() may see it
|
|
24
|
+
# before its listener is attached.
|
|
25
|
+
listener: QueueListener | None = None
|
|
26
|
+
shutting_down: bool = False
|
|
27
|
+
|
|
28
|
+
def _attach_listener(self, listener: QueueListener) -> None:
|
|
29
|
+
"""Install and start `listener`, which must happen after `__init__`.
|
|
30
|
+
|
|
31
|
+
Private: it only orders initialization for subclasses in this module,
|
|
32
|
+
so it is not part of the package's public API.
|
|
33
|
+
|
|
34
|
+
`QueueHandler.__init__` assigns `listener = None` since Python 3.12, so a
|
|
35
|
+
subclass that builds its listener before delegating upwards loses it.
|
|
36
|
+
Handing the listener over here instead makes the order safe by
|
|
37
|
+
construction: this reads `self.queue`, which only exists once
|
|
38
|
+
`QueueHandler.__init__` has run.
|
|
39
|
+
"""
|
|
40
|
+
self.listener = listener
|
|
41
|
+
listener.start()
|
|
42
|
+
|
|
43
|
+
def prepare(self, record: logging.LogRecord) -> logging.LogRecord:
|
|
44
|
+
original_exc_info = record.exc_info
|
|
45
|
+
|
|
46
|
+
# Avoid duplicate exc_text in the log message
|
|
47
|
+
no_exc_record = copy.copy(record)
|
|
48
|
+
no_exc_record.exc_info = None
|
|
49
|
+
prepared = super().prepare(no_exc_record)
|
|
50
|
+
|
|
51
|
+
prepared.exc_info = original_exc_info
|
|
52
|
+
|
|
53
|
+
return prepared
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class DebuggingHandler(logging.StreamHandler[TextIO]):
|
|
57
|
+
"""Logging handler for console environments implemented with `QueueHandler`.
|
|
58
|
+
|
|
59
|
+
Includes:
|
|
60
|
+
- Filters: `NoProductionFilter`
|
|
61
|
+
- Formatter: `ColorfulFormatter`
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(self, stream: TextIO = sys.stderr) -> None:
|
|
65
|
+
super().__init__(stream)
|
|
66
|
+
self.addFilter(NoProductionFilter())
|
|
67
|
+
self.setFormatter(ColorfulFormatter())
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ProductionSyncHandler(logging.StreamHandler[TextIO]):
|
|
71
|
+
"""Logging handler for production environments implemented with `logging.StreamHandler`.
|
|
72
|
+
|
|
73
|
+
Includes:
|
|
74
|
+
- Filters: `ProductionOnlyFilter`
|
|
75
|
+
- Formatter: `CustomJsonFormatter`
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(self, stream: TextIO = sys.stderr) -> None:
|
|
79
|
+
super().__init__(stream)
|
|
80
|
+
self.addFilter(ProductionOnlyFilter())
|
|
81
|
+
self.setFormatter(CustomJsonFormatter())
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class ProductionAsyncHandler(CustomQueueHandler):
|
|
85
|
+
"""Logging handler for production environments implemented with `QueueHandler`.
|
|
86
|
+
|
|
87
|
+
Includes:
|
|
88
|
+
- Filters: `NoBatchFilter`, `ProductionOnlyFilter`
|
|
89
|
+
- Formatter: `CustomJsonFormatter`
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(self, stream: TextIO = sys.stderr) -> None:
|
|
93
|
+
handler = logging.StreamHandler(stream)
|
|
94
|
+
handler.addFilter(ProductionOnlyFilter())
|
|
95
|
+
handler.setFormatter(CustomJsonFormatter())
|
|
96
|
+
|
|
97
|
+
super().__init__(Queue())
|
|
98
|
+
self.shutting_down = False
|
|
99
|
+
|
|
100
|
+
self._attach_listener(QueueListener(self.queue, handler))
|
|
101
|
+
|
|
102
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
103
|
+
if self.shutting_down:
|
|
104
|
+
return
|
|
105
|
+
super().emit(record)
|
|
106
|
+
|
|
107
|
+
def close(self) -> None:
|
|
108
|
+
if self.shutting_down:
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
self.shutting_down = True
|
|
112
|
+
if self.listener:
|
|
113
|
+
self.listener.stop()
|
|
114
|
+
super().close()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class DatadogLogsHandler(HTTPHandler):
|
|
118
|
+
"""Logging handler for sending logs to Datadog using `HTTPHandler` from Python logging."""
|
|
119
|
+
|
|
120
|
+
def __init__(self, service: str, source: str, client_token: str) -> None:
|
|
121
|
+
self.host = "http-intake.logs.datadoghq.com"
|
|
122
|
+
self.url = f"/api/v2/logs?dd-api-key={client_token}&ddsource={source}&service={service}"
|
|
123
|
+
|
|
124
|
+
super().__init__(host=self.host, url=self.url, method="POST", secure=True)
|
|
125
|
+
|
|
126
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
127
|
+
try:
|
|
128
|
+
log_data = self.format(record)
|
|
129
|
+
json_string = json.dumps([log_data])
|
|
130
|
+
|
|
131
|
+
h = self.getConnection(self.host, self.secure)
|
|
132
|
+
h.putrequest(self.method, self.url)
|
|
133
|
+
|
|
134
|
+
h.putheader("Content-Type", "application/json")
|
|
135
|
+
h.putheader("Content-Length", str(len(json_string)))
|
|
136
|
+
h.endheaders()
|
|
137
|
+
|
|
138
|
+
h.send(json_string.encode("utf-8"))
|
|
139
|
+
h.getresponse()
|
|
140
|
+
except Exception: # noqa: BLE001
|
|
141
|
+
self.handleError(record)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class TelemetryAsyncHandler(CustomQueueHandler):
|
|
145
|
+
"""Logging handler for sending logs to telemetry services asynchronously.
|
|
146
|
+
|
|
147
|
+
To enable telemetry, set the `TELEMETRY_OPT_OUT` environment variable different from `true`.
|
|
148
|
+
|
|
149
|
+
Includes:
|
|
150
|
+
- Filters: `ErrorOnlyFilter`, `EnabledTelemetryFilter`
|
|
151
|
+
- Formatter: `CustomJsonFormatter`
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def __init__(self, service: str, source: str, dd_client_token: str) -> None:
|
|
155
|
+
"""Initialize the TelemetryAsyncHandler.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
service: The service name.
|
|
159
|
+
source: The source name.
|
|
160
|
+
dd_client_token: The Datadog Client Token.
|
|
161
|
+
|
|
162
|
+
"""
|
|
163
|
+
handler = DatadogLogsHandler(service, source, dd_client_token)
|
|
164
|
+
handler.setFormatter(CustomJsonFormatter())
|
|
165
|
+
|
|
166
|
+
super().__init__(Queue())
|
|
167
|
+
self.shutting_down = False
|
|
168
|
+
|
|
169
|
+
self._attach_listener(QueueListener(self.queue, handler))
|
|
170
|
+
|
|
171
|
+
self.addFilter(ErrorOnlyFilter())
|
|
172
|
+
self.addFilter(EnabledTelemetryFilter())
|
|
173
|
+
|
|
174
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
175
|
+
if self.shutting_down:
|
|
176
|
+
return
|
|
177
|
+
super().emit(record)
|
|
178
|
+
|
|
179
|
+
def close(self) -> None:
|
|
180
|
+
if self.shutting_down:
|
|
181
|
+
return
|
|
182
|
+
|
|
183
|
+
self.shutting_down = True
|
|
184
|
+
if self.listener:
|
|
185
|
+
self.listener.stop()
|
|
186
|
+
super().close()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from logging.config import _DictConfigArgs
|
|
7
|
+
|
|
8
|
+
# Main formats
|
|
9
|
+
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
|
|
10
|
+
"""
|
|
11
|
+
Default date format for logs.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
PRODUCT_LOGGING: _DictConfigArgs = {
|
|
16
|
+
"version": 1,
|
|
17
|
+
"disable_existing_loggers": False,
|
|
18
|
+
"handlers": {
|
|
19
|
+
"production_handler": {
|
|
20
|
+
"class": "fluidattacks_core.logging.handlers.ProductionSyncHandler",
|
|
21
|
+
"stream": "ext://sys.stderr",
|
|
22
|
+
},
|
|
23
|
+
"debugging_handler": {
|
|
24
|
+
"class": "fluidattacks_core.logging.handlers.DebuggingHandler",
|
|
25
|
+
"stream": "ext://sys.stderr",
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
"root": {
|
|
29
|
+
"handlers": ["production_handler", "debugging_handler"],
|
|
30
|
+
"level": "INFO",
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
"""
|
|
34
|
+
Default logging configuration dict for all the products.
|
|
35
|
+
|
|
36
|
+
Required environment variables:
|
|
37
|
+
- `PRODUCT_ID`
|
|
38
|
+
- `CI_COMMIT_REF_NAME`
|
|
39
|
+
- `CI_COMMIT_SHA`
|
|
40
|
+
"""
|
|
File without changes
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from fluidattacks_core.logging.sources.types import SourceStrategy
|
|
2
|
+
from fluidattacks_core.logging.sources.utils import get_env_var, get_environment, get_version
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class DefaultSource(SourceStrategy):
|
|
6
|
+
@staticmethod
|
|
7
|
+
def detect() -> bool:
|
|
8
|
+
return True
|
|
9
|
+
|
|
10
|
+
@staticmethod
|
|
11
|
+
def log_metadata() -> dict[str, str]:
|
|
12
|
+
return {
|
|
13
|
+
"ddsource": "python",
|
|
14
|
+
"dd.service": get_env_var("PRODUCT_ID") or "unknown",
|
|
15
|
+
"dd.version": get_version(),
|
|
16
|
+
"deployment.environment": get_environment(),
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LambdaSource(SourceStrategy):
|
|
21
|
+
@staticmethod
|
|
22
|
+
def detect() -> bool:
|
|
23
|
+
return get_env_var("AWS_LAMBDA_FUNCTION_NAME") is not None
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def log_metadata() -> dict[str, str]:
|
|
27
|
+
product_id = get_env_var("PRODUCT_ID")
|
|
28
|
+
function_name = get_env_var("AWS_LAMBDA_FUNCTION_NAME")
|
|
29
|
+
service = product_id or function_name or "unknown"
|
|
30
|
+
|
|
31
|
+
trace_header = get_env_var("_X_AMZN_TRACE_ID")
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
"ddsource": "lambda",
|
|
35
|
+
"dd.service": service,
|
|
36
|
+
"dd.version": get_version(),
|
|
37
|
+
"deployment.environment": get_environment(),
|
|
38
|
+
"lambda_metadata.function_name": function_name or "unknown",
|
|
39
|
+
"lambda_metadata.trace_header": trace_header or "unknown",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class BatchSource(SourceStrategy):
|
|
44
|
+
@staticmethod
|
|
45
|
+
def detect() -> bool:
|
|
46
|
+
return get_env_var("AWS_BATCH_JOB_ID") is not None
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def log_metadata() -> dict[str, str]:
|
|
50
|
+
job_id = get_env_var("AWS_BATCH_JOB_ID")
|
|
51
|
+
attempt = get_env_var("AWS_BATCH_JOB_ATTEMPT")
|
|
52
|
+
compute_environment = get_env_var("AWS_BATCH_CE_NAME")
|
|
53
|
+
|
|
54
|
+
product_id = get_env_var("PRODUCT_ID")
|
|
55
|
+
job_definition_name = get_env_var("JOB_DEFINITION_NAME")
|
|
56
|
+
job_name = get_env_var("JOB_NAME")
|
|
57
|
+
job_queue = get_env_var("AWS_BATCH_JQ_NAME")
|
|
58
|
+
service = (
|
|
59
|
+
product_id
|
|
60
|
+
or job_definition_name
|
|
61
|
+
or job_name
|
|
62
|
+
or (f"from-{job_queue}" if job_queue else "unknown")
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
"ddsource": "batch",
|
|
67
|
+
"dd.service": service,
|
|
68
|
+
"dd.version": get_version(),
|
|
69
|
+
"deployment.environment": get_environment(),
|
|
70
|
+
"job_metadata.id": job_id or "unknown",
|
|
71
|
+
"job_metadata.queue": job_queue or "unknown",
|
|
72
|
+
"job_metadata.definition_name": job_definition_name or "unknown",
|
|
73
|
+
"job_metadata.attempt": attempt or "unknown",
|
|
74
|
+
"job_metadata.compute_environment": compute_environment or "unknown",
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class PipelineSource(SourceStrategy):
|
|
79
|
+
@staticmethod
|
|
80
|
+
def detect() -> bool:
|
|
81
|
+
return PipelineSource._get_pipeline_metadata() is not None
|
|
82
|
+
|
|
83
|
+
@staticmethod
|
|
84
|
+
def log_metadata() -> dict[str, str]:
|
|
85
|
+
metadata = PipelineSource._get_pipeline_metadata()
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
"ddsource": "ci",
|
|
89
|
+
"dd.service": get_env_var("PRODUCT_ID") or "unknown",
|
|
90
|
+
"dd.version": get_version(),
|
|
91
|
+
"deployment.environment": get_environment(),
|
|
92
|
+
**(metadata or {}),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _get_pipeline_metadata() -> dict[str, str] | None:
|
|
97
|
+
pipeline = None
|
|
98
|
+
if get_env_var("CI_JOB_ID"):
|
|
99
|
+
pipeline = "gitlab_ci"
|
|
100
|
+
elif get_env_var("CIRCLECI"):
|
|
101
|
+
pipeline = "circleci"
|
|
102
|
+
elif get_env_var("System.JobId"):
|
|
103
|
+
pipeline = "azure_devops"
|
|
104
|
+
elif get_env_var("BUILD_NUMBER"):
|
|
105
|
+
pipeline = "jenkins"
|
|
106
|
+
|
|
107
|
+
if pipeline is None:
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
return {
|
|
111
|
+
"ddsource": f"ci/{pipeline}",
|
|
112
|
+
"pipeline_metadata.type": pipeline,
|
|
113
|
+
**(
|
|
114
|
+
{
|
|
115
|
+
"pipeline_metadata.CI_JOB_ID": get_env_var("CI_JOB_ID") or "unknown",
|
|
116
|
+
"pipeline_metadata.CI_JOB_URL": get_env_var("CI_JOB_URL") or "unknown",
|
|
117
|
+
}
|
|
118
|
+
if pipeline == "gitlab_ci"
|
|
119
|
+
else {}
|
|
120
|
+
),
|
|
121
|
+
**(
|
|
122
|
+
{
|
|
123
|
+
"pipeline_metadata.CIRCLE_BUILD_NUM": get_env_var("CIRCLE_BUILD_NUM")
|
|
124
|
+
or "unknown",
|
|
125
|
+
"pipeline_metadata.CIRCLE_BUILD_URL": get_env_var("CIRCLE_BUILD_URL")
|
|
126
|
+
or "unknown",
|
|
127
|
+
}
|
|
128
|
+
if pipeline == "circleci"
|
|
129
|
+
else {}
|
|
130
|
+
),
|
|
131
|
+
**(
|
|
132
|
+
{
|
|
133
|
+
"pipeline_metadata.System.JobId": get_env_var("System.JobId") or "unknown",
|
|
134
|
+
}
|
|
135
|
+
if pipeline == "azure_devops"
|
|
136
|
+
else {}
|
|
137
|
+
),
|
|
138
|
+
**(
|
|
139
|
+
{
|
|
140
|
+
"pipeline_metadata.BUILD_NUMBER": get_env_var("BUILD_NUMBER") or "unknown",
|
|
141
|
+
"pipeline_metadata.BUILD_ID": get_env_var("BUILD_ID") or "unknown",
|
|
142
|
+
"pipeline_metadata.BUILD_URL": get_env_var("BUILD_URL") or "unknown",
|
|
143
|
+
}
|
|
144
|
+
if pipeline == "jenkins"
|
|
145
|
+
else {}
|
|
146
|
+
),
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class ContainerSource(SourceStrategy):
|
|
151
|
+
@staticmethod
|
|
152
|
+
def detect() -> bool:
|
|
153
|
+
return (
|
|
154
|
+
get_env_var("CONTAINER_IMAGE") is not None
|
|
155
|
+
or get_env_var("CONTAINER_NAME") is not None
|
|
156
|
+
or get_env_var("CONTAINER_IMAGE_PATH") is not None
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def log_metadata() -> dict[str, str]:
|
|
161
|
+
product_id = get_env_var("PRODUCT_ID")
|
|
162
|
+
container_image = get_env_var("CONTAINER_IMAGE")
|
|
163
|
+
service = product_id or container_image or "unknown"
|
|
164
|
+
|
|
165
|
+
return {
|
|
166
|
+
"ddsource": "container",
|
|
167
|
+
"dd.service": service,
|
|
168
|
+
"dd.version": get_version(),
|
|
169
|
+
"deployment.environment": get_environment(),
|
|
170
|
+
"container_metadata.image": container_image or "unknown",
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
__all__ = [
|
|
175
|
+
"BatchSource",
|
|
176
|
+
"ContainerSource",
|
|
177
|
+
"DefaultSource",
|
|
178
|
+
"LambdaSource",
|
|
179
|
+
"PipelineSource",
|
|
180
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SourceStrategy(ABC):
|
|
5
|
+
@staticmethod
|
|
6
|
+
@abstractmethod
|
|
7
|
+
def detect() -> bool:
|
|
8
|
+
"""Detect if the current runtime is using this source."""
|
|
9
|
+
|
|
10
|
+
@staticmethod
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def log_metadata() -> dict[str, str]:
|
|
13
|
+
"""Get the metadata to be added to the log record."""
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def get_env_var(key: str) -> str | None:
|
|
5
|
+
return os.environ.get(key)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_environment() -> str:
|
|
9
|
+
product_environment = os.environ.get("PRODUCT_ENVIRONMENT")
|
|
10
|
+
branch = os.environ.get("CI_COMMIT_REF_NAME", "default")
|
|
11
|
+
return product_environment or ("production" if branch == "trunk" else "development")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_version() -> str:
|
|
15
|
+
product_version = os.environ.get("PRODUCT_VERSION")
|
|
16
|
+
short_commit_sha = os.environ["CI_COMMIT_SHA"][:8] if os.environ.get("CI_COMMIT_SHA") else None
|
|
17
|
+
return product_version or short_commit_sha or "00000000"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_product_id(product_id: str) -> None:
|
|
21
|
+
os.environ["PRODUCT_ID"] = product_id
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def set_product_environment(product_environment: str) -> None:
|
|
25
|
+
os.environ["PRODUCT_ENVIRONMENT"] = product_environment
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def set_product_version(product_version: str) -> None:
|
|
29
|
+
os.environ["PRODUCT_VERSION"] = product_version
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def set_commit_sha(commit_sha: str) -> None:
|
|
33
|
+
os.environ["CI_COMMIT_SHA"] = commit_sha
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def set_commit_ref_name(commit_ref_name: str) -> None:
|
|
37
|
+
os.environ["CI_COMMIT_REF_NAME"] = commit_ref_name
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
DEFAULT_TELEMETRY_METADATA: dict[str, Any] = {}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def set_telemetry_metadata(config: dict[str, Any]) -> None:
|
|
8
|
+
DEFAULT_TELEMETRY_METADATA.update(config)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_telemetry_metadata() -> dict[str, Any]:
|
|
12
|
+
return DEFAULT_TELEMETRY_METADATA
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def debug_logs() -> None:
|
|
16
|
+
"""Test all the log levels in the root logger and a custom logger."""
|
|
17
|
+
root_logger = logging.getLogger()
|
|
18
|
+
|
|
19
|
+
root_logger.debug("This is a debug log")
|
|
20
|
+
root_logger.info("This is an info log")
|
|
21
|
+
root_logger.warning("This is a warning log")
|
|
22
|
+
root_logger.error("This is an error log")
|
|
23
|
+
root_logger.critical("This is a critical log")
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("test-logger")
|
|
26
|
+
logger.debug("This is a debug log")
|
|
27
|
+
logger.info("This is an info log")
|
|
28
|
+
logger.warning("This is a warning log")
|
|
29
|
+
logger.error("This is an error log")
|
|
30
|
+
logger.critical("This is a critical log")
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
msg = "Missing key"
|
|
34
|
+
raise KeyError(msg) # noqa: TRY301
|
|
35
|
+
except KeyError as e:
|
|
36
|
+
root_logger.exception(e) # noqa:TRY401
|
|
37
|
+
logger.exception(e) # noqa:TRY401
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fluidattacks_core_logging
|
|
3
|
+
Version: 12.0.0
|
|
4
|
+
Summary: Fluid Attacks Core Logging Library
|
|
5
|
+
Author-email: Development <development@fluidattacks.com>
|
|
6
|
+
License: MPL-2.0
|
|
7
|
+
Classifier: Development Status :: 1 - Planning
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Requires-Dist: python-json-logger>=3.3.0
|
|
15
|
+
Requires-Dist: simplejson>=3.20.1
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
fluidattacks_core/logging/__init__.py,sha256=y6D12LrvrsMwaveQn5C4Em3RyeS6mP6E9fRpq7gqS4o,1546
|
|
2
|
+
fluidattacks_core/logging/filters.py,sha256=jEJkrvFeoQxCJ9vI8rd4N1FWXTbPdiTlv-7kriB8dQ0,725
|
|
3
|
+
fluidattacks_core/logging/formatters.py,sha256=SCA4k9XvHJknmkTv63uiCBU31VrWOsgA7llEXYYj1uQ,6063
|
|
4
|
+
fluidattacks_core/logging/handlers.py,sha256=NkJjz-y9iVtX8702sXUHPWOsPaEfpua53C_8CVxFNUU,6003
|
|
5
|
+
fluidattacks_core/logging/presets.py,sha256=KU6d6PI61kklJ_o7NgAzU1DahEPM0KwwjTYHo2naHv8,939
|
|
6
|
+
fluidattacks_core/logging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
fluidattacks_core/logging/utils.py,sha256=jbAcwr0L6iPsId3dYEp-vAbtFex2UUU2l2iIk1F60BE,1115
|
|
8
|
+
fluidattacks_core/logging/sources/__init__.py,sha256=bUOsNhmBq9WHLvPxCW3G0Az4D8Wunp0mCCqEYlCsjsI,5937
|
|
9
|
+
fluidattacks_core/logging/sources/types.py,sha256=ial-ASN6e0EV7XeetDknjcMx_HjqL6ilFwzNwqw5YyY,338
|
|
10
|
+
fluidattacks_core/logging/sources/utils.py,sha256=tMoEVqrkVtqMuYtjgG6ih0HOjHyN3tn9G7nZhNoo6IU,1102
|
|
11
|
+
fluidattacks_core_logging-12.0.0.dist-info/METADATA,sha256=k00Ej-sw9ti9g8Ar6dS63fA7IanoMeoKxc3uT0GRqBU,608
|
|
12
|
+
fluidattacks_core_logging-12.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
13
|
+
fluidattacks_core_logging-12.0.0.dist-info/RECORD,,
|