hd-logging 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hd-logging might be problematic. Click here for more details.
- hd_logging-1.0.1/PKG-INFO +342 -0
- hd_logging-1.0.1/README.md +309 -0
- hd_logging-1.0.1/pyproject.toml +46 -0
- hd_logging-1.0.1/src/hd_logging/SizeAndTimeLoggingHandler.py +61 -0
- hd_logging-1.0.1/src/hd_logging/__init__.py +40 -0
- hd_logging-1.0.1/src/hd_logging/env_loader.py +92 -0
- hd_logging-1.0.1/src/hd_logging/env_print.py +199 -0
- hd_logging-1.0.1/src/hd_logging/logger.py +113 -0
- hd_logging-1.0.1/src/hd_logging/otlp_formatter.py +128 -0
- hd_logging-1.0.1/src/hd_logging/py.typed +0 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: hd-logging
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: A comprehensive logging library with OpenTelemetry support, environment variable handling, and advanced log rotation
|
|
5
|
+
Keywords: logging,opentelemetry,environment,rotation
|
|
6
|
+
Author: Hackerdogs.ai
|
|
7
|
+
Author-email: Hackerdogs.ai <support@hackerdogs.ai>
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Requires-Dist: colorlog>=6.9.0
|
|
20
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
21
|
+
Requires-Dist: python-ulid>=1.1.0
|
|
22
|
+
Requires-Dist: typing-extensions>=4.13.2
|
|
23
|
+
Requires-Dist: pytest>=7.0.0 ; extra == 'dev'
|
|
24
|
+
Requires-Dist: black>=23.0.0 ; extra == 'dev'
|
|
25
|
+
Requires-Dist: flake8>=6.0.0 ; extra == 'dev'
|
|
26
|
+
Requires-Dist: mypy>=1.0.0 ; extra == 'dev'
|
|
27
|
+
Requires-Python: >=3.8.1
|
|
28
|
+
Project-URL: Homepage, https://github.com/tejaswiredkar/hd-logging
|
|
29
|
+
Project-URL: Issues, https://github.com/tejaswiredkar/hd-logging/issues
|
|
30
|
+
Project-URL: Repository, https://github.com/tejaswiredkar/hd-logging
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# HD Logging
|
|
35
|
+
|
|
36
|
+
A comprehensive Python logging library with OpenTelemetry support, environment variable handling, and advanced log rotation capabilities.
|
|
37
|
+
|
|
38
|
+
## Features
|
|
39
|
+
|
|
40
|
+
- 🎨 **Colorized Console Output** - Beautiful, color-coded log messages
|
|
41
|
+
- 📊 **OpenTelemetry Integration** - JSON format logging with rich metadata
|
|
42
|
+
- 🔒 **Environment Variable Security** - Automatic sensitive data masking
|
|
43
|
+
- 📁 **Advanced Log Rotation** - Size and time-based rotation with compression
|
|
44
|
+
- ⚙️ **Flexible Configuration** - Environment variables and programmatic setup
|
|
45
|
+
- 🚀 **High Performance** - Optimized for production workloads
|
|
46
|
+
- 🔧 **Easy Integration** - Simple setup with powerful features
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
### Using pip
|
|
51
|
+
```bash
|
|
52
|
+
pip install hd-logging
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Using uv (recommended)
|
|
56
|
+
```bash
|
|
57
|
+
uv add hd-logging
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Development Installation
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/tejaswiredkar/hd-logging.git
|
|
63
|
+
cd hd-logging
|
|
64
|
+
uv sync
|
|
65
|
+
uv pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Quick Start
|
|
69
|
+
|
|
70
|
+
### Basic Usage
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from hd_logging import setup_logger
|
|
74
|
+
|
|
75
|
+
# Create a logger with default settings
|
|
76
|
+
logger = setup_logger("my_app")
|
|
77
|
+
|
|
78
|
+
# Log messages
|
|
79
|
+
logger.info("Application started")
|
|
80
|
+
logger.warning("This is a warning")
|
|
81
|
+
logger.error("An error occurred")
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### OpenTelemetry Format
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from hd_logging import setup_logger
|
|
88
|
+
|
|
89
|
+
# Create a logger with OpenTelemetry JSON format
|
|
90
|
+
logger = setup_logger(
|
|
91
|
+
"my_service",
|
|
92
|
+
use_otlp_format=True,
|
|
93
|
+
service_name="my-service",
|
|
94
|
+
environment="production",
|
|
95
|
+
service_version="1.0.0",
|
|
96
|
+
log_file_path="logs/service.log"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Log with custom attributes
|
|
100
|
+
logger.info("User action performed", extra={
|
|
101
|
+
"user_id": "12345",
|
|
102
|
+
"action": "login",
|
|
103
|
+
"ip_address": "192.168.1.1"
|
|
104
|
+
})
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Environment Variable Integration
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from hd_logging import setup_logger, load_env_file
|
|
111
|
+
|
|
112
|
+
# Load environment variables from .env file
|
|
113
|
+
load_env_file()
|
|
114
|
+
|
|
115
|
+
# Logger will automatically use environment variables
|
|
116
|
+
logger = setup_logger("env_configured")
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Configuration
|
|
120
|
+
|
|
121
|
+
### Environment Variables
|
|
122
|
+
|
|
123
|
+
The library supports configuration through environment variables:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
# Log levels
|
|
127
|
+
LOG_LEVEL=INFO # Console and file log level
|
|
128
|
+
LOG_FILE_OTLP_FORMAT=true # Enable OpenTelemetry format
|
|
129
|
+
|
|
130
|
+
# Service information
|
|
131
|
+
SERVICE_NAME=my-service # Service name for OTLP logs
|
|
132
|
+
ENVIRONMENT=production # Environment name
|
|
133
|
+
SERVICE_VERSION=1.0.0 # Service version
|
|
134
|
+
|
|
135
|
+
# Log file settings
|
|
136
|
+
LOG_FILE=logs/app.log # Log file path
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Programmatic Configuration
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from hd_logging import setup_logger
|
|
143
|
+
import logging
|
|
144
|
+
|
|
145
|
+
logger = setup_logger(
|
|
146
|
+
logger_name="my_app",
|
|
147
|
+
log_file_path="logs/app.log",
|
|
148
|
+
log_level_console=logging.INFO,
|
|
149
|
+
log_level_files=logging.DEBUG,
|
|
150
|
+
use_otlp_format=True,
|
|
151
|
+
service_name="my-service",
|
|
152
|
+
environment="production",
|
|
153
|
+
service_version="1.0.0"
|
|
154
|
+
)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Advanced Features
|
|
158
|
+
|
|
159
|
+
### Log Rotation
|
|
160
|
+
|
|
161
|
+
The library includes advanced log rotation with both size and time-based rotation:
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
# Automatic rotation when:
|
|
165
|
+
# - File size exceeds 20MB (configurable)
|
|
166
|
+
# - Daily rotation at midnight
|
|
167
|
+
# - Automatic compression of rotated files
|
|
168
|
+
# - Retention of 7 days (configurable)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Sensitive Data Masking
|
|
172
|
+
|
|
173
|
+
Automatic masking of sensitive environment variables:
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from hd_logging import log_env_vars_with_masking
|
|
177
|
+
|
|
178
|
+
# Logs environment variables with sensitive data masked
|
|
179
|
+
log_env_vars_with_masking()
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Custom Attributes
|
|
183
|
+
|
|
184
|
+
Add rich metadata to your logs:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
logger.info("Order processed", extra={
|
|
188
|
+
"order_id": "ORD-12345",
|
|
189
|
+
"customer_id": "CUST-67890",
|
|
190
|
+
"amount": 99.99,
|
|
191
|
+
"currency": "USD",
|
|
192
|
+
"payment_method": "credit_card"
|
|
193
|
+
})
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## Examples
|
|
197
|
+
|
|
198
|
+
See the `examples/` directory for comprehensive usage examples:
|
|
199
|
+
|
|
200
|
+
- [Basic Usage](examples/basic_usage.py) - Simple logging setup
|
|
201
|
+
- [OpenTelemetry Usage](examples/opentelemetry_usage.py) - JSON format logging
|
|
202
|
+
- [Environment Variables](examples/environment_usage.py) - Environment handling
|
|
203
|
+
- [Advanced Features](examples/advanced_usage.py) - Advanced logging scenarios
|
|
204
|
+
- [Web Application](examples/web_application_example.py) - Web app integration
|
|
205
|
+
|
|
206
|
+
Run examples:
|
|
207
|
+
```bash
|
|
208
|
+
python examples/basic_usage.py
|
|
209
|
+
python examples/opentelemetry_usage.py
|
|
210
|
+
# ... and more
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
## API Reference
|
|
214
|
+
|
|
215
|
+
### setup_logger()
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
def setup_logger(
|
|
219
|
+
logger_name: str,
|
|
220
|
+
log_file_path: Optional[str] = None,
|
|
221
|
+
log_level_console: Optional[int] = None,
|
|
222
|
+
log_level_files: Optional[int] = None,
|
|
223
|
+
use_otlp_format: bool = None,
|
|
224
|
+
service_name: Optional[str] = None,
|
|
225
|
+
environment: Optional[str] = None,
|
|
226
|
+
service_version: Optional[str] = None
|
|
227
|
+
) -> logging.Logger
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
**Parameters:**
|
|
231
|
+
- `logger_name`: Name of the logger
|
|
232
|
+
- `log_file_path`: Path to log file (default: from LOG_FILE env var)
|
|
233
|
+
- `log_level_console`: Console log level (default: from LOG_LEVEL env var)
|
|
234
|
+
- `log_level_files`: File log level (default: from LOG_LEVEL env var)
|
|
235
|
+
- `use_otlp_format`: Enable OpenTelemetry format (default: from LOG_FILE_OTLP_FORMAT env var)
|
|
236
|
+
- `service_name`: Service name for OTLP logs (default: from SERVICE_NAME env var)
|
|
237
|
+
- `environment`: Environment name (default: from ENVIRONMENT env var)
|
|
238
|
+
- `service_version`: Service version (default: from SERVICE_VERSION env var)
|
|
239
|
+
|
|
240
|
+
### Environment Variable Functions
|
|
241
|
+
|
|
242
|
+
```python
|
|
243
|
+
from hd_logging import (
|
|
244
|
+
load_env_file, # Load .env file
|
|
245
|
+
find_env_file, # Find .env file path
|
|
246
|
+
get_env_file_path, # Get .env file path
|
|
247
|
+
log_env_vars_with_masking, # Log env vars with masking
|
|
248
|
+
log_dotenv_vars_with_masking, # Log .env vars with masking
|
|
249
|
+
get_env_vars_with_masking, # Get env vars with masking
|
|
250
|
+
get_dotenv_vars_with_masking # Get .env vars with masking
|
|
251
|
+
)
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
## Log Formats
|
|
255
|
+
|
|
256
|
+
### Standard Format
|
|
257
|
+
```
|
|
258
|
+
2024-01-15T10:30:45Z - my_app - INFO - Application started - [Component: main, Function: main, Line: 15]
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
### OpenTelemetry JSON Format
|
|
262
|
+
```json
|
|
263
|
+
{
|
|
264
|
+
"timestamp": "2024-01-15T10:30:45.123456Z",
|
|
265
|
+
"severityText": "INFO",
|
|
266
|
+
"body": "Application started",
|
|
267
|
+
"attributes": {
|
|
268
|
+
"service.name": "my-service",
|
|
269
|
+
"environment": "production",
|
|
270
|
+
"logger.name": "my_app",
|
|
271
|
+
"component": "main",
|
|
272
|
+
"function": "main",
|
|
273
|
+
"line": 15
|
|
274
|
+
},
|
|
275
|
+
"resource": {
|
|
276
|
+
"host.name": "server-01",
|
|
277
|
+
"os.type": "linux",
|
|
278
|
+
"service.name": "my-service",
|
|
279
|
+
"service.version": "1.0.0",
|
|
280
|
+
"service.instance.id": "01HZ1234567890ABCDEF",
|
|
281
|
+
"environment": "production"
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
## Requirements
|
|
287
|
+
|
|
288
|
+
- Python 3.8+
|
|
289
|
+
- colorlog >= 6.9.0
|
|
290
|
+
- python-dotenv >= 1.0.0
|
|
291
|
+
- ulid-py >= 1.1.0
|
|
292
|
+
|
|
293
|
+
## Development
|
|
294
|
+
|
|
295
|
+
### Setup Development Environment
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
git clone https://github.com/tejaswiredkar/hd-logging.git
|
|
299
|
+
cd hd-logging
|
|
300
|
+
uv sync
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### Run Tests
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
uv run pytest
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### Code Formatting
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
uv run black src/
|
|
313
|
+
uv run flake8 src/
|
|
314
|
+
uv run mypy src/
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
## Contributing
|
|
318
|
+
|
|
319
|
+
1. Fork the repository
|
|
320
|
+
2. Create a feature branch
|
|
321
|
+
3. Make your changes
|
|
322
|
+
4. Add tests for new functionality
|
|
323
|
+
5. Run the test suite
|
|
324
|
+
6. Submit a pull request
|
|
325
|
+
|
|
326
|
+
## License
|
|
327
|
+
|
|
328
|
+
Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details.
|
|
329
|
+
|
|
330
|
+
## Support
|
|
331
|
+
|
|
332
|
+
- 📧 Email: support@hackerdogs.ai
|
|
333
|
+
- 🐛 Issues: [GitHub Issues](https://github.com/tejaswiredkar/hd-logging/issues)
|
|
334
|
+
- 📖 Documentation: [GitHub Wiki](https://github.com/tejaswiredkar/hd-logging/wiki)
|
|
335
|
+
|
|
336
|
+
## Changelog
|
|
337
|
+
|
|
338
|
+
See [CHANGELOG.md](CHANGELOG.md) for a history of changes.
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
342
|
+
Made with ❤️ by [Hackerdogs.ai](https://hackerdogs.ai)
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# HD Logging
|
|
2
|
+
|
|
3
|
+
A comprehensive Python logging library with OpenTelemetry support, environment variable handling, and advanced log rotation capabilities.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- 🎨 **Colorized Console Output** - Beautiful, color-coded log messages
|
|
8
|
+
- 📊 **OpenTelemetry Integration** - JSON format logging with rich metadata
|
|
9
|
+
- 🔒 **Environment Variable Security** - Automatic sensitive data masking
|
|
10
|
+
- 📁 **Advanced Log Rotation** - Size and time-based rotation with compression
|
|
11
|
+
- ⚙️ **Flexible Configuration** - Environment variables and programmatic setup
|
|
12
|
+
- 🚀 **High Performance** - Optimized for production workloads
|
|
13
|
+
- 🔧 **Easy Integration** - Simple setup with powerful features
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
### Using pip
|
|
18
|
+
```bash
|
|
19
|
+
pip install hd-logging
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### Using uv (recommended)
|
|
23
|
+
```bash
|
|
24
|
+
uv add hd-logging
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### Development Installation
|
|
28
|
+
```bash
|
|
29
|
+
git clone https://github.com/tejaswiredkar/hd-logging.git
|
|
30
|
+
cd hd-logging
|
|
31
|
+
uv sync
|
|
32
|
+
uv pip install -e .
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Quick Start
|
|
36
|
+
|
|
37
|
+
### Basic Usage
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from hd_logging import setup_logger
|
|
41
|
+
|
|
42
|
+
# Create a logger with default settings
|
|
43
|
+
logger = setup_logger("my_app")
|
|
44
|
+
|
|
45
|
+
# Log messages
|
|
46
|
+
logger.info("Application started")
|
|
47
|
+
logger.warning("This is a warning")
|
|
48
|
+
logger.error("An error occurred")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### OpenTelemetry Format
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from hd_logging import setup_logger
|
|
55
|
+
|
|
56
|
+
# Create a logger with OpenTelemetry JSON format
|
|
57
|
+
logger = setup_logger(
|
|
58
|
+
"my_service",
|
|
59
|
+
use_otlp_format=True,
|
|
60
|
+
service_name="my-service",
|
|
61
|
+
environment="production",
|
|
62
|
+
service_version="1.0.0",
|
|
63
|
+
log_file_path="logs/service.log"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# Log with custom attributes
|
|
67
|
+
logger.info("User action performed", extra={
|
|
68
|
+
"user_id": "12345",
|
|
69
|
+
"action": "login",
|
|
70
|
+
"ip_address": "192.168.1.1"
|
|
71
|
+
})
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Environment Variable Integration
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from hd_logging import setup_logger, load_env_file
|
|
78
|
+
|
|
79
|
+
# Load environment variables from .env file
|
|
80
|
+
load_env_file()
|
|
81
|
+
|
|
82
|
+
# Logger will automatically use environment variables
|
|
83
|
+
logger = setup_logger("env_configured")
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Configuration
|
|
87
|
+
|
|
88
|
+
### Environment Variables
|
|
89
|
+
|
|
90
|
+
The library supports configuration through environment variables:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# Log levels
|
|
94
|
+
LOG_LEVEL=INFO # Console and file log level
|
|
95
|
+
LOG_FILE_OTLP_FORMAT=true # Enable OpenTelemetry format
|
|
96
|
+
|
|
97
|
+
# Service information
|
|
98
|
+
SERVICE_NAME=my-service # Service name for OTLP logs
|
|
99
|
+
ENVIRONMENT=production # Environment name
|
|
100
|
+
SERVICE_VERSION=1.0.0 # Service version
|
|
101
|
+
|
|
102
|
+
# Log file settings
|
|
103
|
+
LOG_FILE=logs/app.log # Log file path
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Programmatic Configuration
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from hd_logging import setup_logger
|
|
110
|
+
import logging
|
|
111
|
+
|
|
112
|
+
logger = setup_logger(
|
|
113
|
+
logger_name="my_app",
|
|
114
|
+
log_file_path="logs/app.log",
|
|
115
|
+
log_level_console=logging.INFO,
|
|
116
|
+
log_level_files=logging.DEBUG,
|
|
117
|
+
use_otlp_format=True,
|
|
118
|
+
service_name="my-service",
|
|
119
|
+
environment="production",
|
|
120
|
+
service_version="1.0.0"
|
|
121
|
+
)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Advanced Features
|
|
125
|
+
|
|
126
|
+
### Log Rotation
|
|
127
|
+
|
|
128
|
+
The library includes advanced log rotation with both size and time-based rotation:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
# Automatic rotation when:
|
|
132
|
+
# - File size exceeds 20MB (configurable)
|
|
133
|
+
# - Daily rotation at midnight
|
|
134
|
+
# - Automatic compression of rotated files
|
|
135
|
+
# - Retention of 7 days (configurable)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Sensitive Data Masking
|
|
139
|
+
|
|
140
|
+
Automatic masking of sensitive environment variables:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from hd_logging import log_env_vars_with_masking
|
|
144
|
+
|
|
145
|
+
# Logs environment variables with sensitive data masked
|
|
146
|
+
log_env_vars_with_masking()
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### Custom Attributes
|
|
150
|
+
|
|
151
|
+
Add rich metadata to your logs:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
logger.info("Order processed", extra={
|
|
155
|
+
"order_id": "ORD-12345",
|
|
156
|
+
"customer_id": "CUST-67890",
|
|
157
|
+
"amount": 99.99,
|
|
158
|
+
"currency": "USD",
|
|
159
|
+
"payment_method": "credit_card"
|
|
160
|
+
})
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## Examples
|
|
164
|
+
|
|
165
|
+
See the `examples/` directory for comprehensive usage examples:
|
|
166
|
+
|
|
167
|
+
- [Basic Usage](examples/basic_usage.py) - Simple logging setup
|
|
168
|
+
- [OpenTelemetry Usage](examples/opentelemetry_usage.py) - JSON format logging
|
|
169
|
+
- [Environment Variables](examples/environment_usage.py) - Environment handling
|
|
170
|
+
- [Advanced Features](examples/advanced_usage.py) - Advanced logging scenarios
|
|
171
|
+
- [Web Application](examples/web_application_example.py) - Web app integration
|
|
172
|
+
|
|
173
|
+
Run examples:
|
|
174
|
+
```bash
|
|
175
|
+
python examples/basic_usage.py
|
|
176
|
+
python examples/opentelemetry_usage.py
|
|
177
|
+
# ... and more
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## API Reference
|
|
181
|
+
|
|
182
|
+
### setup_logger()
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
def setup_logger(
|
|
186
|
+
logger_name: str,
|
|
187
|
+
log_file_path: Optional[str] = None,
|
|
188
|
+
log_level_console: Optional[int] = None,
|
|
189
|
+
log_level_files: Optional[int] = None,
|
|
190
|
+
use_otlp_format: bool = None,
|
|
191
|
+
service_name: Optional[str] = None,
|
|
192
|
+
environment: Optional[str] = None,
|
|
193
|
+
service_version: Optional[str] = None
|
|
194
|
+
) -> logging.Logger
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
**Parameters:**
|
|
198
|
+
- `logger_name`: Name of the logger
|
|
199
|
+
- `log_file_path`: Path to log file (default: from LOG_FILE env var)
|
|
200
|
+
- `log_level_console`: Console log level (default: from LOG_LEVEL env var)
|
|
201
|
+
- `log_level_files`: File log level (default: from LOG_LEVEL env var)
|
|
202
|
+
- `use_otlp_format`: Enable OpenTelemetry format (default: from LOG_FILE_OTLP_FORMAT env var)
|
|
203
|
+
- `service_name`: Service name for OTLP logs (default: from SERVICE_NAME env var)
|
|
204
|
+
- `environment`: Environment name (default: from ENVIRONMENT env var)
|
|
205
|
+
- `service_version`: Service version (default: from SERVICE_VERSION env var)
|
|
206
|
+
|
|
207
|
+
### Environment Variable Functions
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
from hd_logging import (
|
|
211
|
+
load_env_file, # Load .env file
|
|
212
|
+
find_env_file, # Find .env file path
|
|
213
|
+
get_env_file_path, # Get .env file path
|
|
214
|
+
log_env_vars_with_masking, # Log env vars with masking
|
|
215
|
+
log_dotenv_vars_with_masking, # Log .env vars with masking
|
|
216
|
+
get_env_vars_with_masking, # Get env vars with masking
|
|
217
|
+
get_dotenv_vars_with_masking # Get .env vars with masking
|
|
218
|
+
)
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Log Formats
|
|
222
|
+
|
|
223
|
+
### Standard Format
|
|
224
|
+
```
|
|
225
|
+
2024-01-15T10:30:45Z - my_app - INFO - Application started - [Component: main, Function: main, Line: 15]
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
### OpenTelemetry JSON Format
|
|
229
|
+
```json
|
|
230
|
+
{
|
|
231
|
+
"timestamp": "2024-01-15T10:30:45.123456Z",
|
|
232
|
+
"severityText": "INFO",
|
|
233
|
+
"body": "Application started",
|
|
234
|
+
"attributes": {
|
|
235
|
+
"service.name": "my-service",
|
|
236
|
+
"environment": "production",
|
|
237
|
+
"logger.name": "my_app",
|
|
238
|
+
"component": "main",
|
|
239
|
+
"function": "main",
|
|
240
|
+
"line": 15
|
|
241
|
+
},
|
|
242
|
+
"resource": {
|
|
243
|
+
"host.name": "server-01",
|
|
244
|
+
"os.type": "linux",
|
|
245
|
+
"service.name": "my-service",
|
|
246
|
+
"service.version": "1.0.0",
|
|
247
|
+
"service.instance.id": "01HZ1234567890ABCDEF",
|
|
248
|
+
"environment": "production"
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
## Requirements
|
|
254
|
+
|
|
255
|
+
- Python 3.8+
|
|
256
|
+
- colorlog >= 6.9.0
|
|
257
|
+
- python-dotenv >= 1.0.0
|
|
258
|
+
- ulid-py >= 1.1.0
|
|
259
|
+
|
|
260
|
+
## Development
|
|
261
|
+
|
|
262
|
+
### Setup Development Environment
|
|
263
|
+
|
|
264
|
+
```bash
|
|
265
|
+
git clone https://github.com/tejaswiredkar/hd-logging.git
|
|
266
|
+
cd hd-logging
|
|
267
|
+
uv sync
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
### Run Tests
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
uv run pytest
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
### Code Formatting
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
uv run black src/
|
|
280
|
+
uv run flake8 src/
|
|
281
|
+
uv run mypy src/
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
## Contributing
|
|
285
|
+
|
|
286
|
+
1. Fork the repository
|
|
287
|
+
2. Create a feature branch
|
|
288
|
+
3. Make your changes
|
|
289
|
+
4. Add tests for new functionality
|
|
290
|
+
5. Run the test suite
|
|
291
|
+
6. Submit a pull request
|
|
292
|
+
|
|
293
|
+
## License
|
|
294
|
+
|
|
295
|
+
Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details.
|
|
296
|
+
|
|
297
|
+
## Support
|
|
298
|
+
|
|
299
|
+
- 📧 Email: support@hackerdogs.ai
|
|
300
|
+
- 🐛 Issues: [GitHub Issues](https://github.com/tejaswiredkar/hd-logging/issues)
|
|
301
|
+
- 📖 Documentation: [GitHub Wiki](https://github.com/tejaswiredkar/hd-logging/wiki)
|
|
302
|
+
|
|
303
|
+
## Changelog
|
|
304
|
+
|
|
305
|
+
See [CHANGELOG.md](CHANGELOG.md) for a history of changes.
|
|
306
|
+
|
|
307
|
+
---
|
|
308
|
+
|
|
309
|
+
Made with ❤️ by [Hackerdogs.ai](https://hackerdogs.ai)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hd-logging"
|
|
3
|
+
version = "1.0.1"
|
|
4
|
+
description = "A comprehensive logging library with OpenTelemetry support, environment variable handling, and advanced log rotation"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Hackerdogs.ai", email = "support@hackerdogs.ai" }
|
|
8
|
+
]
|
|
9
|
+
license = { text = "Apache-2.0" }
|
|
10
|
+
requires-python = ">=3.8.1"
|
|
11
|
+
keywords = ["logging", "opentelemetry", "environment", "rotation"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"License :: OSI Approved :: Apache Software License",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.8",
|
|
18
|
+
"Programming Language :: Python :: 3.9",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"colorlog>=6.9.0",
|
|
26
|
+
"python-dotenv>=1.0.0",
|
|
27
|
+
"python-ulid>=1.1.0",
|
|
28
|
+
"typing-extensions>=4.13.2",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest>=7.0.0",
|
|
34
|
+
"black>=23.0.0",
|
|
35
|
+
"flake8>=6.0.0",
|
|
36
|
+
"mypy>=1.0.0",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/tejaswiredkar/hd-logging"
|
|
41
|
+
Repository = "https://github.com/tejaswiredkar/hd-logging"
|
|
42
|
+
Issues = "https://github.com/tejaswiredkar/hd-logging/issues"
|
|
43
|
+
|
|
44
|
+
[build-system]
|
|
45
|
+
requires = ["uv_build>=0.8.22,<0.9.0"]
|
|
46
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Size and Time Rotating Logging Handler with Compression
|
|
3
|
+
Custom logging handler that rotates logs based on both size and time, with automatic compression.
|
|
4
|
+
Supports both plain text and OpenTelemetry JSON formats.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
|
|
8
|
+
import colorlog
|
|
9
|
+
import time
|
|
10
|
+
import os
|
|
11
|
+
import gzip
|
|
12
|
+
from hd_logging.otlp_formatter import OpenTelemetryFormatter
|
|
13
|
+
|
|
14
|
+
# Reference: https://stackoverflow.com/questions/29602352/how-to-mix-logging-handlers-file-timed-and-compress-log-in-the-same-config-f
|
|
15
|
+
class SizeAndTimeLoggingHandler(TimedRotatingFileHandler):
|
|
16
|
+
""" My rotating file hander to compress rotated file """
|
|
17
|
+
def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None,
|
|
18
|
+
delay=0, when='h', interval=1, utc=False, use_otlp_format=False,
|
|
19
|
+
service_name=None, environment=None, service_version=None):
|
|
20
|
+
if maxBytes > 0:
|
|
21
|
+
mode = 'a'
|
|
22
|
+
TimedRotatingFileHandler.__init__(
|
|
23
|
+
self, filename, when, interval, backupCount, encoding, delay, utc)
|
|
24
|
+
self.maxBytes = maxBytes
|
|
25
|
+
self.backupCount = backupCount
|
|
26
|
+
|
|
27
|
+
# OpenTelemetry format support
|
|
28
|
+
self.use_otlp_format = use_otlp_format
|
|
29
|
+
if use_otlp_format:
|
|
30
|
+
from hd_logging.otlp_formatter import OpenTelemetryFormatter
|
|
31
|
+
self.formatter = OpenTelemetryFormatter(
|
|
32
|
+
service_name=service_name or "hd_logging",
|
|
33
|
+
environment=environment or "development",
|
|
34
|
+
service_version=service_version or "1.0.0"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def shouldRollover(self, record):
|
|
38
|
+
""" Determine if rollover should occur. """
|
|
39
|
+
# Check rollover by size
|
|
40
|
+
if self.stream is None: # delay was set...
|
|
41
|
+
self.stream = self._open()
|
|
42
|
+
if self.maxBytes > 0: # are we rolling over?
|
|
43
|
+
msg = "%s\n" % self.format(record)
|
|
44
|
+
self.stream.seek(0, 2) #due to non-posix-compliant Windows feature
|
|
45
|
+
if self.stream.tell() + len(msg) >= self.maxBytes:
|
|
46
|
+
return 1
|
|
47
|
+
# Check rollover by time
|
|
48
|
+
t = int(time.time())
|
|
49
|
+
if t >= self.rolloverAt:
|
|
50
|
+
return 1
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
def rotate(self, source, dest):
|
|
54
|
+
""" Compress rotated log file """
|
|
55
|
+
os.rename(source, dest)
|
|
56
|
+
f_in = open(dest, 'rb')
|
|
57
|
+
f_out = gzip.open("%s.gz" % dest, 'wb')
|
|
58
|
+
f_out.writelines(f_in)
|
|
59
|
+
f_out.close()
|
|
60
|
+
f_in.close()
|
|
61
|
+
os.remove(dest)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HD Logging - A comprehensive logging library with OpenTelemetry support.
|
|
3
|
+
|
|
4
|
+
This package provides advanced logging capabilities including:
|
|
5
|
+
- Colorized console output
|
|
6
|
+
- OpenTelemetry JSON format support
|
|
7
|
+
- Environment variable handling with sensitive data masking
|
|
8
|
+
- Advanced log rotation with compression
|
|
9
|
+
- Size and time-based log rotation
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from .logger import setup_logger
|
|
13
|
+
from .env_loader import load_env_file, find_env_file, get_env_file_path
|
|
14
|
+
from .env_print import (
|
|
15
|
+
get_env_vars_with_masking,
|
|
16
|
+
log_env_vars_with_masking,
|
|
17
|
+
log_dotenv_vars_with_masking,
|
|
18
|
+
get_dotenv_vars_with_masking,
|
|
19
|
+
env_print
|
|
20
|
+
)
|
|
21
|
+
from .otlp_formatter import OpenTelemetryFormatter
|
|
22
|
+
from .SizeAndTimeLoggingHandler import SizeAndTimeLoggingHandler
|
|
23
|
+
|
|
24
|
+
__version__ = "1.0.0"
|
|
25
|
+
__author__ = "Hackerdogs.ai"
|
|
26
|
+
__email__ = "support@hackerdogs.ai"
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"setup_logger",
|
|
30
|
+
"load_env_file",
|
|
31
|
+
"find_env_file",
|
|
32
|
+
"get_env_file_path",
|
|
33
|
+
"get_env_vars_with_masking",
|
|
34
|
+
"log_env_vars_with_masking",
|
|
35
|
+
"log_dotenv_vars_with_masking",
|
|
36
|
+
"get_dotenv_vars_with_masking",
|
|
37
|
+
"env_print",
|
|
38
|
+
"OpenTelemetryFormatter",
|
|
39
|
+
"SizeAndTimeLoggingHandler",
|
|
40
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Environment File Loader
|
|
3
|
+
Simple utility to find and load .env files from common locations.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import glob
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def find_env_file(start_path: Optional[str] = None) -> Optional[Path]:
|
|
13
|
+
"""
|
|
14
|
+
Find .env file in common locations using glob patterns.
|
|
15
|
+
|
|
16
|
+
Searches in:
|
|
17
|
+
1. Current directory
|
|
18
|
+
2. One level up
|
|
19
|
+
3. Two levels up
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
start_path: Starting directory path (defaults to current working directory)
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Path to .env file if found, None otherwise
|
|
26
|
+
"""
|
|
27
|
+
if start_path is None:
|
|
28
|
+
start_path = os.getcwd()
|
|
29
|
+
|
|
30
|
+
# Convert to Path object for easier manipulation
|
|
31
|
+
start = Path(start_path).resolve()
|
|
32
|
+
|
|
33
|
+
# Search patterns: current dir, 1 level up, 2 levels up
|
|
34
|
+
search_paths = [
|
|
35
|
+
start, # Current directory
|
|
36
|
+
start.parent, # One level up
|
|
37
|
+
start.parent.parent, # Two levels up
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
# Use glob to search for .env files in each directory
|
|
41
|
+
for search_path in search_paths:
|
|
42
|
+
if search_path.exists():
|
|
43
|
+
# Use glob to find .env files in this directory
|
|
44
|
+
env_files = glob.glob(str(search_path / ".env"))
|
|
45
|
+
if env_files:
|
|
46
|
+
return Path(env_files[0]) # Return the first .env file found
|
|
47
|
+
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def load_env_file(start_path: Optional[str] = None, override: bool = True) -> bool:
|
|
52
|
+
"""
|
|
53
|
+
Find and load .env file from common locations.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
start_path: Starting directory path (defaults to current working directory)
|
|
57
|
+
override: Whether to override existing environment variables
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
True if .env file was found and loaded, False otherwise
|
|
61
|
+
"""
|
|
62
|
+
try:
|
|
63
|
+
from dotenv import load_dotenv
|
|
64
|
+
|
|
65
|
+
env_file = find_env_file(start_path)
|
|
66
|
+
|
|
67
|
+
if env_file:
|
|
68
|
+
load_dotenv(env_file, override=override)
|
|
69
|
+
return True
|
|
70
|
+
else:
|
|
71
|
+
# Fallback to default behavior
|
|
72
|
+
load_dotenv(override=override)
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
except ImportError:
|
|
76
|
+
print("Warning: python-dotenv not installed. Using system environment variables only.")
|
|
77
|
+
print("Install with: pip install python-dotenv")
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def get_env_file_path(start_path: Optional[str] = None) -> Optional[str]:
|
|
82
|
+
"""
|
|
83
|
+
Get the path to the .env file that would be loaded.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
start_path: Starting directory path (defaults to current working directory)
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
String path to .env file if found, None otherwise
|
|
90
|
+
"""
|
|
91
|
+
env_file = find_env_file(start_path)
|
|
92
|
+
return str(env_file) if env_file else None
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Environment Variable Logger Module
|
|
4
|
+
Simple functions to return and log environment variables with sensitive data masking.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Dict, Any
|
|
12
|
+
from dotenv import dotenv_values, find_dotenv
|
|
13
|
+
|
|
14
|
+
# Add the project root to the Python path
|
|
15
|
+
try:
|
|
16
|
+
project_root = Path(__file__).parent.parent
|
|
17
|
+
sys.path.append(str(project_root))
|
|
18
|
+
except Exception as e:
|
|
19
|
+
print(f"❌ Error setting up project path: {e}")
|
|
20
|
+
|
|
21
|
+
# Logger setup with fallback
|
|
22
|
+
try:
|
|
23
|
+
from hd_logging.logger import setup_logger
|
|
24
|
+
logger = setup_logger(__name__, log_file_path="logs/env_print.log")
|
|
25
|
+
except Exception as e:
|
|
26
|
+
print(f"❌ Error setting up logger: {e}")
|
|
27
|
+
# Fallback to basic logging
|
|
28
|
+
import logging
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
logger.setLevel(logging.INFO)
|
|
31
|
+
if not logger.handlers:
|
|
32
|
+
handler = logging.StreamHandler()
|
|
33
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
34
|
+
handler.setFormatter(formatter)
|
|
35
|
+
logger.addHandler(handler)
|
|
36
|
+
|
|
37
|
+
def mask_sensitive_value(value: str, show_chars: int = 2) -> str:
|
|
38
|
+
"""Mask sensitive values by showing only the first few characters."""
|
|
39
|
+
try:
|
|
40
|
+
if not value or len(value) <= show_chars:
|
|
41
|
+
return "*" * len(value) if value else ""
|
|
42
|
+
return value[:show_chars] + "*" * (len(value) - show_chars)
|
|
43
|
+
except Exception as e:
|
|
44
|
+
logger.error(f"❌ Error masking value: {e}")
|
|
45
|
+
return "***ERROR***"
|
|
46
|
+
|
|
47
|
+
def is_sensitive_variable(var_name: str) -> bool:
|
|
48
|
+
"""Check if an environment variable name contains sensitive keywords."""
|
|
49
|
+
try:
|
|
50
|
+
if not var_name:
|
|
51
|
+
return True
|
|
52
|
+
sensitive_patterns = ['_KEY', 'DATABASE_URL', 'PASSWORD', 'SECRET']
|
|
53
|
+
var_name_upper = var_name.upper()
|
|
54
|
+
return any(pattern in var_name_upper for pattern in sensitive_patterns)
|
|
55
|
+
except Exception as e:
|
|
56
|
+
logger.error(f"❌ Error checking variable sensitivity: {e}")
|
|
57
|
+
return True
|
|
58
|
+
|
|
59
|
+
def get_env_vars_with_masking() -> Dict[str, Any]:
|
|
60
|
+
"""
|
|
61
|
+
Function 1: Return all environment variables with sensitive values masked.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
Dictionary with all environment variables, sensitive ones masked
|
|
65
|
+
"""
|
|
66
|
+
try:
|
|
67
|
+
env_vars = {}
|
|
68
|
+
all_env_vars = dict(os.environ)
|
|
69
|
+
|
|
70
|
+
for var_name, var_value in all_env_vars.items():
|
|
71
|
+
if is_sensitive_variable(var_name):
|
|
72
|
+
env_vars[var_name] = mask_sensitive_value(var_value)
|
|
73
|
+
else:
|
|
74
|
+
env_vars[var_name] = var_value
|
|
75
|
+
|
|
76
|
+
return env_vars
|
|
77
|
+
except Exception as e:
|
|
78
|
+
logger.error(f"❌ Error getting environment variables: {e}")
|
|
79
|
+
return {}
|
|
80
|
+
|
|
81
|
+
def log_env_vars_with_masking():
|
|
82
|
+
"""
|
|
83
|
+
Function 2: Log all environment variables with sensitive data masked.
|
|
84
|
+
"""
|
|
85
|
+
try:
|
|
86
|
+
logger.info("🔍 Logging all environment variables with sensitive data masked...")
|
|
87
|
+
|
|
88
|
+
env_vars = get_env_vars_with_masking()
|
|
89
|
+
sensitive_count = 0
|
|
90
|
+
regular_count = 0
|
|
91
|
+
|
|
92
|
+
for var_name, var_value in env_vars.items():
|
|
93
|
+
if is_sensitive_variable(var_name):
|
|
94
|
+
sensitive_count += 1
|
|
95
|
+
logger.info(f"🔐 {var_name} = {var_value}")
|
|
96
|
+
else:
|
|
97
|
+
regular_count += 1
|
|
98
|
+
logger.info(f"📋 {var_name} = {var_value}")
|
|
99
|
+
|
|
100
|
+
logger.info(f"📊 Summary: {sensitive_count} sensitive, {regular_count} regular, {len(env_vars)} total")
|
|
101
|
+
logger.info("✅ Environment variable logging completed")
|
|
102
|
+
|
|
103
|
+
except Exception as e:
|
|
104
|
+
logger.error(f"❌ Error logging environment variables: {e}")
|
|
105
|
+
|
|
106
|
+
def log_dotenv_vars_with_masking():
|
|
107
|
+
"""
|
|
108
|
+
Function 3: Load, log, and mask variables from a .env file.
|
|
109
|
+
"""
|
|
110
|
+
try:
|
|
111
|
+
dotenv_path = find_dotenv()
|
|
112
|
+
logger.info(f"🔍 Searching for .env file...")
|
|
113
|
+
|
|
114
|
+
if not dotenv_path:
|
|
115
|
+
logger.warning("⚠️ .env file not found in current directory or parent directories.")
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
logger.info(f"📄 Found and reading variables from: {dotenv_path}")
|
|
119
|
+
dotenv_vars = dotenv_values(dotenv_path)
|
|
120
|
+
|
|
121
|
+
if not dotenv_vars:
|
|
122
|
+
logger.info("✅ .env file is empty or could not be parsed.")
|
|
123
|
+
return
|
|
124
|
+
|
|
125
|
+
logger.info(f"📄 Logging {len(dotenv_vars)} variables from .env file...")
|
|
126
|
+
sensitive_count = 0
|
|
127
|
+
regular_count = 0
|
|
128
|
+
|
|
129
|
+
for var_name, var_value in dotenv_vars.items():
|
|
130
|
+
if is_sensitive_variable(var_name):
|
|
131
|
+
sensitive_count += 1
|
|
132
|
+
logger.info(f"🔐 {var_name} = {mask_sensitive_value(var_value)}")
|
|
133
|
+
else:
|
|
134
|
+
regular_count += 1
|
|
135
|
+
logger.info(f"📋 {var_name} = {var_value}")
|
|
136
|
+
|
|
137
|
+
logger.info(f"📊 Summary: {sensitive_count} sensitive, {regular_count} regular, {len(dotenv_vars)} total from .env")
|
|
138
|
+
logger.info("✅ .env file logging completed")
|
|
139
|
+
|
|
140
|
+
except Exception as e:
|
|
141
|
+
logger.error(f"❌ Error logging variables from .env file: {e}")
|
|
142
|
+
|
|
143
|
+
def get_dotenv_vars_with_masking() -> Dict[str, Any]:
|
|
144
|
+
"""
|
|
145
|
+
Function 4: Return all variables from a .env file with sensitive values masked.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Dictionary with all .env variables, sensitive ones masked
|
|
149
|
+
"""
|
|
150
|
+
try:
|
|
151
|
+
dotenv_path = find_dotenv()
|
|
152
|
+
if not dotenv_path:
|
|
153
|
+
logger.warning("⚠️ .env file not found by find_dotenv() for get_dotenv_vars_with_masking.")
|
|
154
|
+
return {}
|
|
155
|
+
|
|
156
|
+
dotenv_vars = dotenv_values(dotenv_path)
|
|
157
|
+
if not dotenv_vars:
|
|
158
|
+
return {}
|
|
159
|
+
|
|
160
|
+
masked_vars = {}
|
|
161
|
+
for var_name, var_value in dotenv_vars.items():
|
|
162
|
+
if is_sensitive_variable(var_name):
|
|
163
|
+
masked_vars[var_name] = mask_sensitive_value(var_value)
|
|
164
|
+
else:
|
|
165
|
+
masked_vars[var_name] = var_value
|
|
166
|
+
|
|
167
|
+
return masked_vars
|
|
168
|
+
|
|
169
|
+
except Exception as e:
|
|
170
|
+
logger.error(f"❌ Error getting variables from .env file: {e}")
|
|
171
|
+
return {}
|
|
172
|
+
|
|
173
|
+
# Convenience aliases
|
|
174
|
+
env_print = log_env_vars_with_masking
|
|
175
|
+
|
|
176
|
+
if __name__ == "__main__":
|
|
177
|
+
# Example usage
|
|
178
|
+
print("🔍 Environment Variable Logger - Simple Usage")
|
|
179
|
+
print("=" * 50)
|
|
180
|
+
|
|
181
|
+
# Function 1: Get all env vars with masking
|
|
182
|
+
print("\n1. Getting all environment variables with masking:")
|
|
183
|
+
env_vars = get_env_vars_with_masking()
|
|
184
|
+
print(f" Found {len(env_vars)} environment variables")
|
|
185
|
+
|
|
186
|
+
# Function 2: Log all env vars with masking
|
|
187
|
+
print("\n2. Logging all environment variables with masking:")
|
|
188
|
+
log_env_vars_with_masking()
|
|
189
|
+
|
|
190
|
+
# Function 3: Log all .env vars with masking
|
|
191
|
+
print("\n3. Logging all variables from .env file with masking:")
|
|
192
|
+
log_dotenv_vars_with_masking()
|
|
193
|
+
|
|
194
|
+
# Function 4: Get all .env vars with masking
|
|
195
|
+
print("\n4. Getting all variables from .env file with masking:")
|
|
196
|
+
dotenv_vars_masked = get_dotenv_vars_with_masking()
|
|
197
|
+
print(f" Found {len(dotenv_vars_masked)} variables in .env file")
|
|
198
|
+
|
|
199
|
+
print("\n✅ Example completed")
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from typing import Optional
|
|
4
|
+
import colorlog
|
|
5
|
+
from hd_logging.SizeAndTimeLoggingHandler import SizeAndTimeLoggingHandler as STLH
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
def setup_logger(
|
|
9
|
+
logger_name: str,
|
|
10
|
+
log_file_path: Optional[str] = None,
|
|
11
|
+
log_level_console: Optional[int] = None,
|
|
12
|
+
log_level_files: Optional[int] = None,
|
|
13
|
+
use_otlp_format: bool = None,
|
|
14
|
+
service_name: Optional[str] = None,
|
|
15
|
+
environment: Optional[str] = None,
|
|
16
|
+
service_version: Optional[str] = None
|
|
17
|
+
) -> logging.Logger:
|
|
18
|
+
"""
|
|
19
|
+
Set up a standardized logger with colorized console output and size+time rotating file handler.
|
|
20
|
+
- Prevents duplicate handlers.
|
|
21
|
+
- Uses ISO 8601 timestamps.
|
|
22
|
+
- Log levels and file path can be set via environment variables or function arguments.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
logger_name (str): Name of the logger (use 'api_service' for main app).
|
|
26
|
+
log_file_path (str, optional): Path to the log file. Defaults to 'hd_shared.log' if not specified.
|
|
27
|
+
log_level_console (int, optional): Console log level. Defaults to LOG_LEVEL env or logging.INFO.
|
|
28
|
+
log_level_files (int, optional): File log level. Defaults to LOG_LEVEL env or logging.INFO.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
logging.Logger: Configured logger instance.
|
|
32
|
+
"""
|
|
33
|
+
# Environment/config defaults
|
|
34
|
+
log_file_path = log_file_path or os.getenv("LOG_FILE", "logs/hd_logging.log")
|
|
35
|
+
env_log_level = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
36
|
+
log_level_console = log_level_console or getattr(logging, env_log_level)
|
|
37
|
+
log_level_files = log_level_files or getattr(logging, env_log_level)
|
|
38
|
+
|
|
39
|
+
# OpenTelemetry format configuration
|
|
40
|
+
if use_otlp_format is None:
|
|
41
|
+
use_otlp_format = os.getenv("LOG_FILE_OTLP_FORMAT", "true").lower() == "true"
|
|
42
|
+
|
|
43
|
+
service_name = service_name or os.getenv("SERVICE_NAME", "hd_logging")
|
|
44
|
+
environment = environment or os.getenv("ENVIRONMENT", "development")
|
|
45
|
+
service_version = service_version or os.getenv("SERVICE_VERSION", "1.0.0")
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(logger_name)
|
|
48
|
+
logger.setLevel(min(log_level_console, log_level_files)) # Set to lowest for capturing all
|
|
49
|
+
|
|
50
|
+
# Prevent duplicate handlers
|
|
51
|
+
if getattr(logger, "_custom_handlers_set", False):
|
|
52
|
+
return logger
|
|
53
|
+
|
|
54
|
+
iso_time_format = "%Y-%m-%dT%H:%M:%S%z"
|
|
55
|
+
# Set the converter for all Formatters to use UTC
|
|
56
|
+
logging.Formatter.converter = time.gmtime
|
|
57
|
+
# File formatter
|
|
58
|
+
file_formatter = logging.Formatter(
|
|
59
|
+
'%(asctime)s - %(name)s - %(levelname)s - %(message)s - [Component: %(module)s, Function: %(funcName)s, Line: %(lineno)d]',
|
|
60
|
+
datefmt=iso_time_format
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Colorized console formatter
|
|
64
|
+
console_formatter = colorlog.ColoredFormatter(
|
|
65
|
+
"%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s - [Component: %(module)s, Function: %(funcName)s, Line: %(lineno)d]",
|
|
66
|
+
log_colors={
|
|
67
|
+
'DEBUG': 'cyan',
|
|
68
|
+
'INFO': 'green',
|
|
69
|
+
'WARNING': 'yellow',
|
|
70
|
+
'ERROR': 'red',
|
|
71
|
+
'CRITICAL': 'bold_red',
|
|
72
|
+
},
|
|
73
|
+
datefmt=iso_time_format
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Console handler
|
|
77
|
+
console_handler = logging.StreamHandler()
|
|
78
|
+
console_handler.setLevel(log_level_console)
|
|
79
|
+
console_handler.setFormatter(console_formatter)
|
|
80
|
+
logger.addHandler(console_handler)
|
|
81
|
+
|
|
82
|
+
# Ensure log directory exists if log_file_path is set
|
|
83
|
+
if log_file_path:
|
|
84
|
+
log_dir = os.path.dirname(log_file_path)
|
|
85
|
+
if log_dir and not os.path.exists(log_dir):
|
|
86
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
87
|
+
# Size and time rotating file handler
|
|
88
|
+
stime_handler = STLH(
|
|
89
|
+
filename=log_file_path,
|
|
90
|
+
when="midnight",
|
|
91
|
+
interval=1,
|
|
92
|
+
backupCount=7,
|
|
93
|
+
maxBytes=20_000_000,
|
|
94
|
+
use_otlp_format=use_otlp_format,
|
|
95
|
+
service_name=service_name,
|
|
96
|
+
environment=environment,
|
|
97
|
+
service_version=service_version
|
|
98
|
+
)
|
|
99
|
+
stime_handler.setLevel(log_level_files)
|
|
100
|
+
|
|
101
|
+
# Set formatter based on OTLP format preference
|
|
102
|
+
if use_otlp_format:
|
|
103
|
+
# OTLP formatter is already set in the handler
|
|
104
|
+
pass
|
|
105
|
+
else:
|
|
106
|
+
stime_handler.setFormatter(file_formatter)
|
|
107
|
+
|
|
108
|
+
logger.addHandler(stime_handler)
|
|
109
|
+
|
|
110
|
+
# Mark as configured to prevent duplicate handlers
|
|
111
|
+
logger._custom_handlers_set = True
|
|
112
|
+
|
|
113
|
+
return logger
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OpenTelemetry Log Formatter
|
|
3
|
+
Converts Python logging records to OpenTelemetry JSON format.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import socket
|
|
9
|
+
import platform
|
|
10
|
+
from ulid import ULID
|
|
11
|
+
import time
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from typing import Dict, Any, Optional
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class OpenTelemetryFormatter:
|
|
17
|
+
"""Formats log records to OpenTelemetry JSON format."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, service_name: str = "hd_logging",
|
|
20
|
+
environment: str = "development", service_version: str = "1.0.0"):
|
|
21
|
+
self.service_name = service_name
|
|
22
|
+
self.environment = environment
|
|
23
|
+
self.service_version = service_version
|
|
24
|
+
self.resource_attributes = self._get_resource_attributes()
|
|
25
|
+
|
|
26
|
+
def _get_resource_attributes(self) -> Dict[str, str]:
|
|
27
|
+
"""Get resource attributes for the service."""
|
|
28
|
+
return {
|
|
29
|
+
"host.name": self._get_hostname(),
|
|
30
|
+
"os.type": self._get_os_type(),
|
|
31
|
+
"service.name": self.service_name,
|
|
32
|
+
"service.version": self.service_version,
|
|
33
|
+
"service.instance.id": str(ULID()),
|
|
34
|
+
"environment": self.environment
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
def _get_hostname(self) -> str:
|
|
38
|
+
"""Get the hostname of the system."""
|
|
39
|
+
try:
|
|
40
|
+
return socket.gethostname()
|
|
41
|
+
except:
|
|
42
|
+
return "unknown"
|
|
43
|
+
|
|
44
|
+
def _get_os_type(self) -> str:
|
|
45
|
+
"""Get the operating system type."""
|
|
46
|
+
return platform.system().lower()
|
|
47
|
+
|
|
48
|
+
def _format_timestamp(self, timestamp: float) -> str:
|
|
49
|
+
"""Format timestamp to ISO 8601 format."""
|
|
50
|
+
return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat()
|
|
51
|
+
|
|
52
|
+
def _extract_attributes(self, record) -> Dict[str, Any]:
|
|
53
|
+
"""Extract attributes from the log record."""
|
|
54
|
+
# Standard logging attributes
|
|
55
|
+
attributes = {
|
|
56
|
+
"service.name": self.service_name,
|
|
57
|
+
"environment": self.environment,
|
|
58
|
+
"logger.name": record.name,
|
|
59
|
+
"component": getattr(record, 'module', 'unknown'),
|
|
60
|
+
"function": getattr(record, 'funcName', 'unknown'),
|
|
61
|
+
"line": getattr(record, 'lineno', 0),
|
|
62
|
+
"thread.name": getattr(record, 'threadName', 'unknown'),
|
|
63
|
+
"process.name": getattr(record, 'processName', 'unknown'),
|
|
64
|
+
"process.id": getattr(record, 'process', 0)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
# Add any custom attributes from the record (otlp_attributes)
|
|
68
|
+
if hasattr(record, 'otlp_attributes'):
|
|
69
|
+
attributes.update(record.otlp_attributes)
|
|
70
|
+
|
|
71
|
+
# Add ALL custom attributes from extra parameter
|
|
72
|
+
# Get all attributes from the record and filter out standard logging ones
|
|
73
|
+
standard_attrs = {
|
|
74
|
+
'args', 'created', 'exc_info', 'exc_text', 'filename', 'funcName',
|
|
75
|
+
'levelname', 'levelno', 'lineno', 'module', 'msecs', 'msg', 'name',
|
|
76
|
+
'pathname', 'process', 'processName', 'relativeCreated', 'stack_info',
|
|
77
|
+
'taskName', 'thread', 'threadName', 'getMessage', 'otlp_attributes'
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
for attr_name in dir(record):
|
|
81
|
+
if not attr_name.startswith('_') and attr_name not in standard_attrs:
|
|
82
|
+
try:
|
|
83
|
+
attr_value = getattr(record, attr_name)
|
|
84
|
+
if not callable(attr_value) and attr_value is not None:
|
|
85
|
+
attributes[attr_name] = attr_value
|
|
86
|
+
except (AttributeError, TypeError):
|
|
87
|
+
# Skip attributes that can't be accessed
|
|
88
|
+
pass
|
|
89
|
+
|
|
90
|
+
# Add exception info if present
|
|
91
|
+
if record.exc_info:
|
|
92
|
+
attributes["exception.type"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
|
93
|
+
attributes["exception.message"] = str(record.exc_info[1]) if record.exc_info[1] else ""
|
|
94
|
+
|
|
95
|
+
return attributes
|
|
96
|
+
|
|
97
|
+
def format(self, record) -> str:
|
|
98
|
+
"""Format the log record to OpenTelemetry JSON format."""
|
|
99
|
+
try:
|
|
100
|
+
otlp_record = {
|
|
101
|
+
"timestamp": self._format_timestamp(record.created),
|
|
102
|
+
"severityText": record.levelname,
|
|
103
|
+
"body": record.getMessage(),
|
|
104
|
+
"attributes": self._extract_attributes(record),
|
|
105
|
+
"resource": self.resource_attributes
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
# Add tracing information if available
|
|
109
|
+
if hasattr(record, 'traceId'):
|
|
110
|
+
otlp_record["traceId"] = record.traceId
|
|
111
|
+
if hasattr(record, 'spanId'):
|
|
112
|
+
otlp_record["spanId"] = record.spanId
|
|
113
|
+
|
|
114
|
+
return json.dumps(otlp_record, ensure_ascii=False)
|
|
115
|
+
|
|
116
|
+
except Exception as e:
|
|
117
|
+
# Fallback to simple JSON if formatting fails
|
|
118
|
+
fallback_record = {
|
|
119
|
+
"timestamp": self._format_timestamp(record.created),
|
|
120
|
+
"severityText": record.levelname,
|
|
121
|
+
"body": record.getMessage(),
|
|
122
|
+
"error": f"Failed to format log record: {str(e)}"
|
|
123
|
+
}
|
|
124
|
+
return json.dumps(fallback_record, ensure_ascii=False)
|
|
125
|
+
|
|
126
|
+
def formatTime(self, record, datefmt=None):
|
|
127
|
+
"""Format the timestamp (for compatibility with logging.Formatter)."""
|
|
128
|
+
return self._format_timestamp(record.created)
|
|
File without changes
|