polaris-python 0.0.1.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polaris/__init__.py +74 -0
- polaris/_version.py +16 -0
- polaris/api/__init__.py +55 -0
- polaris/api/_log_file.py +232 -0
- polaris/api/config.py +145 -0
- polaris/api/consumer.py +240 -0
- polaris/api/log.py +414 -0
- polaris/api/lossless.py +73 -0
- polaris/api/provider.py +152 -0
- polaris/api/skill.py +162 -0
- polaris/pkg/__init__.py +15 -0
- polaris/pkg/common/__init__.py +15 -0
- polaris/pkg/common/plugin.py +33 -0
- polaris/pkg/config/__init__.py +15 -0
- polaris/pkg/config/api.py +1205 -0
- polaris/pkg/config/default.py +92 -0
- polaris/pkg/config/event.py +76 -0
- polaris/pkg/config/loader.py +392 -0
- polaris/pkg/config/stat_reporter.py +100 -0
- polaris/pkg/flow/__init__.py +15 -0
- polaris/pkg/flow/admin_server.py +104 -0
- polaris/pkg/flow/async_flow.py +55 -0
- polaris/pkg/flow/base_flow.py +111 -0
- polaris/pkg/flow/impl.py +1277 -0
- polaris/pkg/flow/lossless.py +605 -0
- polaris/pkg/flow/notify.py +46 -0
- polaris/pkg/flow/register.py +210 -0
- polaris/pkg/flow/skill.py +110 -0
- polaris/pkg/flow/sync_flow.py +20 -0
- polaris/pkg/model/__init__.py +15 -0
- polaris/pkg/model/context.py +53 -0
- polaris/pkg/model/contract.py +89 -0
- polaris/pkg/model/core.py +51 -0
- polaris/pkg/model/engine.py +83 -0
- polaris/pkg/model/error.py +94 -0
- polaris/pkg/model/event.py +159 -0
- polaris/pkg/model/lossless.py +102 -0
- polaris/pkg/model/rule.py +152 -0
- polaris/pkg/model/server.py +288 -0
- polaris/pkg/model/service.py +820 -0
- polaris/pkg/model/skill.py +179 -0
- polaris/pkg/model/stat.py +168 -0
- polaris/pkg/model/util.py +58 -0
- polaris/pkg/version/__init__.py +18 -0
- polaris/plugin/__init__.py +32 -0
- polaris/plugin/circuit_breaker/__init__.py +19 -0
- polaris/plugin/circuit_breaker/base.py +54 -0
- polaris/plugin/circuit_breaker/composite.py +265 -0
- polaris/plugin/circuit_breaker/config.py +36 -0
- polaris/plugin/event/__init__.py +21 -0
- polaris/plugin/event/pushgateway.py +202 -0
- polaris/plugin/load_balancer/__init__.py +24 -0
- polaris/plugin/load_balancer/base.py +30 -0
- polaris/plugin/load_balancer/random.py +39 -0
- polaris/plugin/load_balancer/weighted_random.py +50 -0
- polaris/plugin/local_registry/__init__.py +18 -0
- polaris/plugin/local_registry/base.py +47 -0
- polaris/plugin/local_registry/file_cache.py +335 -0
- polaris/plugin/local_registry/local_cache.py +433 -0
- polaris/plugin/server_connector/__init__.py +20 -0
- polaris/plugin/server_connector/base.py +104 -0
- polaris/plugin/server_connector/convert.py +149 -0
- polaris/plugin/server_connector/grpc_connector/__init__.py +21 -0
- polaris/plugin/server_connector/grpc_connector/connection.py +401 -0
- polaris/plugin/server_connector/grpc_connector/grpc_client.py +384 -0
- polaris/plugin/server_connector/grpc_connector/tls.py +45 -0
- polaris/plugin/service_router/__init__.py +28 -0
- polaris/plugin/service_router/base.py +105 -0
- polaris/plugin/service_router/isolated.py +35 -0
- polaris/plugin/service_router/metadata.py +84 -0
- polaris/plugin/service_router/recover.py +62 -0
- polaris/plugin/service_router/util.py +54 -0
- polaris/plugin/skill/__init__.py +16 -0
- polaris/plugin/skill/connector.py +127 -0
- polaris/plugin/skill/persistence.py +94 -0
- polaris/plugin/stat_reporter/__init__.py +16 -0
- polaris/plugin/stat_reporter/base.py +34 -0
- polaris/plugin/stat_reporter/prometheus.py +375 -0
- polaris/plugin_manager.py +146 -0
- polaris_python-0.0.1.dev3.dist-info/METADATA +213 -0
- polaris_python-0.0.1.dev3.dist-info/RECORD +84 -0
- polaris_python-0.0.1.dev3.dist-info/WHEEL +5 -0
- polaris_python-0.0.1.dev3.dist-info/licenses/LICENSE +18 -0
- polaris_python-0.0.1.dev3.dist-info/top_level.txt +1 -0
polaris/api/consumer.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# Tencent is pleased to support the open source community by making polaris-python available.
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2021 Tencent. All rights reserved.
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the BSD 3-Clause License (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# https://opensource.org/licenses/BSD-3-Clause
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software distributed
|
|
12
|
+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
13
|
+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
14
|
+
# specific language governing permissions and limitations under the License.
|
|
15
|
+
|
|
16
|
+
from threading import RLock
|
|
17
|
+
|
|
18
|
+
from polaris.pkg.config.api import Configuration
|
|
19
|
+
from polaris.pkg.model.service import (GetInstancesRequest, GetOneInstanceRequest, InstancesResponse,
|
|
20
|
+
ServiceCallResult, Instance)
|
|
21
|
+
from polaris.pkg.model.rule import ServiceRuleResponse, GetServiceRuleRequest
|
|
22
|
+
from polaris.api.config import SDKContext, init_context_by_config, init_context_by_file
|
|
23
|
+
from polaris.api.log import LogOptions
|
|
24
|
+
from polaris.pkg.model.error import SDKError, ErrorCode
|
|
25
|
+
from polaris.pkg.flow.async_flow import CacheResult
|
|
26
|
+
from polaris.pkg.model.core import EventType
|
|
27
|
+
from polaris.pkg.model.event import SDKEventName, SDKEventType
|
|
28
|
+
|
|
29
|
+
class ServiceWatch:
|
|
30
|
+
"""An active instance watch owned by one ConsumerAPI."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, consumer, request, snapshot, listener=None):
|
|
33
|
+
self._consumer = consumer
|
|
34
|
+
self.namespace = request.namespace
|
|
35
|
+
self.service = request.service
|
|
36
|
+
self.instances = snapshot
|
|
37
|
+
self.listener = listener
|
|
38
|
+
self._lock = RLock()
|
|
39
|
+
self._closed = False
|
|
40
|
+
|
|
41
|
+
def close(self):
|
|
42
|
+
"""Stop this watch. Repeated close is harmless."""
|
|
43
|
+
self._consumer.unwatch_service(self)
|
|
44
|
+
|
|
45
|
+
def __enter__(self):
|
|
46
|
+
return self
|
|
47
|
+
|
|
48
|
+
def __exit__(self, *_):
|
|
49
|
+
self.close()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ConsumerAPI:
|
|
53
|
+
def __init__(self, context: SDKContext):
|
|
54
|
+
self.context = context
|
|
55
|
+
|
|
56
|
+
def check_available(self):
|
|
57
|
+
if not self.context:
|
|
58
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidArgument, "consumeAPI can not be None")
|
|
59
|
+
if self.context.is_destroy():
|
|
60
|
+
raise SDKError(ErrorCode.ErrCodeInvalidStateError, "consumeAPI: api instance has been destroyed")
|
|
61
|
+
|
|
62
|
+
def get_one_instance(self, request: GetOneInstanceRequest) -> Instance:
|
|
63
|
+
self.check_available()
|
|
64
|
+
request.validate()
|
|
65
|
+
|
|
66
|
+
return self.context.get_engine().sync_get_one_instance(request)
|
|
67
|
+
|
|
68
|
+
def get_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
69
|
+
"""Return locally filtered candidates without selecting or consuming a half-open permit."""
|
|
70
|
+
self.check_available()
|
|
71
|
+
request.validate()
|
|
72
|
+
|
|
73
|
+
return self.context.get_engine().sync_get_instances(request)
|
|
74
|
+
|
|
75
|
+
def get_all_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
76
|
+
"""Return every cached instance, including isolated and unhealthy nodes."""
|
|
77
|
+
self.check_available()
|
|
78
|
+
request.validate()
|
|
79
|
+
return self.context.get_engine().sync_get_all_instances(request)
|
|
80
|
+
|
|
81
|
+
def get_healthy_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
82
|
+
"""Return candidates using only isolation and recovery filters."""
|
|
83
|
+
self.check_available()
|
|
84
|
+
request.validate()
|
|
85
|
+
return self.context.get_engine().sync_get_healthy_instances(request)
|
|
86
|
+
|
|
87
|
+
def get_services(self):
|
|
88
|
+
"""Return the service resources currently cached by this context."""
|
|
89
|
+
self.check_available()
|
|
90
|
+
return self.context.get_engine().sync_get_services()
|
|
91
|
+
|
|
92
|
+
async def async_get_services(self):
|
|
93
|
+
"""Asynchronously return the service resources cached by this context."""
|
|
94
|
+
self.check_available()
|
|
95
|
+
return await self.context.get_engine().async_get_services()
|
|
96
|
+
|
|
97
|
+
def sync_update_service_call_result(self, request: ServiceCallResult):
|
|
98
|
+
self.check_available()
|
|
99
|
+
request.validate()
|
|
100
|
+
|
|
101
|
+
return self.context.get_engine().sync_update_service_call_result(request)
|
|
102
|
+
|
|
103
|
+
async def async_get_one_instance(self, request: GetOneInstanceRequest) -> Instance:
|
|
104
|
+
self.check_available()
|
|
105
|
+
request.validate()
|
|
106
|
+
|
|
107
|
+
return await self.context.get_engine().async_get_one_instance(req=request)
|
|
108
|
+
|
|
109
|
+
async def async_get_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
110
|
+
self.check_available()
|
|
111
|
+
request.validate()
|
|
112
|
+
|
|
113
|
+
return await self.context.get_engine().async_get_instances(req=request)
|
|
114
|
+
|
|
115
|
+
async def async_get_all_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
116
|
+
"""Asynchronously return the full snapshot."""
|
|
117
|
+
self.check_available()
|
|
118
|
+
request.validate()
|
|
119
|
+
return await self.context.get_engine().async_get_all_instances(request)
|
|
120
|
+
|
|
121
|
+
async def async_get_healthy_instances(self, request: GetInstancesRequest) -> InstancesResponse:
|
|
122
|
+
"""Asynchronously return isolation and recovery candidates."""
|
|
123
|
+
self.check_available()
|
|
124
|
+
request.validate()
|
|
125
|
+
return await self.context.get_engine().async_get_healthy_instances(request)
|
|
126
|
+
|
|
127
|
+
def get_route_rule(self, request: GetServiceRuleRequest) -> ServiceRuleResponse:
|
|
128
|
+
"""Synchronously reject unsupported non-instance resource loading."""
|
|
129
|
+
self.check_available()
|
|
130
|
+
request.validate()
|
|
131
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidArgument, "only instance resources are cached")
|
|
132
|
+
|
|
133
|
+
async def async_get_route_rule(self, request: GetServiceRuleRequest) -> ServiceRuleResponse:
|
|
134
|
+
"""Asynchronously reject unsupported non-instance resource loading."""
|
|
135
|
+
self.check_available()
|
|
136
|
+
request.validate()
|
|
137
|
+
return await self.context.get_engine().sync_get_service_rule(event_type=EventType.Routing, req=request)
|
|
138
|
+
|
|
139
|
+
def watch_service(self, request: GetInstancesRequest, listener=None) -> ServiceWatch:
|
|
140
|
+
"""Load and pin one service, forwarding its cache lifecycle events."""
|
|
141
|
+
if listener is not None and not callable(listener):
|
|
142
|
+
raise ValueError("listener must be callable")
|
|
143
|
+
snapshot = self.get_all_instances(request)
|
|
144
|
+
return self._start_watch(request, snapshot, listener)
|
|
145
|
+
|
|
146
|
+
async def async_watch_service(self, request: GetInstancesRequest, listener=None) -> ServiceWatch:
|
|
147
|
+
"""Asynchronously load and pin one service."""
|
|
148
|
+
if listener is not None and not callable(listener):
|
|
149
|
+
raise ValueError("listener must be callable")
|
|
150
|
+
snapshot = await self.async_get_all_instances(request)
|
|
151
|
+
return self._start_watch(request, snapshot, listener)
|
|
152
|
+
|
|
153
|
+
def _start_watch(self, request, snapshot, listener):
|
|
154
|
+
self.check_available()
|
|
155
|
+
self.context.get_engine().registry.watch_service(request.namespace, request.service)
|
|
156
|
+
event_center = self.context.get_event_center()
|
|
157
|
+
if event_center is None:
|
|
158
|
+
self.context.get_engine().registry.unwatch_service(request.namespace, request.service)
|
|
159
|
+
raise SDKError(ErrorCode.ErrCodeInvalidStateError, "SDKContext event center is unavailable")
|
|
160
|
+
|
|
161
|
+
def filtered_listener(event):
|
|
162
|
+
if (event.event_type != SDKEventType.INSTANCE
|
|
163
|
+
or event.namespace != request.namespace
|
|
164
|
+
or event.service != request.service
|
|
165
|
+
or event.event_name not in (
|
|
166
|
+
SDKEventName.ON_SERVICE_ADDED,
|
|
167
|
+
SDKEventName.ON_SERVICE_UPDATED,
|
|
168
|
+
SDKEventName.ON_SERVICE_DELETED,
|
|
169
|
+
)):
|
|
170
|
+
return
|
|
171
|
+
if listener is not None:
|
|
172
|
+
listener(event)
|
|
173
|
+
|
|
174
|
+
event_center.register_listener(filtered_listener, SDKEventType.INSTANCE)
|
|
175
|
+
watch = ServiceWatch(self, request, snapshot, listener)
|
|
176
|
+
watch._event_listener = filtered_listener
|
|
177
|
+
watch._event_center = event_center
|
|
178
|
+
return watch
|
|
179
|
+
|
|
180
|
+
def unwatch_service(self, watch: ServiceWatch):
|
|
181
|
+
"""Release a watch created by this ConsumerAPI."""
|
|
182
|
+
if not isinstance(watch, ServiceWatch) or watch._consumer is not self:
|
|
183
|
+
raise ValueError("watch must belong to this ConsumerAPI")
|
|
184
|
+
with watch._lock:
|
|
185
|
+
if watch._closed:
|
|
186
|
+
return
|
|
187
|
+
watch._closed = True
|
|
188
|
+
if getattr(watch, "_event_center", None) is not None:
|
|
189
|
+
watch._event_center.unregister_listener(
|
|
190
|
+
watch._event_listener, SDKEventType.INSTANCE,
|
|
191
|
+
)
|
|
192
|
+
watch._event_center = None
|
|
193
|
+
if self.context is not None and not self.context.is_destroy():
|
|
194
|
+
self.context.get_engine().registry.unwatch_service(watch.namespace, watch.service)
|
|
195
|
+
|
|
196
|
+
async def async_unwatch_service(self, watch: ServiceWatch):
|
|
197
|
+
"""Asynchronously release a watch created by this ConsumerAPI."""
|
|
198
|
+
self.unwatch_service(watch)
|
|
199
|
+
|
|
200
|
+
def update_service_call_result(self, request: ServiceCallResult):
|
|
201
|
+
"""Synchronously report invocation metrics and the existing local instance result."""
|
|
202
|
+
return self.sync_update_service_call_result(request)
|
|
203
|
+
|
|
204
|
+
async def async_update_service_call_result(self, request: ServiceCallResult):
|
|
205
|
+
"""Asynchronously report invocation metrics on this context's owner loop."""
|
|
206
|
+
self.check_available()
|
|
207
|
+
request.validate()
|
|
208
|
+
|
|
209
|
+
return await self.context.get_engine().async_update_service_call_result(request)
|
|
210
|
+
|
|
211
|
+
def get_service_contract(self, request):
|
|
212
|
+
"""Synchronously fetch a contract from the registry."""
|
|
213
|
+
self.check_available()
|
|
214
|
+
request.validate()
|
|
215
|
+
return self.context.get_engine().sync_contract_call("get_service_contract", request)
|
|
216
|
+
|
|
217
|
+
async def async_get_service_contract(self, request):
|
|
218
|
+
"""Asynchronously fetch a contract from the registry."""
|
|
219
|
+
self.check_available()
|
|
220
|
+
request.validate()
|
|
221
|
+
return await self.context.get_engine().async_contract_call("get_service_contract", request)
|
|
222
|
+
|
|
223
|
+
def destroy(self):
|
|
224
|
+
if self.context:
|
|
225
|
+
self.context.destroy()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def create_consumer_by_file(path: str, *, log_options: LogOptions | None = None) -> ConsumerAPI:
|
|
229
|
+
context = init_context_by_file(path, log_options=log_options)
|
|
230
|
+
return ConsumerAPI(context)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def create_consumer_by_config(config: Configuration, *, log_options: LogOptions | None = None) -> ConsumerAPI:
|
|
234
|
+
context = init_context_by_config(config, log_options=log_options)
|
|
235
|
+
return ConsumerAPI(context)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def create_consumer_by_default_config_file(*, log_options: LogOptions | None = None) -> ConsumerAPI:
|
|
239
|
+
from polaris.api.config import init_context
|
|
240
|
+
return ConsumerAPI(context=init_context(log_options=log_options))
|
polaris/api/log.py
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
# Tencent is pleased to support the open source community by making polaris-python available.
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2021 Tencent. All rights reserved.
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the BSD 3-Clause License (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# https://opensource.org/licenses/BSD-3-Clause
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software distributed
|
|
12
|
+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
13
|
+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
14
|
+
# specific language governing permissions and limitations under the License.
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from contextvars import ContextVar
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from threading import RLock
|
|
24
|
+
|
|
25
|
+
from polaris.api._log_file import (
|
|
26
|
+
DEFAULT_BACKUP_COUNT, DEFAULT_MAX_AGE_DAYS, DEFAULT_MAX_BYTES,
|
|
27
|
+
ExclusiveRotatingFileHandler, LogRetention, reserve_log_files,
|
|
28
|
+
)
|
|
29
|
+
from polaris.pkg.model.error import ErrorCode, SDKError
|
|
30
|
+
|
|
31
|
+
default_log_dir = "./polaris/log"
|
|
32
|
+
default_log_path = f"{default_log_dir}/base/polaris.log"
|
|
33
|
+
LOG_CATEGORIES = (
|
|
34
|
+
"base", "network", "cache", "route", "circuitbreaker", "statReport", "event",
|
|
35
|
+
"lossless", "admin",
|
|
36
|
+
)
|
|
37
|
+
default_logger: logging.Logger | None = None
|
|
38
|
+
_logger_lock = RLock()
|
|
39
|
+
_context_logger = ContextVar("polaris_context_logger", default=None)
|
|
40
|
+
_context_log_files = set()
|
|
41
|
+
_process_loggers = {}
|
|
42
|
+
_process_log_paths = {}
|
|
43
|
+
_process_file_locks = {}
|
|
44
|
+
_process_retention = None
|
|
45
|
+
_process_rotation = (DEFAULT_MAX_BYTES, DEFAULT_BACKUP_COUNT, DEFAULT_MAX_AGE_DAYS)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class LogOptions:
|
|
50
|
+
"""Assign log options in code; each context captures their values at creation."""
|
|
51
|
+
|
|
52
|
+
level: str | None = None
|
|
53
|
+
file_path: str | Path | None = None
|
|
54
|
+
max_bytes: int = DEFAULT_MAX_BYTES
|
|
55
|
+
backup_count: int = DEFAULT_BACKUP_COUNT
|
|
56
|
+
log_dir: str | Path | None = None
|
|
57
|
+
max_age_days: int = DEFAULT_MAX_AGE_DAYS
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _validate_category(category):
|
|
61
|
+
if category not in LOG_CATEGORIES:
|
|
62
|
+
raise ValueError(f"Unknown log category: {category}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _log_paths(file_path=None, log_dir=None, client_id=None):
|
|
66
|
+
"""Resolve every category now, so later environment changes cannot move logs."""
|
|
67
|
+
if file_path is not None and log_dir is not None:
|
|
68
|
+
raise ValueError("Specify either log_dir or file_path")
|
|
69
|
+
if file_path is None and log_dir is None:
|
|
70
|
+
configured_path = os.getenv("POLARIS_LOG_PATH")
|
|
71
|
+
if configured_path:
|
|
72
|
+
candidate = Path(configured_path).expanduser()
|
|
73
|
+
if candidate.is_dir() or not candidate.suffix or configured_path.endswith(("/", "\\")):
|
|
74
|
+
log_dir = candidate
|
|
75
|
+
else:
|
|
76
|
+
file_path = candidate
|
|
77
|
+
else:
|
|
78
|
+
log_dir = default_log_dir
|
|
79
|
+
if log_dir is not None:
|
|
80
|
+
root = Path(log_dir).expanduser().resolve()
|
|
81
|
+
stem, suffix = "polaris", ".log"
|
|
82
|
+
base = root / "base" / f"{stem}{suffix}"
|
|
83
|
+
else:
|
|
84
|
+
base = Path(file_path).expanduser().resolve()
|
|
85
|
+
root = base.parent.parent if base.parent.name == "base" else base.parent
|
|
86
|
+
stem, suffix = base.stem, base.suffix or ".log"
|
|
87
|
+
identity = f"-{client_id}" if client_id is not None else ""
|
|
88
|
+
paths = {
|
|
89
|
+
category: (root / category / f"{stem}-{category}{identity}{suffix}").resolve()
|
|
90
|
+
for category in LOG_CATEGORIES if category != "base"
|
|
91
|
+
}
|
|
92
|
+
paths["base"] = base.with_name(f"{stem}{identity}{suffix}") if identity else base
|
|
93
|
+
return paths
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _create_file_logger(name, path, level, max_bytes, backup_count, max_age_days=DEFAULT_MAX_AGE_DAYS):
|
|
97
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
handler = ExclusiveRotatingFileHandler(path, "a", max_bytes, backup_count,
|
|
99
|
+
encoding="utf-8", max_age_days=max_age_days)
|
|
100
|
+
handler.setFormatter(logging.Formatter("[%(asctime)s][%(levelname)s][%(process)d]: %(message)s"))
|
|
101
|
+
instance = logging.Logger(name, level=level)
|
|
102
|
+
instance.addHandler(handler)
|
|
103
|
+
instance.propagate = False
|
|
104
|
+
return instance, handler
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class _ContextCategoryLogger(logging.LoggerAdapter):
|
|
108
|
+
def __init__(self, instance, handler, client_id, path):
|
|
109
|
+
super().__init__(instance, {"client_id": client_id})
|
|
110
|
+
self.handler = handler
|
|
111
|
+
self.file_path = path
|
|
112
|
+
|
|
113
|
+
def process(self, msg, kwargs):
|
|
114
|
+
"""Prefix every entry with this context's stable client ID."""
|
|
115
|
+
msg, kwargs = super().process(msg, kwargs)
|
|
116
|
+
return f"[clientId={self.extra['client_id']}] {msg}", kwargs
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ContextLogger(_ContextCategoryLogger):
|
|
120
|
+
"""Own independent rotating category logs for one SDK context."""
|
|
121
|
+
|
|
122
|
+
def __init__(self, client_id, options: LogOptions | None = None):
|
|
123
|
+
options = options if options is not None else LogOptions()
|
|
124
|
+
if not isinstance(options, LogOptions):
|
|
125
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, "Context log options must be LogOptions")
|
|
126
|
+
level = options.level if options.level is not None else os.getenv("POLARIS_LOG_LEVEL", "info")
|
|
127
|
+
if not isinstance(level, str):
|
|
128
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, "Context log level must be a string")
|
|
129
|
+
level = level.lower()
|
|
130
|
+
if level not in LogLevelMap:
|
|
131
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, f"Invalid context log level: {level}")
|
|
132
|
+
if (type(options.max_bytes) is not int or options.max_bytes <= 0
|
|
133
|
+
or type(options.backup_count) is not int or options.backup_count < 0
|
|
134
|
+
or type(options.max_age_days) is not int or options.max_age_days < 0):
|
|
135
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, "Invalid context log rotation options")
|
|
136
|
+
if options.file_path is not None and not isinstance(options.file_path, (str, Path)):
|
|
137
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, "Context log file path must be a string or Path")
|
|
138
|
+
if options.log_dir is not None and not isinstance(options.log_dir, (str, Path)):
|
|
139
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, "Context log directory must be a string or Path")
|
|
140
|
+
try:
|
|
141
|
+
self._paths = _log_paths(options.file_path, options.log_dir,
|
|
142
|
+
client_id=None if options.file_path is not None else client_id)
|
|
143
|
+
except (OSError, ValueError) as error:
|
|
144
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig, str(error)) from error
|
|
145
|
+
self._level = LogLevelMap[level]
|
|
146
|
+
self._max_bytes = options.max_bytes
|
|
147
|
+
self._backup_count = options.backup_count
|
|
148
|
+
self._max_age_days = options.max_age_days
|
|
149
|
+
self._close_lock = RLock()
|
|
150
|
+
self._closed = False
|
|
151
|
+
self._owner_pid = os.getpid()
|
|
152
|
+
self._categories = {}
|
|
153
|
+
self._file_locks = {}
|
|
154
|
+
self._retention = None
|
|
155
|
+
with _logger_lock:
|
|
156
|
+
collisions = set(self._paths.values()) & _context_log_files
|
|
157
|
+
if collisions:
|
|
158
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig,
|
|
159
|
+
f"Context log file is already in use: {next(iter(collisions))}")
|
|
160
|
+
try:
|
|
161
|
+
self._file_locks = reserve_log_files(self._paths.values())
|
|
162
|
+
instance, handler = _create_file_logger(
|
|
163
|
+
f"polaris.{client_id}.base", self._paths["base"], self._level,
|
|
164
|
+
self._max_bytes, self._backup_count, self._max_age_days,
|
|
165
|
+
)
|
|
166
|
+
except (OSError, ValueError) as error:
|
|
167
|
+
for lock in self._file_locks.values():
|
|
168
|
+
lock.close()
|
|
169
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig,
|
|
170
|
+
f"Cannot open context log file: {self._paths['base']}") from error
|
|
171
|
+
super().__init__(instance, handler, client_id, self._paths["base"])
|
|
172
|
+
self._categories["base"] = self
|
|
173
|
+
_context_log_files.update(self._paths.values())
|
|
174
|
+
family_paths = ()
|
|
175
|
+
if options.file_path is None:
|
|
176
|
+
family_paths = [path.with_name(f"{path.stem[:-len(client_id) - 1]}{path.suffix}")
|
|
177
|
+
for path in self._paths.values()]
|
|
178
|
+
self._retention = LogRetention(self._file_locks, self._max_age_days, self._backup_count, family_paths)
|
|
179
|
+
self._retention.add_handler(handler)
|
|
180
|
+
try:
|
|
181
|
+
self._retention.start()
|
|
182
|
+
except Exception:
|
|
183
|
+
self.close()
|
|
184
|
+
raise
|
|
185
|
+
|
|
186
|
+
def get_logger(self, category="base"):
|
|
187
|
+
"""Open a category on first use, sharing this context's captured options."""
|
|
188
|
+
_validate_category(category)
|
|
189
|
+
if self._owner_pid != os.getpid():
|
|
190
|
+
raise SDKError(ErrorCode.ErrCodeInvalidStateError, "Create an independent SDK context after fork")
|
|
191
|
+
with self._close_lock:
|
|
192
|
+
if self._closed:
|
|
193
|
+
raise SDKError(ErrorCode.ErrCodeInvalidStateError, "Context logger has been closed")
|
|
194
|
+
if category not in self._categories:
|
|
195
|
+
path = self._paths[category]
|
|
196
|
+
with self._retention.lock:
|
|
197
|
+
try:
|
|
198
|
+
instance, handler = _create_file_logger(
|
|
199
|
+
f"polaris.{self.extra['client_id']}.{category}", path, self._level,
|
|
200
|
+
self._max_bytes, self._backup_count, self._max_age_days,
|
|
201
|
+
)
|
|
202
|
+
except (OSError, ValueError) as error:
|
|
203
|
+
raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig,
|
|
204
|
+
f"Cannot open context log file: {path}") from error
|
|
205
|
+
self._retention.add_handler(handler)
|
|
206
|
+
self._categories[category] = _ContextCategoryLogger(instance, handler, self.extra['client_id'], path)
|
|
207
|
+
return self._categories[category]
|
|
208
|
+
|
|
209
|
+
def close(self):
|
|
210
|
+
"""Close all owned category handlers and release their paths."""
|
|
211
|
+
with self._close_lock:
|
|
212
|
+
if self._closed:
|
|
213
|
+
return
|
|
214
|
+
self._closed = True
|
|
215
|
+
if self._retention is not None:
|
|
216
|
+
self._retention.close()
|
|
217
|
+
for instance in self._categories.values():
|
|
218
|
+
instance.logger.removeHandler(instance.handler)
|
|
219
|
+
instance.handler.close()
|
|
220
|
+
with _logger_lock:
|
|
221
|
+
for lock in self._file_locks.values():
|
|
222
|
+
lock.close()
|
|
223
|
+
self._file_locks.clear()
|
|
224
|
+
_context_log_files.difference_update(self._paths.values())
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@contextmanager
|
|
228
|
+
def use_context_logger(instance):
|
|
229
|
+
"""Bind logging to a context for one thread or asynchronous task."""
|
|
230
|
+
token = _context_logger.set(instance)
|
|
231
|
+
try:
|
|
232
|
+
yield
|
|
233
|
+
finally:
|
|
234
|
+
_context_logger.reset(token)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class LogLevel:
|
|
238
|
+
Debug = "debug"
|
|
239
|
+
Info = "info"
|
|
240
|
+
Warn = "warn"
|
|
241
|
+
Error = "error"
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
LogLevelMap = {
|
|
245
|
+
"debug": logging.DEBUG,
|
|
246
|
+
"info": logging.INFO,
|
|
247
|
+
"warn": logging.WARN,
|
|
248
|
+
"error": logging.ERROR
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _get_log_level(level):
|
|
253
|
+
return LogLevelMap.get(level, logging.INFO)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def init_log(level=None, file_path=None, *, log_dir=None, max_bytes=DEFAULT_MAX_BYTES,
|
|
257
|
+
backup_count=DEFAULT_BACKUP_COUNT, max_age_days=DEFAULT_MAX_AGE_DAYS):
|
|
258
|
+
"""Initialize process logging; explicit options take precedence over POLARIS_LOG_* values."""
|
|
259
|
+
global default_logger
|
|
260
|
+
with _logger_lock:
|
|
261
|
+
if default_logger is None:
|
|
262
|
+
if level is None:
|
|
263
|
+
level = os.getenv("POLARIS_LOG_LEVEL", LogLevel.Info).lower()
|
|
264
|
+
if log_dir is None:
|
|
265
|
+
if file_path is None:
|
|
266
|
+
file_path = _log_paths()["base"]
|
|
267
|
+
default_logger = create_log(level, file_path, max_bytes, backup_count, max_age_days=max_age_days)
|
|
268
|
+
else:
|
|
269
|
+
default_logger = create_log(level, file_path, max_bytes, backup_count,
|
|
270
|
+
log_dir=log_dir, max_age_days=max_age_days)
|
|
271
|
+
return default_logger
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def create_log(level=LogLevel.Info, file_path=None, max_bytes=DEFAULT_MAX_BYTES, backup_count=DEFAULT_BACKUP_COUNT,
|
|
275
|
+
*, log_dir=None, max_age_days=DEFAULT_MAX_AGE_DAYS):
|
|
276
|
+
"""Create process logs; category files are opened on first use."""
|
|
277
|
+
global default_logger, _process_rotation, _process_retention
|
|
278
|
+
if (type(max_bytes) is not int or max_bytes <= 0
|
|
279
|
+
or type(backup_count) is not int or backup_count < 0
|
|
280
|
+
or type(max_age_days) is not int or max_age_days < 0):
|
|
281
|
+
raise ValueError("Invalid process log rotation options")
|
|
282
|
+
if file_path is None and log_dir is None:
|
|
283
|
+
log_dir = default_log_dir
|
|
284
|
+
paths = _log_paths(file_path, log_dir)
|
|
285
|
+
with _logger_lock:
|
|
286
|
+
retained = _process_file_locks if _process_retention is None or _process_retention.owner_pid == os.getpid() else {}
|
|
287
|
+
locks = reserve_log_files(paths.values(), retained=retained)
|
|
288
|
+
try:
|
|
289
|
+
instance, handler = _create_file_logger("polaris", paths["base"], _get_log_level(level),
|
|
290
|
+
max_bytes, backup_count, max_age_days)
|
|
291
|
+
except Exception:
|
|
292
|
+
for path, lock in locks.items():
|
|
293
|
+
if path not in retained:
|
|
294
|
+
lock.close()
|
|
295
|
+
raise
|
|
296
|
+
replaces_default = default_logger is not None and default_logger is _process_loggers.get("base")
|
|
297
|
+
_close_process_logs(retained=locks)
|
|
298
|
+
_process_file_locks.update(locks)
|
|
299
|
+
_process_log_paths.update(paths)
|
|
300
|
+
_process_rotation = (max_bytes, backup_count, max_age_days)
|
|
301
|
+
_process_loggers["base"] = instance
|
|
302
|
+
_process_retention = LogRetention(_process_file_locks, max_age_days, backup_count)
|
|
303
|
+
_process_retention.add_handler(handler)
|
|
304
|
+
try:
|
|
305
|
+
_process_retention.start()
|
|
306
|
+
except Exception:
|
|
307
|
+
_close_process_logs()
|
|
308
|
+
raise
|
|
309
|
+
if replaces_default:
|
|
310
|
+
default_logger = instance
|
|
311
|
+
return instance
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _close_process_logs(retained=None):
|
|
315
|
+
global _process_retention
|
|
316
|
+
if _process_retention is not None:
|
|
317
|
+
_process_retention.close()
|
|
318
|
+
_process_retention = None
|
|
319
|
+
for instance in _process_loggers.values():
|
|
320
|
+
for handler in instance.handlers[:]:
|
|
321
|
+
instance.removeHandler(handler)
|
|
322
|
+
handler.close()
|
|
323
|
+
_process_loggers.clear()
|
|
324
|
+
_process_log_paths.clear()
|
|
325
|
+
retained = retained or {}
|
|
326
|
+
for path, lock in _process_file_locks.items():
|
|
327
|
+
if path not in retained:
|
|
328
|
+
lock.close()
|
|
329
|
+
_process_file_locks.clear()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def set_logger(instance):
|
|
333
|
+
"""Set the process logger used outside context-bound SDK operations."""
|
|
334
|
+
global default_logger
|
|
335
|
+
with _logger_lock:
|
|
336
|
+
if instance is not _process_loggers.get("base"):
|
|
337
|
+
_close_process_logs()
|
|
338
|
+
default_logger = instance
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _get_logger(category="base"):
|
|
342
|
+
bound_logger = _context_logger.get()
|
|
343
|
+
if bound_logger is not None:
|
|
344
|
+
if isinstance(bound_logger, ContextLogger):
|
|
345
|
+
return bound_logger.get_logger(category)
|
|
346
|
+
return bound_logger
|
|
347
|
+
global default_logger
|
|
348
|
+
with _logger_lock:
|
|
349
|
+
if default_logger is None:
|
|
350
|
+
default_logger = logging.getLogger("polaris")
|
|
351
|
+
default_logger.addHandler(logging.NullHandler())
|
|
352
|
+
if category != "base" and default_logger is _process_loggers.get("base"):
|
|
353
|
+
if _process_retention.owner_pid != os.getpid():
|
|
354
|
+
raise SDKError(ErrorCode.ErrCodeInvalidStateError, "Reinitialize process logging after fork")
|
|
355
|
+
if category not in _process_loggers:
|
|
356
|
+
with _process_retention.lock:
|
|
357
|
+
instance, handler = _create_file_logger(
|
|
358
|
+
f"polaris.{category}", _process_log_paths[category], default_logger.level, *_process_rotation,
|
|
359
|
+
)
|
|
360
|
+
_process_retention.add_handler(handler)
|
|
361
|
+
_process_loggers[category] = instance
|
|
362
|
+
return _process_loggers[category]
|
|
363
|
+
return default_logger
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class CategoryLogger:
|
|
367
|
+
"""Resolve the current context at emission time, including across async tasks."""
|
|
368
|
+
|
|
369
|
+
def __init__(self, category):
|
|
370
|
+
_validate_category(category)
|
|
371
|
+
self.category = category
|
|
372
|
+
|
|
373
|
+
def debug(self, msg, *args, **kwargs):
|
|
374
|
+
_emit(self.category, "debug", msg, args, kwargs)
|
|
375
|
+
|
|
376
|
+
def info(self, msg, *args, **kwargs):
|
|
377
|
+
_emit(self.category, "info", msg, args, kwargs)
|
|
378
|
+
|
|
379
|
+
def warning(self, msg, *args, **kwargs):
|
|
380
|
+
_emit(self.category, "warning", msg, args, kwargs)
|
|
381
|
+
|
|
382
|
+
def error(self, msg, *args, **kwargs):
|
|
383
|
+
_emit(self.category, "error", msg, args, kwargs)
|
|
384
|
+
|
|
385
|
+
def __getattr__(self, name):
|
|
386
|
+
return getattr(_get_logger(self.category), name)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def get_logger(category="base"):
|
|
390
|
+
"""Get a context-aware logger for a polaris-go category such as network or cache."""
|
|
391
|
+
return CategoryLogger(category)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _emit(category, method, msg, args, kwargs):
|
|
395
|
+
try:
|
|
396
|
+
getattr(_get_logger(category), method)(msg, *args, **kwargs)
|
|
397
|
+
except Exception as err: # pylint: disable=broad-except
|
|
398
|
+
logging.getLogger("polaris").error("log err! msg: %s; args: %s; err: %s", msg, json.dumps(args), repr(err))
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def debug(msg, *args, **kwargs):
|
|
402
|
+
_emit("base", "debug", msg, args, kwargs)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def info(msg, *args, **kwargs):
|
|
406
|
+
_emit("base", "info", msg, args, kwargs)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def warning(msg, *args, **kwargs):
|
|
410
|
+
_emit("base", "warning", msg, args, kwargs)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def error(msg, *args, **kwargs):
|
|
414
|
+
_emit("base", "error", msg, args, kwargs)
|