polaris-python 0.0.1.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polaris/__init__.py +74 -0
- polaris/_version.py +16 -0
- polaris/api/__init__.py +55 -0
- polaris/api/_log_file.py +232 -0
- polaris/api/config.py +145 -0
- polaris/api/consumer.py +240 -0
- polaris/api/log.py +414 -0
- polaris/api/lossless.py +73 -0
- polaris/api/provider.py +152 -0
- polaris/api/skill.py +162 -0
- polaris/pkg/__init__.py +15 -0
- polaris/pkg/common/__init__.py +15 -0
- polaris/pkg/common/plugin.py +33 -0
- polaris/pkg/config/__init__.py +15 -0
- polaris/pkg/config/api.py +1205 -0
- polaris/pkg/config/default.py +92 -0
- polaris/pkg/config/event.py +76 -0
- polaris/pkg/config/loader.py +392 -0
- polaris/pkg/config/stat_reporter.py +100 -0
- polaris/pkg/flow/__init__.py +15 -0
- polaris/pkg/flow/admin_server.py +104 -0
- polaris/pkg/flow/async_flow.py +55 -0
- polaris/pkg/flow/base_flow.py +111 -0
- polaris/pkg/flow/impl.py +1277 -0
- polaris/pkg/flow/lossless.py +605 -0
- polaris/pkg/flow/notify.py +46 -0
- polaris/pkg/flow/register.py +210 -0
- polaris/pkg/flow/skill.py +110 -0
- polaris/pkg/flow/sync_flow.py +20 -0
- polaris/pkg/model/__init__.py +15 -0
- polaris/pkg/model/context.py +53 -0
- polaris/pkg/model/contract.py +89 -0
- polaris/pkg/model/core.py +51 -0
- polaris/pkg/model/engine.py +83 -0
- polaris/pkg/model/error.py +94 -0
- polaris/pkg/model/event.py +159 -0
- polaris/pkg/model/lossless.py +102 -0
- polaris/pkg/model/rule.py +152 -0
- polaris/pkg/model/server.py +288 -0
- polaris/pkg/model/service.py +820 -0
- polaris/pkg/model/skill.py +179 -0
- polaris/pkg/model/stat.py +168 -0
- polaris/pkg/model/util.py +58 -0
- polaris/pkg/version/__init__.py +18 -0
- polaris/plugin/__init__.py +32 -0
- polaris/plugin/circuit_breaker/__init__.py +19 -0
- polaris/plugin/circuit_breaker/base.py +54 -0
- polaris/plugin/circuit_breaker/composite.py +265 -0
- polaris/plugin/circuit_breaker/config.py +36 -0
- polaris/plugin/event/__init__.py +21 -0
- polaris/plugin/event/pushgateway.py +202 -0
- polaris/plugin/load_balancer/__init__.py +24 -0
- polaris/plugin/load_balancer/base.py +30 -0
- polaris/plugin/load_balancer/random.py +39 -0
- polaris/plugin/load_balancer/weighted_random.py +50 -0
- polaris/plugin/local_registry/__init__.py +18 -0
- polaris/plugin/local_registry/base.py +47 -0
- polaris/plugin/local_registry/file_cache.py +335 -0
- polaris/plugin/local_registry/local_cache.py +433 -0
- polaris/plugin/server_connector/__init__.py +20 -0
- polaris/plugin/server_connector/base.py +104 -0
- polaris/plugin/server_connector/convert.py +149 -0
- polaris/plugin/server_connector/grpc_connector/__init__.py +21 -0
- polaris/plugin/server_connector/grpc_connector/connection.py +401 -0
- polaris/plugin/server_connector/grpc_connector/grpc_client.py +384 -0
- polaris/plugin/server_connector/grpc_connector/tls.py +45 -0
- polaris/plugin/service_router/__init__.py +28 -0
- polaris/plugin/service_router/base.py +105 -0
- polaris/plugin/service_router/isolated.py +35 -0
- polaris/plugin/service_router/metadata.py +84 -0
- polaris/plugin/service_router/recover.py +62 -0
- polaris/plugin/service_router/util.py +54 -0
- polaris/plugin/skill/__init__.py +16 -0
- polaris/plugin/skill/connector.py +127 -0
- polaris/plugin/skill/persistence.py +94 -0
- polaris/plugin/stat_reporter/__init__.py +16 -0
- polaris/plugin/stat_reporter/base.py +34 -0
- polaris/plugin/stat_reporter/prometheus.py +375 -0
- polaris/plugin_manager.py +146 -0
- polaris_python-0.0.1.dev3.dist-info/METADATA +213 -0
- polaris_python-0.0.1.dev3.dist-info/RECORD +84 -0
- polaris_python-0.0.1.dev3.dist-info/WHEEL +5 -0
- polaris_python-0.0.1.dev3.dist-info/licenses/LICENSE +18 -0
- polaris_python-0.0.1.dev3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
# Tencent is pleased to support the open source community by making polaris-python available.
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2021 Tencent. All rights reserved.
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the BSD 3-Clause License (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# https://opensource.org/licenses/BSD-3-Clause
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
"""Lossless online/offline flow, mirroring the Java lossless plugin suite."""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import ipaddress
|
|
23
|
+
import math
|
|
24
|
+
import time
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from datetime import datetime, timezone
|
|
27
|
+
from typing import Dict, List, Optional
|
|
28
|
+
|
|
29
|
+
from aiohttp import ClientSession, ClientTimeout
|
|
30
|
+
from aiohttp import web
|
|
31
|
+
|
|
32
|
+
import polaris.api.log as logger
|
|
33
|
+
from polaris.pkg.config.api import Configuration, LosslessConfig
|
|
34
|
+
from polaris.pkg.model.lossless import (
|
|
35
|
+
InstanceProperties,
|
|
36
|
+
LosslessActionProvider,
|
|
37
|
+
LosslessInstance,
|
|
38
|
+
RegisterStatus,
|
|
39
|
+
)
|
|
40
|
+
from polaris.pkg.model.server import InstanceDiscoverRequest
|
|
41
|
+
from polaris.pkg.model.service import Instance, ServiceInstances
|
|
42
|
+
from polarismesh_specification.api.v1.traffic_manage import lossless_pb2
|
|
43
|
+
|
|
44
|
+
# DiscoverRequest.DiscoverRequestType.LOSSLESS
|
|
45
|
+
DISCOVER_TYPE_LOSSLESS = 103
|
|
46
|
+
DELAY_BY_TIME = "DELAY_BY_TIME"
|
|
47
|
+
DELAY_BY_HEALTH_CHECK = "DELAY_BY_HEALTH_CHECK"
|
|
48
|
+
STRATEGIES = (DELAY_BY_TIME, DELAY_BY_HEALTH_CHECK)
|
|
49
|
+
RULE_REFRESH_INTERVAL_SECONDS = 30.0
|
|
50
|
+
|
|
51
|
+
lossless_logger = logger.get_logger("lossless")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class LosslessOptions:
|
|
56
|
+
"""Effective lossless settings, with server rules taking precedence."""
|
|
57
|
+
|
|
58
|
+
delay_register_enable: bool
|
|
59
|
+
delay_register_interval_ms: int
|
|
60
|
+
health_check_interval_ms: int
|
|
61
|
+
strategy: str
|
|
62
|
+
health_check_path: str
|
|
63
|
+
warmup_enable: bool
|
|
64
|
+
warmup_interval_second: int
|
|
65
|
+
warmup_curvature: float
|
|
66
|
+
warmup_overload_protection: bool
|
|
67
|
+
warmup_overload_threshold: int
|
|
68
|
+
offline_enable: bool
|
|
69
|
+
readiness_enable: bool
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _submessage(parent, name: str):
|
|
73
|
+
"""Return a present proto submessage, or None for missing/default."""
|
|
74
|
+
if parent is None or not parent.HasField(name):
|
|
75
|
+
return None
|
|
76
|
+
return getattr(parent, name)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _extract_rules(response) -> List:
|
|
80
|
+
rules = list(getattr(response, "losslessRuleList", []) or [])
|
|
81
|
+
if rules:
|
|
82
|
+
return rules
|
|
83
|
+
if getattr(response, "HasField", lambda *_: False)("lossless"):
|
|
84
|
+
return [response.lossless]
|
|
85
|
+
return []
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _match_rule(instance: LosslessInstance, rules: List):
|
|
89
|
+
"""Match by metadata when rules define labels, otherwise use the first rule."""
|
|
90
|
+
if not rules:
|
|
91
|
+
return None
|
|
92
|
+
metadata_rules = {}
|
|
93
|
+
for rule in rules:
|
|
94
|
+
metadata = dict(getattr(rule, "metadata", {}) or {})
|
|
95
|
+
for key, value in metadata.items():
|
|
96
|
+
metadata_rules.setdefault(key, {})[value] = rule
|
|
97
|
+
if not metadata_rules:
|
|
98
|
+
return rules[0]
|
|
99
|
+
for key, values in metadata_rules.items():
|
|
100
|
+
matched = values.get(instance.metadata.get(key))
|
|
101
|
+
if matched is not None:
|
|
102
|
+
return matched
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _options_from_rule(rule, config: LosslessConfig) -> LosslessOptions:
|
|
107
|
+
online = _submessage(rule, "losslessOnline")
|
|
108
|
+
delay = _submessage(online, "delayRegister")
|
|
109
|
+
warmup = _submessage(online, "warmup")
|
|
110
|
+
readiness = _submessage(online, "readiness")
|
|
111
|
+
offline = _submessage(rule, "losslessOffline")
|
|
112
|
+
strategy = config.strategy
|
|
113
|
+
if delay is not None:
|
|
114
|
+
try:
|
|
115
|
+
strategy = lossless_pb2.DelayRegister.DelayStrategy.Name(delay.strategy)
|
|
116
|
+
except ValueError:
|
|
117
|
+
strategy = config.strategy
|
|
118
|
+
if strategy not in STRATEGIES:
|
|
119
|
+
strategy = DELAY_BY_TIME
|
|
120
|
+
interval_second = delay.intervalSecond if delay is not None and delay.intervalSecond > 0 else 0
|
|
121
|
+
try:
|
|
122
|
+
health_check_second = int(delay.healthCheckIntervalSecond) if delay is not None else 0
|
|
123
|
+
except (TypeError, ValueError):
|
|
124
|
+
health_check_second = 0
|
|
125
|
+
return LosslessOptions(
|
|
126
|
+
delay_register_enable=getattr(delay, "enable", True) if delay is not None else True,
|
|
127
|
+
delay_register_interval_ms=interval_second * 1000 if interval_second > 0 else config.delay_register_interval,
|
|
128
|
+
health_check_interval_ms=(
|
|
129
|
+
health_check_second * 1000 if health_check_second > 0 else config.health_check_interval
|
|
130
|
+
),
|
|
131
|
+
strategy=strategy,
|
|
132
|
+
health_check_path=delay.healthCheckPath if delay is not None and delay.healthCheckPath else config.health_check_path,
|
|
133
|
+
warmup_enable=bool(getattr(warmup, "enable", False)) if warmup is not None else False,
|
|
134
|
+
warmup_interval_second=int(getattr(warmup, "intervalSecond", 0) or 0) if warmup is not None else 0,
|
|
135
|
+
warmup_curvature=float(warmup.curvature) if warmup is not None else 1.0,
|
|
136
|
+
warmup_overload_protection=(
|
|
137
|
+
bool(getattr(warmup, "enableOverloadProtection", False)) if warmup is not None else False
|
|
138
|
+
),
|
|
139
|
+
warmup_overload_threshold=int(getattr(warmup, "overloadProtectionThreshold", 0) or 0) if warmup else 0,
|
|
140
|
+
offline_enable=getattr(offline, "enable", True) if offline is not None else True,
|
|
141
|
+
readiness_enable=getattr(readiness, "enable", True) if readiness is not None else True,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _options_from_config(config: LosslessConfig) -> LosslessOptions:
|
|
146
|
+
return LosslessOptions(
|
|
147
|
+
delay_register_enable=True,
|
|
148
|
+
delay_register_interval_ms=config.delay_register_interval,
|
|
149
|
+
health_check_interval_ms=config.health_check_interval,
|
|
150
|
+
strategy=config.strategy,
|
|
151
|
+
health_check_path=config.health_check_path,
|
|
152
|
+
warmup_enable=False,
|
|
153
|
+
warmup_interval_second=0,
|
|
154
|
+
warmup_curvature=1.0,
|
|
155
|
+
warmup_overload_protection=False,
|
|
156
|
+
warmup_overload_threshold=0,
|
|
157
|
+
offline_enable=True,
|
|
158
|
+
readiness_enable=True,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _parse_ctime(instance: Instance) -> Optional[datetime]:
|
|
163
|
+
value = instance.instance.ctime.value if instance.instance.HasField("ctime") else ""
|
|
164
|
+
if not value:
|
|
165
|
+
return None
|
|
166
|
+
try:
|
|
167
|
+
parsed = datetime.fromisoformat(value)
|
|
168
|
+
except ValueError:
|
|
169
|
+
return None
|
|
170
|
+
if parsed.tzinfo is None:
|
|
171
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
172
|
+
return parsed
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class _WarmupInstance(Instance):
|
|
176
|
+
"""Present a warmed-up instance weight without mutating the cached snapshot."""
|
|
177
|
+
|
|
178
|
+
def __init__(self, instance: Instance, weight: int):
|
|
179
|
+
super().__init__(instance.instance, instance.service_key, instance.local_value)
|
|
180
|
+
self._warmup_weight = weight
|
|
181
|
+
|
|
182
|
+
def get_weight(self):
|
|
183
|
+
return self._warmup_weight
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class LosslessManager:
|
|
187
|
+
"""Coordinate delayed registration, lossless deregister and warmup weights."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, config: Configuration, connector) -> None:
|
|
190
|
+
self._config = config
|
|
191
|
+
self._lossless_config = config.get_provider().get_lossless()
|
|
192
|
+
self._connector = connector
|
|
193
|
+
self.enabled = self._lossless_config.enable
|
|
194
|
+
self._action_providers: Dict[LosslessInstance, LosslessActionProvider] = {}
|
|
195
|
+
self._register_statuses: Dict[LosslessInstance, RegisterStatus] = {}
|
|
196
|
+
self._register_timestamps: Dict[LosslessInstance, float] = {}
|
|
197
|
+
self._register_tasks: Dict[LosslessInstance, asyncio.Task] = {}
|
|
198
|
+
self._register_generations: Dict[LosslessInstance, int] = {}
|
|
199
|
+
self._desired_statuses: Dict[LosslessInstance, RegisterStatus] = {}
|
|
200
|
+
self._rules_cache: Dict[tuple, tuple] = {}
|
|
201
|
+
self._rules_fetching: set = set()
|
|
202
|
+
self._rule_refresh_tasks: Dict[tuple, set] = {}
|
|
203
|
+
self._rule_query_timeout = config.get_global().get_api().get_timeout() / 1000
|
|
204
|
+
self._closed = False
|
|
205
|
+
lossless_logger.info(
|
|
206
|
+
"lossless controller initialized enabled=%s strategy=%s "
|
|
207
|
+
"delayRegisterInterval=%dms healthCheckInterval=%dms",
|
|
208
|
+
self.enabled, self._lossless_config.get_strategy(),
|
|
209
|
+
self._lossless_config.get_delay_register_interval(),
|
|
210
|
+
self._lossless_config.get_health_check_interval(),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
def set_action_provider(self, instance: LosslessInstance, provider: LosslessActionProvider) -> None:
|
|
214
|
+
self._action_providers[instance] = provider
|
|
215
|
+
|
|
216
|
+
def get_action_provider(self, instance: LosslessInstance) -> Optional[LosslessActionProvider]:
|
|
217
|
+
return self._action_providers.get(instance)
|
|
218
|
+
|
|
219
|
+
async def fetch_rules(self, namespace: str, service: str) -> List:
|
|
220
|
+
"""Return cached rules immediately, or load them within the API timeout."""
|
|
221
|
+
key = (namespace, service)
|
|
222
|
+
cached = self._rules_cache.get(key)
|
|
223
|
+
if cached is not None:
|
|
224
|
+
if time.monotonic() - cached[2] >= RULE_REFRESH_INTERVAL_SECONDS:
|
|
225
|
+
self.touch_rules(namespace, service)
|
|
226
|
+
return list(cached[0])
|
|
227
|
+
return await self._fetch_rules_remote(namespace, service)
|
|
228
|
+
|
|
229
|
+
async def _fetch_rules_remote(self, namespace: str, service: str) -> List:
|
|
230
|
+
"""Refresh one rule resource with a bounded overall query wait."""
|
|
231
|
+
key = (namespace, service)
|
|
232
|
+
try:
|
|
233
|
+
request = InstanceDiscoverRequest(
|
|
234
|
+
type=DISCOVER_TYPE_LOSSLESS, name=service, namespace=namespace, revision="",
|
|
235
|
+
)
|
|
236
|
+
async with asyncio.timeout(self._rule_query_timeout):
|
|
237
|
+
response = await self._connector.discover(request)
|
|
238
|
+
rules = _extract_rules(response)
|
|
239
|
+
except Exception as error:
|
|
240
|
+
lossless_logger.warning("failed to fetch lossless rules for %s/%s: %s", namespace, service, error)
|
|
241
|
+
cached = self._rules_cache.get(key)
|
|
242
|
+
return list(cached[0]) if cached else []
|
|
243
|
+
revision = response.service.revision.value if response.service.HasField("revision") else ""
|
|
244
|
+
previous = self._rules_cache.get(key)
|
|
245
|
+
self._rules_cache[key] = (rules, revision, time.monotonic())
|
|
246
|
+
log_rules = (lossless_logger.info if previous is None or previous[1] != revision
|
|
247
|
+
else lossless_logger.debug)
|
|
248
|
+
log_rules(
|
|
249
|
+
"lossless rules fetched namespace=%s service=%s rules=%d revision=%s",
|
|
250
|
+
namespace, service, len(rules), revision,
|
|
251
|
+
)
|
|
252
|
+
return list(rules)
|
|
253
|
+
|
|
254
|
+
def touch_rules(self, namespace: str, service: str) -> None:
|
|
255
|
+
"""Refresh stale rules in the background; used by the consumer warmup path."""
|
|
256
|
+
if self._closed:
|
|
257
|
+
return
|
|
258
|
+
key = (namespace, service)
|
|
259
|
+
cached = self._rules_cache.get(key)
|
|
260
|
+
if cached is not None and time.monotonic() - cached[2] < RULE_REFRESH_INTERVAL_SECONDS:
|
|
261
|
+
return
|
|
262
|
+
if key in self._rules_fetching:
|
|
263
|
+
return
|
|
264
|
+
|
|
265
|
+
async def refresh() -> None:
|
|
266
|
+
try:
|
|
267
|
+
await self._fetch_rules_remote(namespace, service)
|
|
268
|
+
finally:
|
|
269
|
+
self._rules_fetching.discard(key)
|
|
270
|
+
|
|
271
|
+
self._rules_fetching.add(key)
|
|
272
|
+
task = asyncio.create_task(refresh(), name=f"polaris-lossless-rules-{namespace}-{service}")
|
|
273
|
+
self._rule_refresh_tasks.setdefault(key, set()).add(task)
|
|
274
|
+
task.add_done_callback(self._rule_refresh_tasks[key].discard)
|
|
275
|
+
|
|
276
|
+
def _cached_rules(self, namespace: str, service: str) -> List:
|
|
277
|
+
cached = self._rules_cache.get((namespace, service))
|
|
278
|
+
return list(cached[0]) if cached else []
|
|
279
|
+
|
|
280
|
+
async def effective_options(self, instance: LosslessInstance) -> LosslessOptions:
|
|
281
|
+
"""Resolve server rules first, then fall back to local configuration."""
|
|
282
|
+
rules = await self.fetch_rules(instance.namespace, instance.service)
|
|
283
|
+
rule = _match_rule(instance, rules)
|
|
284
|
+
if rule is not None:
|
|
285
|
+
return _options_from_rule(rule, self._lossless_config)
|
|
286
|
+
return _options_from_config(self._lossless_config)
|
|
287
|
+
|
|
288
|
+
def cached_options(self, instance: LosslessInstance) -> LosslessOptions:
|
|
289
|
+
rule = _match_rule(instance, self._cached_rules(instance.namespace, instance.service))
|
|
290
|
+
if rule is not None:
|
|
291
|
+
return _options_from_rule(rule, self._lossless_config)
|
|
292
|
+
return _options_from_config(self._lossless_config)
|
|
293
|
+
|
|
294
|
+
def _next_register_generation(self, instance: LosslessInstance) -> int:
|
|
295
|
+
generation = self._register_generations.get(instance, 0) + 1
|
|
296
|
+
self._register_generations[instance] = generation
|
|
297
|
+
return generation
|
|
298
|
+
|
|
299
|
+
def _is_current_register(self, instance: LosslessInstance, generation: int) -> bool:
|
|
300
|
+
return not self._closed and self._register_generations.get(instance) == generation
|
|
301
|
+
|
|
302
|
+
async def lossless_register(self, instance: LosslessInstance) -> None:
|
|
303
|
+
"""Gate the real registration behind delay or health checking."""
|
|
304
|
+
generation = self._next_register_generation(instance)
|
|
305
|
+
provider = self._action_providers.get(instance)
|
|
306
|
+
if provider is None:
|
|
307
|
+
lossless_logger.warning("no lossless action provider for %s, skip lossless register", instance)
|
|
308
|
+
return
|
|
309
|
+
self._desired_statuses[instance] = RegisterStatus.REGISTERED
|
|
310
|
+
if not self.enabled:
|
|
311
|
+
lossless_logger.info("lossless disabled, register %s directly", instance)
|
|
312
|
+
await self._do_register(instance, generation)
|
|
313
|
+
return
|
|
314
|
+
options = await self.effective_options(instance)
|
|
315
|
+
if not self._is_current_register(instance, generation):
|
|
316
|
+
lossless_logger.info("lossless register for %s was superseded", instance)
|
|
317
|
+
return
|
|
318
|
+
provider = self._action_providers.get(instance)
|
|
319
|
+
if provider is None:
|
|
320
|
+
lossless_logger.warning("no lossless action provider for %s, skip lossless register", instance)
|
|
321
|
+
return
|
|
322
|
+
if not options.delay_register_enable:
|
|
323
|
+
lossless_logger.info("delay register disabled for %s, register now", instance)
|
|
324
|
+
await self._do_register(instance, generation)
|
|
325
|
+
return
|
|
326
|
+
previous = self._register_tasks.pop(instance, None)
|
|
327
|
+
if previous is not None and not previous.done():
|
|
328
|
+
previous.cancel()
|
|
329
|
+
if options.strategy == DELAY_BY_HEALTH_CHECK and provider.is_enable_health_check():
|
|
330
|
+
self._register_tasks[instance] = asyncio.create_task(
|
|
331
|
+
self._run_health_check_register(instance, options, generation),
|
|
332
|
+
name=f"polaris-lossless-register-{instance.namespace}-{instance.service}",
|
|
333
|
+
)
|
|
334
|
+
else:
|
|
335
|
+
self._register_tasks[instance] = asyncio.create_task(
|
|
336
|
+
self._run_delayed_register(instance, options, generation),
|
|
337
|
+
name=f"polaris-lossless-register-{instance.namespace}-{instance.service}",
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
async def _run_delayed_register(
|
|
341
|
+
self, instance: LosslessInstance, options: LosslessOptions, generation: int) -> None:
|
|
342
|
+
await asyncio.sleep(options.delay_register_interval_ms / 1000)
|
|
343
|
+
if not self._is_current_register(instance, generation):
|
|
344
|
+
return
|
|
345
|
+
lossless_logger.info("lossless delay finished for %s, start register", instance)
|
|
346
|
+
await self._do_register(instance, generation)
|
|
347
|
+
|
|
348
|
+
async def _run_health_check_register(
|
|
349
|
+
self, instance: LosslessInstance, options: LosslessOptions, generation: int) -> None:
|
|
350
|
+
provider = self._action_providers.get(instance)
|
|
351
|
+
if provider is None:
|
|
352
|
+
return
|
|
353
|
+
while self._is_current_register(instance, generation):
|
|
354
|
+
await asyncio.sleep(options.health_check_interval_ms / 1000)
|
|
355
|
+
if not self._is_current_register(instance, generation):
|
|
356
|
+
return
|
|
357
|
+
try:
|
|
358
|
+
ready = await provider.do_health_check()
|
|
359
|
+
except Exception as error:
|
|
360
|
+
lossless_logger.error("lossless health check failed for %s: %s", instance, error)
|
|
361
|
+
ready = False
|
|
362
|
+
if ready:
|
|
363
|
+
lossless_logger.info("lossless health check passed for %s, start register", instance)
|
|
364
|
+
await self._do_register(instance, generation)
|
|
365
|
+
return
|
|
366
|
+
|
|
367
|
+
async def _do_register(self, instance: LosslessInstance, generation: int | None = None) -> None:
|
|
368
|
+
if generation is not None and not self._is_current_register(instance, generation):
|
|
369
|
+
return
|
|
370
|
+
if self._action_providers.get(instance) is None:
|
|
371
|
+
return
|
|
372
|
+
if generation is None:
|
|
373
|
+
self._next_register_generation(instance)
|
|
374
|
+
self._desired_statuses[instance] = RegisterStatus.REGISTERED
|
|
375
|
+
await self._reconcile_instance(instance)
|
|
376
|
+
|
|
377
|
+
async def lossless_deregister(self, instance: LosslessInstance) -> None:
|
|
378
|
+
"""Deregister immediately so consumers can remove the instance."""
|
|
379
|
+
self._next_register_generation(instance)
|
|
380
|
+
if self._action_providers.get(instance) is None:
|
|
381
|
+
lossless_logger.warning("no lossless action provider for %s, skip lossless deregister", instance)
|
|
382
|
+
return
|
|
383
|
+
self._desired_statuses[instance] = RegisterStatus.UNREGISTERED
|
|
384
|
+
task = self._register_tasks.pop(instance, None)
|
|
385
|
+
if task is not None and not task.done():
|
|
386
|
+
task.cancel()
|
|
387
|
+
self._register_statuses[instance] = RegisterStatus.UNREGISTERED
|
|
388
|
+
await self._reconcile_instance(instance)
|
|
389
|
+
|
|
390
|
+
async def _reconcile_instance(self, instance: LosslessInstance) -> None:
|
|
391
|
+
"""Apply the latest intent after an RPC, replaying when an older RPC raced it."""
|
|
392
|
+
properties = InstanceProperties()
|
|
393
|
+
while not self._closed:
|
|
394
|
+
generation = self._register_generations.get(instance)
|
|
395
|
+
desired = self._desired_statuses.get(instance)
|
|
396
|
+
if generation is None or desired is None:
|
|
397
|
+
return
|
|
398
|
+
provider = self._action_providers.get(instance)
|
|
399
|
+
if provider is None:
|
|
400
|
+
lossless_logger.warning("no lossless action provider for %s during replay", instance)
|
|
401
|
+
return
|
|
402
|
+
|
|
403
|
+
if desired == RegisterStatus.REGISTERED:
|
|
404
|
+
await provider.do_register(properties)
|
|
405
|
+
else:
|
|
406
|
+
await provider.do_deregister()
|
|
407
|
+
|
|
408
|
+
if self._closed:
|
|
409
|
+
return
|
|
410
|
+
if (self._register_generations.get(instance) == generation
|
|
411
|
+
and self._desired_statuses.get(instance) == desired):
|
|
412
|
+
if desired == RegisterStatus.REGISTERED:
|
|
413
|
+
self._register_statuses[instance] = RegisterStatus.REGISTERED
|
|
414
|
+
self._register_timestamps[instance] = time.time()
|
|
415
|
+
else:
|
|
416
|
+
self._register_statuses[instance] = RegisterStatus.UNREGISTERED
|
|
417
|
+
return
|
|
418
|
+
|
|
419
|
+
# A newer lifecycle intent arrived while this RPC was in flight. The
|
|
420
|
+
# completed RPC may have changed the provider after that intent, so
|
|
421
|
+
# replay the current intent instead of trusting the stale result.
|
|
422
|
+
lossless_logger.info("replaying latest lossless intent for %s", instance)
|
|
423
|
+
|
|
424
|
+
def check_register_status(self) -> RegisterStatus:
|
|
425
|
+
"""Return REGISTERED only when every tracked instance is registered."""
|
|
426
|
+
if not self._action_providers:
|
|
427
|
+
return RegisterStatus.UNREGISTERED
|
|
428
|
+
for instance, provider in self._action_providers.items():
|
|
429
|
+
if not self._readiness_enabled(instance):
|
|
430
|
+
continue
|
|
431
|
+
if self._register_statuses.get(instance) != RegisterStatus.REGISTERED:
|
|
432
|
+
return RegisterStatus.UNREGISTERED
|
|
433
|
+
return RegisterStatus.REGISTERED
|
|
434
|
+
|
|
435
|
+
def _readiness_enabled(self, instance: LosslessInstance) -> bool:
|
|
436
|
+
return self.cached_options(instance).readiness_enable
|
|
437
|
+
|
|
438
|
+
def _offline_instances(self) -> List[LosslessInstance]:
|
|
439
|
+
return [
|
|
440
|
+
instance for instance in self._action_providers
|
|
441
|
+
if self.cached_options(instance).offline_enable
|
|
442
|
+
]
|
|
443
|
+
|
|
444
|
+
async def offline_all(self):
|
|
445
|
+
"""Deregister every instance that has lossless offline enabled."""
|
|
446
|
+
instances = self._offline_instances()
|
|
447
|
+
if not instances:
|
|
448
|
+
return None
|
|
449
|
+
for instance in instances:
|
|
450
|
+
await self.lossless_deregister(instance)
|
|
451
|
+
return instances
|
|
452
|
+
|
|
453
|
+
def apply_warmup_weights(self, instances: ServiceInstances) -> ServiceInstances:
|
|
454
|
+
"""Return a snapshot with warmup-adjusted weights, matching Java overload protection."""
|
|
455
|
+
namespace = instances.get_namespace()
|
|
456
|
+
service = instances.get_service()
|
|
457
|
+
rules = self._cached_rules(namespace, service)
|
|
458
|
+
if not rules or not instances.instances:
|
|
459
|
+
return instances
|
|
460
|
+
now = time.time()
|
|
461
|
+
warming = []
|
|
462
|
+
for node in instances.instances:
|
|
463
|
+
rule = _match_rule(LosslessInstance(
|
|
464
|
+
namespace=node.get_namespace(), service=node.get_service(),
|
|
465
|
+
host=node.get_host(), port=node.get_port(), metadata=dict(node.get_metadata() or {}),
|
|
466
|
+
), rules)
|
|
467
|
+
if rule is None:
|
|
468
|
+
continue
|
|
469
|
+
options = _options_from_rule(rule, self._lossless_config)
|
|
470
|
+
if not options.warmup_enable or options.warmup_interval_second <= 0:
|
|
471
|
+
continue
|
|
472
|
+
ctime = _parse_ctime(node)
|
|
473
|
+
if ctime is None:
|
|
474
|
+
continue
|
|
475
|
+
uptime = max(0.0, now - ctime.timestamp())
|
|
476
|
+
if uptime >= options.warmup_interval_second:
|
|
477
|
+
continue
|
|
478
|
+
warming.append((node, options, uptime))
|
|
479
|
+
if not warming:
|
|
480
|
+
return instances
|
|
481
|
+
total = len(instances.instances)
|
|
482
|
+
warming_percent = len(warming) * 100 // total
|
|
483
|
+
for _, options, _ in warming:
|
|
484
|
+
if options.warmup_overload_protection and warming_percent >= options.warmup_overload_threshold:
|
|
485
|
+
return instances
|
|
486
|
+
result = _copy_instances(instances)
|
|
487
|
+
for node, options, uptime in warming:
|
|
488
|
+
base_weight = node.get_weight()
|
|
489
|
+
ratio = uptime / options.warmup_interval_second
|
|
490
|
+
weight = min(base_weight, math.ceil(math.pow(ratio, options.warmup_curvature) * base_weight))
|
|
491
|
+
if weight < base_weight:
|
|
492
|
+
index = result.instances.index(node)
|
|
493
|
+
result.instances[index] = _WarmupInstance(node, weight)
|
|
494
|
+
lossless_logger.debug(
|
|
495
|
+
"lossless warmup applied namespace=%s service=%s instances=%d",
|
|
496
|
+
namespace, service, len(warming),
|
|
497
|
+
)
|
|
498
|
+
return result
|
|
499
|
+
|
|
500
|
+
def warmup_start_timestamp(self, instance: LosslessInstance) -> Optional[float]:
|
|
501
|
+
return self._register_timestamps.get(instance)
|
|
502
|
+
|
|
503
|
+
async def close(self) -> None:
|
|
504
|
+
self._closed = True
|
|
505
|
+
tasks = [task for task in self._register_tasks.values() if not task.done()]
|
|
506
|
+
for task_set in self._rule_refresh_tasks.values():
|
|
507
|
+
tasks.extend(task for task in task_set if not task.done())
|
|
508
|
+
for task in tasks:
|
|
509
|
+
task.cancel()
|
|
510
|
+
if tasks:
|
|
511
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
512
|
+
self._register_tasks.clear()
|
|
513
|
+
self._rule_refresh_tasks.clear()
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _copy_instances(instances: ServiceInstances) -> ServiceInstances:
|
|
517
|
+
import copy
|
|
518
|
+
|
|
519
|
+
result = copy.copy(instances)
|
|
520
|
+
result.instances = list(instances.instances)
|
|
521
|
+
return result
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
class HttpLosslessActionProvider(LosslessActionProvider):
|
|
525
|
+
"""Action provider that wraps real register/deregister and HTTP health checks."""
|
|
526
|
+
|
|
527
|
+
def __init__(self, instance: LosslessInstance, register_action, deregister_action,
|
|
528
|
+
strategy_resolver, health_check_path_resolver) -> None:
|
|
529
|
+
self._instance = instance
|
|
530
|
+
self._register_action = register_action
|
|
531
|
+
self._deregister_action = deregister_action
|
|
532
|
+
self._strategy_resolver = strategy_resolver
|
|
533
|
+
self._health_check_path_resolver = health_check_path_resolver
|
|
534
|
+
|
|
535
|
+
def get_name(self) -> str:
|
|
536
|
+
return "http"
|
|
537
|
+
|
|
538
|
+
async def do_health_check(self) -> bool:
|
|
539
|
+
path = self._health_check_path_resolver()
|
|
540
|
+
url = f"http://127.0.0.1:{self._instance.port}{path}"
|
|
541
|
+
timeout = ClientTimeout(total=2)
|
|
542
|
+
try:
|
|
543
|
+
async with ClientSession(timeout=timeout) as session:
|
|
544
|
+
async with session.get(url, headers={"User-Agent": "polaris"}) as response:
|
|
545
|
+
if response.status < 400:
|
|
546
|
+
return True
|
|
547
|
+
lossless_logger.error(
|
|
548
|
+
"lossless health check returned status=%d for %s", response.status, self._instance,
|
|
549
|
+
)
|
|
550
|
+
return False
|
|
551
|
+
except Exception as error:
|
|
552
|
+
lossless_logger.error("lossless health check failed for %s: %s", self._instance, error)
|
|
553
|
+
return False
|
|
554
|
+
|
|
555
|
+
def is_enable_health_check(self) -> bool:
|
|
556
|
+
return self._strategy_resolver() == DELAY_BY_HEALTH_CHECK
|
|
557
|
+
|
|
558
|
+
async def do_register(self, instance_properties: InstanceProperties):
|
|
559
|
+
await self._register_action(instance_properties)
|
|
560
|
+
|
|
561
|
+
async def do_deregister(self):
|
|
562
|
+
await self._deregister_action()
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def lossless_http_handlers(manager: LosslessManager):
|
|
566
|
+
"""Build the Java-compatible /offline and /readiness endpoints."""
|
|
567
|
+
|
|
568
|
+
async def offline(request: web.Request) -> web.Response:
|
|
569
|
+
remote = request.remote or ""
|
|
570
|
+
try:
|
|
571
|
+
loopback = ipaddress.ip_address(remote).is_loopback
|
|
572
|
+
except ValueError:
|
|
573
|
+
loopback = False
|
|
574
|
+
if not loopback:
|
|
575
|
+
return web.Response(text="only localhost can call this path", status=403)
|
|
576
|
+
if not manager.enabled:
|
|
577
|
+
return web.Response(text="no policy", status=500)
|
|
578
|
+
if not manager._action_providers:
|
|
579
|
+
return web.Response(text="no action", status=200)
|
|
580
|
+
instances = manager._offline_instances()
|
|
581
|
+
if not instances:
|
|
582
|
+
return web.Response(status=404)
|
|
583
|
+
try:
|
|
584
|
+
await manager.offline_all()
|
|
585
|
+
except Exception as error:
|
|
586
|
+
lossless_logger.error("failed to execute lossless offline: %s", error)
|
|
587
|
+
return web.Response(text=f"failed: {error}", status=500)
|
|
588
|
+
return web.Response(text="ok", status=200)
|
|
589
|
+
|
|
590
|
+
async def readiness(request: web.Request) -> web.Response:
|
|
591
|
+
del request
|
|
592
|
+
instances = [
|
|
593
|
+
instance for instance in manager._action_providers
|
|
594
|
+
if manager._readiness_enabled(instance)
|
|
595
|
+
]
|
|
596
|
+
if not instances:
|
|
597
|
+
return web.Response(status=404)
|
|
598
|
+
status = manager.check_register_status()
|
|
599
|
+
return web.Response(text=status.value, status=200 if status == RegisterStatus.REGISTERED else 503)
|
|
600
|
+
|
|
601
|
+
return (
|
|
602
|
+
("*", "/offline", offline),
|
|
603
|
+
("GET", "/readiness", readiness),
|
|
604
|
+
("GET", "/online", readiness),
|
|
605
|
+
)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Tencent is pleased to support the open source community by making polaris-python available.
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2021 Tencent. All rights reserved.
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the BSD 3-Clause License (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# https://opensource.org/licenses/BSD-3-Clause
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software distributed
|
|
12
|
+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
13
|
+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
14
|
+
# specific language governing permissions and limitations under the License.
|
|
15
|
+
|
|
16
|
+
from polaris.pkg.model.core import ServiceKey
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# 上下文表示
|
|
20
|
+
class ContextKey:
|
|
21
|
+
def __init__(self, service_key: ServiceKey, operation: str):
|
|
22
|
+
# 服务信息
|
|
23
|
+
self.service_key = service_key
|
|
24
|
+
# 操作信息
|
|
25
|
+
self.operation = operation
|
|
26
|
+
|
|
27
|
+
def string(self):
|
|
28
|
+
return "ServiceKey:{service_key}. Operation:{opratrion}".format(service_key=self.service_key,
|
|
29
|
+
opratrion=self.operation)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SingleNotifyContext:
|
|
33
|
+
def __int__(self, name: ContextKey):
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class CombineNotifyContext:
|
|
38
|
+
def __init__(self, service_key: ServiceKey, awit_count: int, ):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
KeySourceRoute = "sourceRoute"
|
|
43
|
+
KeyDstRoute = "destinationRoute"
|
|
44
|
+
KeyDstInstances = "destinationInstances"
|
|
45
|
+
KeyServiceRoute = "serviceRoute"
|
|
46
|
+
KeyServiceInstance = "serviceInstance"
|