dbca-utils 3.0.11__tar.gz → 3.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/PKG-INFO +4 -2
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/pyproject.toml +4 -2
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/apps.py +3 -6
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/healthcheck/healthcheck.py +458 -257
- dbca_utils-3.0.11/src/dbca_utils/healthcheck/urls.py +0 -16
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/LICENSE +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/README.md +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/__init__.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/healthcheck/__init__.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/middleware.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/models.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/src/dbca_utils/utils.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/__init__.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/apps.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/migrations/0001_initial.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/migrations/__init__.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/models.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/settings.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/templates/tests/test_model_list.html +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/tests.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/urls.py +0 -0
- {dbca_utils-3.0.11 → dbca_utils-3.0.12}/tests/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbca-utils
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.12
|
|
4
4
|
Summary: Utilities for DBCA Django apps
|
|
5
5
|
Author-Email: Rocky Chen <rocky.chen@dbca.wa.gov.au>, Ashley Felton <ashley.felton@dbca.wa.gov.au>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -23,7 +23,9 @@ Project-URL: Changelog, https://github.com/dbca-wa/dbca-utils/blob/main/CHANGELO
|
|
|
23
23
|
Project-URL: GitHub, https://github.com/dbca-wa/dbca-utils
|
|
24
24
|
Requires-Python: <4.0,>=3.12
|
|
25
25
|
Requires-Dist: django<6.2,>=5.2
|
|
26
|
-
Requires-Dist: markupsafe>=3.0
|
|
26
|
+
Requires-Dist: markupsafe>=3.0
|
|
27
|
+
Requires-Dist: psutil>=7.2
|
|
28
|
+
Requires-Dist: requests>=2.33
|
|
27
29
|
Description-Content-Type: text/markdown
|
|
28
30
|
|
|
29
31
|
# Overview
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dbca-utils"
|
|
3
|
-
version = "3.0.
|
|
3
|
+
version = "3.0.12"
|
|
4
4
|
description = "Utilities for DBCA Django apps"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Rocky Chen", email = "rocky.chen@dbca.wa.gov.au" },
|
|
@@ -26,7 +26,9 @@ classifiers = [
|
|
|
26
26
|
requires-python = ">=3.12,<4.0"
|
|
27
27
|
dependencies = [
|
|
28
28
|
"django>=5.2,<6.2",
|
|
29
|
-
"markupsafe>=3.0
|
|
29
|
+
"markupsafe>=3.0",
|
|
30
|
+
"psutil>=7.2",
|
|
31
|
+
"requests>=2.33",
|
|
30
32
|
]
|
|
31
33
|
|
|
32
34
|
[project.urls]
|
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
import atexit
|
|
2
|
-
|
|
3
1
|
from django.apps import AppConfig
|
|
2
|
+
|
|
4
3
|
from .healthcheck import healthcheck
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
class DbcaUtilsConfig(AppConfig):
|
|
8
|
-
name =
|
|
7
|
+
name = "dbca_utils"
|
|
9
8
|
|
|
10
9
|
def ready(self):
|
|
11
10
|
if healthcheck.HEALTHCHECK_ENABLED:
|
|
12
|
-
healthcheck.
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
healthcheck.register_healthcheck_urls()
|
|
@@ -1,46 +1,48 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import importlib
|
|
3
|
-
from collections import OrderedDict
|
|
4
2
|
import logging
|
|
5
|
-
import
|
|
6
|
-
import subprocess
|
|
7
|
-
import random
|
|
3
|
+
import os
|
|
8
4
|
import re
|
|
9
|
-
import
|
|
5
|
+
import secrets
|
|
10
6
|
import socket
|
|
11
|
-
import
|
|
7
|
+
import time
|
|
8
|
+
from collections import OrderedDict
|
|
12
9
|
from datetime import datetime
|
|
13
10
|
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
import psutil
|
|
12
|
+
import requests
|
|
16
13
|
from django.conf import settings
|
|
17
|
-
from django.http import HttpResponseForbidden, JsonResponse,HttpResponseServerError
|
|
18
|
-
from django.core.signals import request_started
|
|
19
14
|
from django.core.cache import cache
|
|
15
|
+
from django.core.signals import request_started
|
|
16
|
+
from django.http import JsonResponse
|
|
17
|
+
from django.urls import include, path, reverse
|
|
18
|
+
from django.utils import timezone
|
|
20
19
|
|
|
21
20
|
logger = logging.getLogger(__name__)
|
|
21
|
+
KB = 1024
|
|
22
|
+
MB = 1024**2
|
|
23
|
+
GB = 1024**3
|
|
22
24
|
|
|
23
25
|
|
|
24
|
-
#WORKLOADS means the number of WORKLOADS should be started.
|
|
25
|
-
#If WORKLOADS is dynamic, please don't set it.
|
|
26
|
-
HEALTHCHECK_ENABLED = os.environ.get("HEALTHCHECK_ENABLED","true").lower() == "true"
|
|
26
|
+
# WORKLOADS means the number of WORKLOADS should be started.
|
|
27
|
+
# If WORKLOADS is dynamic, please don't set it.
|
|
28
|
+
HEALTHCHECK_ENABLED = os.environ.get("HEALTHCHECK_ENABLED", "true").lower() == "true"
|
|
27
29
|
if not HEALTHCHECK_ENABLED:
|
|
28
30
|
HEALTHCHECK_ENABLED = True if cache else None
|
|
29
31
|
|
|
30
|
-
HEALTHCHECK_SYSTEMDATA_ENABLED = os.environ.get("HEALTHCHECK_SYSTEMDATA_ENABLED","true").lower() == "true"
|
|
31
|
-
HEALTHCHECK_PROCESSDATA_ENABLED = os.environ.get("HEALTHCHECK_PROCESSDATA_ENABLED","true").lower() == "true"
|
|
32
|
+
HEALTHCHECK_SYSTEMDATA_ENABLED = os.environ.get("HEALTHCHECK_SYSTEMDATA_ENABLED", "true").lower() == "true"
|
|
33
|
+
HEALTHCHECK_PROCESSDATA_ENABLED = os.environ.get("HEALTHCHECK_PROCESSDATA_ENABLED", "true").lower() == "true"
|
|
32
34
|
|
|
33
|
-
CACHE_PREFIX = os.environ.get("CACHE_PREFIX","")
|
|
34
|
-
PORT = int(os.environ.get("WORKLOAD_PORT",8080))
|
|
35
|
-
WORKLOADS = int(os.environ.get("WORKLOADS",0))
|
|
36
|
-
WORKLOAD_DEPLOYMENT = os.environ.get("WORKLOAD_DEPLOYMENT","true").lower() == "true"
|
|
37
|
-
if WORKLOADS < 0
|
|
35
|
+
CACHE_PREFIX = os.environ.get("CACHE_PREFIX", "")
|
|
36
|
+
PORT = int(os.environ.get("WORKLOAD_PORT", 8080))
|
|
37
|
+
WORKLOADS = int(os.environ.get("WORKLOADS", 0))
|
|
38
|
+
WORKLOAD_DEPLOYMENT = os.environ.get("WORKLOAD_DEPLOYMENT", "true").lower() == "true"
|
|
39
|
+
if WORKLOADS < 0:
|
|
38
40
|
WORKLOADS = 0
|
|
39
|
-
WORKLOAD_FAILED_THRESHOLD = int(os.environ.get("WORKLOAD_FAILED_THRESHOLD",2))
|
|
41
|
+
WORKLOAD_FAILED_THRESHOLD = int(os.environ.get("WORKLOAD_FAILED_THRESHOLD", 2))
|
|
40
42
|
|
|
41
|
-
WORKLOAD_VOLUMES = os.environ.get("WORKLOAD_VOLUMES","automatic")
|
|
43
|
+
WORKLOAD_VOLUMES = os.environ.get("WORKLOAD_VOLUMES", "automatic")
|
|
42
44
|
|
|
43
|
-
if not WORKLOAD_VOLUMES or WORKLOAD_VOLUMES.lower() in ("disabled","false"):
|
|
45
|
+
if not WORKLOAD_VOLUMES or WORKLOAD_VOLUMES.lower() in ("disabled", "false"):
|
|
44
46
|
WORKLOAD_VOLUMES_ENABLED = False
|
|
45
47
|
WORKLOAD_VOLUMES = None
|
|
46
48
|
elif WORKLOAD_VOLUMES.lower() == "automatic":
|
|
@@ -51,31 +53,56 @@ else:
|
|
|
51
53
|
WORKLOAD_VOLUMES_ENABLED = True if WORKLOAD_VOLUMES else False
|
|
52
54
|
|
|
53
55
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
def generate_secret(length=32):
|
|
57
|
+
"""Generate a cryptographically secure random secret for bearer-token authentication.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
length (int): Length of the token in bytes (default 32).
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
str: A URL-safe base64-encoded secret string.
|
|
64
|
+
"""
|
|
65
|
+
return secrets.token_urlsafe(length)
|
|
56
66
|
|
|
57
|
-
def generate_secret():
|
|
58
|
-
return "".join(RANDOM_CHARS[random.randint(0,RANDOM_CHARS_MAX_INDEX)] for i in range(0,32))
|
|
59
67
|
|
|
60
68
|
secret = None
|
|
61
69
|
|
|
70
|
+
|
|
62
71
|
def get_workloadname(index):
|
|
72
|
+
"""Return the canonical workload name for a given numeric index.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
index (int): The workload index.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
str: Formatted name, e.g. ``"workload0"``.
|
|
79
|
+
"""
|
|
63
80
|
return "workload{}".format(index)
|
|
64
81
|
|
|
82
|
+
|
|
65
83
|
def get_local_ip():
|
|
84
|
+
"""Attempt to determine the local IP address using a UDP socket trick.
|
|
85
|
+
|
|
86
|
+
Connects to a non-routable address (192.168.1.1) to populate the socket
|
|
87
|
+
routing table and read the bound local address. Falls back to 127.0.0.1.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
str: The detected local IP or ``"127.0.0.1"`` on failure.
|
|
91
|
+
"""
|
|
66
92
|
# Create a UDP socket
|
|
67
93
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
68
94
|
try:
|
|
69
95
|
# Connect to a dummy external IP (doesn't have to be reachable)
|
|
70
|
-
s.connect((
|
|
96
|
+
s.connect(("192.168.1.1", 1))
|
|
71
97
|
ip = s.getsockname()[0]
|
|
72
98
|
except Exception:
|
|
73
99
|
# Fallback to localhost if network is down
|
|
74
|
-
ip =
|
|
100
|
+
ip = "127.0.0.1"
|
|
75
101
|
finally:
|
|
76
102
|
s.close()
|
|
77
103
|
return ip
|
|
78
104
|
|
|
105
|
+
|
|
79
106
|
hostname = socket.gethostname()
|
|
80
107
|
if WORKLOAD_DEPLOYMENT:
|
|
81
108
|
registerhostname = hostname
|
|
@@ -89,28 +116,32 @@ item_version = "__version__"
|
|
|
89
116
|
key_workloads = "{}__workloads__".format(CACHE_PREFIX)
|
|
90
117
|
key_workloads_lock = "{}lock__".format(key_workloads)
|
|
91
118
|
|
|
92
|
-
|
|
119
|
+
|
|
120
|
+
def register_webappserver(*args, **kwargs):
|
|
93
121
|
"""
|
|
94
|
-
Register
|
|
95
|
-
|
|
96
|
-
|
|
122
|
+
Register this web server instance with the shared workload cache.
|
|
123
|
+
|
|
124
|
+
Registers the workload in Django cache shared across all workloads and
|
|
125
|
+
generates a bearer token for inter-workload authentication. Disconnects
|
|
126
|
+
itself from the ``request_started`` signal after successful registration
|
|
127
|
+
so it only runs once.
|
|
97
128
|
"""
|
|
98
129
|
pid = os.getpid()
|
|
99
130
|
global secret
|
|
100
|
-
logger.debug("Register the webapp server '{}({}).{}'.".format(hostname,ip,pid))
|
|
131
|
+
logger.debug("Register the webapp server '{}({}).{}'.".format(hostname, ip, pid))
|
|
101
132
|
try:
|
|
102
133
|
workloads_changed = False
|
|
103
|
-
workloads = cache.get(key_workloads) or {item_version:0}
|
|
134
|
+
workloads = cache.get(key_workloads) or {item_version: 0}
|
|
104
135
|
if registerhostname not in workloads:
|
|
105
|
-
#not registered by other webservers running in the same workload
|
|
136
|
+
# not registered by other webservers running in the same workload
|
|
106
137
|
secret = generate_secret()
|
|
107
|
-
workloads[registerhostname] = [[ip,PORT],secret,0]
|
|
138
|
+
workloads[registerhostname] = [[ip, PORT], secret, 0]
|
|
108
139
|
workloads_changed = True
|
|
109
140
|
else:
|
|
110
|
-
#already registered by other webservers, check whether the data is correct
|
|
141
|
+
# already registered by other webservers, check whether the data is correct
|
|
111
142
|
data = workloads[registerhostname]
|
|
112
|
-
if not isinstance(data[0],list):
|
|
113
|
-
data[0] = [ip,PORT]
|
|
143
|
+
if not isinstance(data[0], list):
|
|
144
|
+
data[0] = [ip, PORT]
|
|
114
145
|
workloads_changed = True
|
|
115
146
|
if data[0][0] != ip:
|
|
116
147
|
data[0][0] = ip
|
|
@@ -122,51 +153,58 @@ def register_webappserver(*args,**kwargs):
|
|
|
122
153
|
data[2] = 0
|
|
123
154
|
workloads_changed = True
|
|
124
155
|
if workloads_changed:
|
|
125
|
-
#workload data is changed.
|
|
156
|
+
# workload data is changed.
|
|
126
157
|
secret = generate_secret()
|
|
127
158
|
data[1] = secret
|
|
128
159
|
else:
|
|
129
|
-
#workload data is not changed.
|
|
160
|
+
# workload data is not changed.
|
|
130
161
|
secret = data[1]
|
|
131
162
|
|
|
132
163
|
if workloads_changed:
|
|
133
|
-
#save thw workloads data to cache
|
|
164
|
+
# save thw workloads data to cache
|
|
134
165
|
save_workloads(workloads)
|
|
135
|
-
|
|
136
|
-
except Exception as ex:
|
|
137
|
-
logger.error("Failed to register the webapp webserver '{}({}).{}'. {}: {}".format(hostname,ip,pid,ex.__class__.__name__,str(ex)))
|
|
138
|
-
#Failed to register workload, remove the server register file
|
|
139
|
-
try:
|
|
140
|
-
os.remove(registerfile)
|
|
141
|
-
except Excepton as ex:
|
|
142
|
-
if os.path.exists(registerfile):
|
|
143
|
-
logger.error("Failed to remove webapp webserver register file '{}'.{}: {}".format(registerfile,ex.__class__.__name__,str(ex)))
|
|
144
166
|
|
|
145
|
-
|
|
167
|
+
except Exception as ex:
|
|
168
|
+
logger.error(
|
|
169
|
+
"Failed to register the webapp webserver '{}({}).{}'. {}: {}".format(hostname, ip, pid, ex.__class__.__name__, str(ex))
|
|
170
|
+
)
|
|
171
|
+
# Ignore the exception
|
|
146
172
|
return
|
|
147
173
|
|
|
148
|
-
#register successfully, no need to register again.
|
|
149
|
-
#disconnect the receiver, no need to register again.
|
|
174
|
+
# register successfully, no need to register again.
|
|
175
|
+
# disconnect the receiver, no need to register again.
|
|
150
176
|
request_started.disconnect(dispatch_uid="register_webappserver")
|
|
151
|
-
logger.debug("Successfully register the webserver({}<{}>:{}.{}) to the cache.".format(hostname,ip,PORT,pid))
|
|
177
|
+
logger.debug("Successfully register the webserver({}<{}>:{}.{}) to the cache.".format(hostname, ip, PORT, pid))
|
|
152
178
|
|
|
153
179
|
|
|
154
|
-
#register the signal receiver to register the workload
|
|
155
|
-
#the signal receiver will be disconnected after successful registration
|
|
180
|
+
# register the signal receiver to register the workload
|
|
181
|
+
# the signal receiver will be disconnected after successful registration
|
|
156
182
|
if HEALTHCHECK_ENABLED:
|
|
157
|
-
#healthcheck is not initied
|
|
158
|
-
request_started.connect(register_webappserver,dispatch_uid="register_webappserver")
|
|
183
|
+
# healthcheck is not initied
|
|
184
|
+
request_started.connect(register_webappserver, dispatch_uid="register_webappserver")
|
|
159
185
|
|
|
160
186
|
|
|
161
187
|
VALID_WORKLOAD_VOLUMES = None
|
|
188
|
+
|
|
189
|
+
|
|
162
190
|
def get_volumes_healthdata():
|
|
191
|
+
"""Collect disk-usage metrics for configured workload volumes.
|
|
192
|
+
|
|
193
|
+
When WORKLOAD_VOLUMES is ``"automatic"``, only CIFS/NFS/SSHFS/DavFS2
|
|
194
|
+
mounts are inspected. Otherwise the explicitly listed mount points are
|
|
195
|
+
used.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
dict: Mapping of mountpoint -> usage stats (size, used, pcent, unit).
|
|
199
|
+
dict: With an ``"error"`` key if psutil raises an exception.
|
|
200
|
+
"""
|
|
163
201
|
global VALID_WORKLOAD_VOLUMES
|
|
164
202
|
try:
|
|
165
203
|
if VALID_WORKLOAD_VOLUMES is None:
|
|
166
204
|
volumes = []
|
|
167
205
|
for partition in psutil.disk_partitions(all=True):
|
|
168
206
|
if WORKLOAD_VOLUMES is None:
|
|
169
|
-
if partition.fstype.lower() not in ("cifs","nfs","sshfs","davfs2"):
|
|
207
|
+
if partition.fstype.lower() not in ("cifs", "nfs", "sshfs", "davfs2"):
|
|
170
208
|
continue
|
|
171
209
|
volumes.append(partition.mountpoint)
|
|
172
210
|
elif partition.mountpoint in WORKLOAD_VOLUMES:
|
|
@@ -179,46 +217,79 @@ def get_volumes_healthdata():
|
|
|
179
217
|
|
|
180
218
|
volumesdata = {}
|
|
181
219
|
for volume in VALID_WORKLOAD_VOLUMES:
|
|
182
|
-
diskusage
|
|
183
|
-
if diskusage.total /
|
|
184
|
-
#large than 10G, use 'G' as unit
|
|
185
|
-
volumesdata[volume] = {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
220
|
+
diskusage = psutil.disk_usage(volume)
|
|
221
|
+
if diskusage.total / GB >= 10:
|
|
222
|
+
# large than 10G, use 'G' as unit
|
|
223
|
+
volumesdata[volume] = {
|
|
224
|
+
"size": round(diskusage.total / GB),
|
|
225
|
+
"used": round(diskusage.used / GB),
|
|
226
|
+
"pcent": 100 * diskusage.used / diskusage.total,
|
|
227
|
+
"unit": "G",
|
|
228
|
+
}
|
|
229
|
+
elif diskusage.total / MB >= 10:
|
|
230
|
+
# large than 10M, use 'M' as unit
|
|
231
|
+
volumesdata[volume] = {
|
|
232
|
+
"size": round(diskusage.total / MB),
|
|
233
|
+
"used": round(diskusage.used / MB),
|
|
234
|
+
"pcent": 100 * diskusage.used / diskusage.total,
|
|
235
|
+
"unit": "M",
|
|
236
|
+
}
|
|
189
237
|
else:
|
|
190
|
-
volumesdata[volume] = {
|
|
238
|
+
volumesdata[volume] = {
|
|
239
|
+
"size": round(diskusage.total / KB),
|
|
240
|
+
"used": round(diskusage.used / KB),
|
|
241
|
+
"pcent": 100 * diskusage.used / diskusage.total,
|
|
242
|
+
"unit": "K",
|
|
243
|
+
}
|
|
191
244
|
|
|
192
245
|
return volumesdata
|
|
193
246
|
except Exception as ex:
|
|
194
|
-
import traceback
|
|
247
|
+
import traceback
|
|
248
|
+
|
|
195
249
|
traceback.print_exc()
|
|
196
|
-
return "Failed to volumes usage data.{}: {}".format(ex.__class__.__name__,str(ex))
|
|
250
|
+
return {"error": "Failed to get volumes usage data.{}: {}".format(ex.__class__.__name__, str(ex))}
|
|
251
|
+
|
|
197
252
|
|
|
198
253
|
def get_workload_system_healthdata():
|
|
199
|
-
|
|
200
|
-
|
|
254
|
+
"""Gather system-wide resource metrics.
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
dict: Contains cpu_pcent, cpucores_pcent, memory_total, memory_used,
|
|
258
|
+
memory_pcent, bytes_sent and bytes_recv.
|
|
259
|
+
"""
|
|
260
|
+
cpu_pcent = psutil.cpu_percent(percpu=False)
|
|
261
|
+
cpucores_pcent = psutil.cpu_percent(percpu=True)
|
|
201
262
|
memoryinfo = psutil.virtual_memory()
|
|
202
263
|
netio = psutil.net_io_counters()
|
|
203
264
|
|
|
204
265
|
return {
|
|
205
|
-
"cpu_pcent":cpu_pcent,
|
|
206
|
-
"cpucores_pcent":cpucores_pcent,
|
|
207
|
-
"memory_total": memoryinfo.total /
|
|
208
|
-
"memory_used": (memoryinfo.total - memoryinfo.available) /
|
|
266
|
+
"cpu_pcent": cpu_pcent,
|
|
267
|
+
"cpucores_pcent": cpucores_pcent,
|
|
268
|
+
"memory_total": memoryinfo.total / GB,
|
|
269
|
+
"memory_used": (memoryinfo.total - memoryinfo.available) / GB,
|
|
209
270
|
"memory_pcent": (memoryinfo.total - memoryinfo.available) * 100 / memoryinfo.total,
|
|
210
271
|
"bytes_sent": netio.bytes_sent,
|
|
211
|
-
"bytes_recv": netio.bytes_recv
|
|
272
|
+
"bytes_recv": netio.bytes_recv,
|
|
212
273
|
}
|
|
213
274
|
|
|
275
|
+
|
|
214
276
|
def get_process_healthdata(proc):
|
|
277
|
+
"""Build a health-data dictionary for a single psutil.Process.
|
|
278
|
+
|
|
279
|
+
Args:
|
|
280
|
+
proc (psutil.Process): The process to inspect.
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
dict: start_time, cpu_num, cpu_pcent, pmemory, vmemory.
|
|
284
|
+
In DEBUG mode also includes cmdline and a ``currentprocess`` flag.
|
|
285
|
+
"""
|
|
215
286
|
memoryinfo = proc.memory_info()
|
|
216
287
|
result = {
|
|
217
|
-
"start_time":timezone.make_aware(datetime.fromtimestamp(proc.create_time())).strftime("%Y-%m-%dT%H:%M:%S"),
|
|
288
|
+
"start_time": timezone.make_aware(datetime.fromtimestamp(proc.create_time())).strftime("%Y-%m-%dT%H:%M:%S"),
|
|
218
289
|
"cpu_num": proc.cpu_num(),
|
|
219
290
|
"cpu_pcent": proc.cpu_percent(),
|
|
220
|
-
"pmemory":memoryinfo.rss /
|
|
221
|
-
"vmemory":memoryinfo.vms /
|
|
291
|
+
"pmemory": memoryinfo.rss / MB,
|
|
292
|
+
"vmemory": memoryinfo.vms / MB,
|
|
222
293
|
}
|
|
223
294
|
if settings.DEBUG:
|
|
224
295
|
result["cmdline"] = proc.cmdline()
|
|
@@ -227,13 +298,25 @@ def get_process_healthdata(proc):
|
|
|
227
298
|
|
|
228
299
|
return result
|
|
229
300
|
|
|
301
|
+
|
|
230
302
|
rootproc = None
|
|
231
303
|
curprocpid = None
|
|
304
|
+
|
|
305
|
+
|
|
232
306
|
def get_workload_app_healthdata(perprocess=True):
|
|
233
307
|
"""
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
308
|
+
Aggregate health data for all processes belonging to the webapp.
|
|
309
|
+
|
|
310
|
+
Traverses the process tree upward from the current PID to find the
|
|
311
|
+
root application process, then walks all descendants collecting
|
|
312
|
+
cpu/memory statistics.
|
|
313
|
+
|
|
314
|
+
Args:
|
|
315
|
+
perprocess (bool): If True, includes a nested ``children`` tree of per-process data.
|
|
316
|
+
|
|
317
|
+
Returns:
|
|
318
|
+
dict: Aggregated totals (cpu_total, pmemory_total, etc.) plus
|
|
319
|
+
per-process details when requested.
|
|
237
320
|
"""
|
|
238
321
|
global rootproc
|
|
239
322
|
global curprocpid
|
|
@@ -241,10 +324,10 @@ def get_workload_app_healthdata(perprocess=True):
|
|
|
241
324
|
curprocpid = os.getpid()
|
|
242
325
|
|
|
243
326
|
if not rootproc:
|
|
244
|
-
#the the root proc
|
|
245
|
-
#get the pid of the current process
|
|
327
|
+
# the the root proc
|
|
328
|
+
# get the pid of the current process
|
|
246
329
|
curproc = psutil.Process(curprocpid)
|
|
247
|
-
#find the parent
|
|
330
|
+
# find the parent
|
|
248
331
|
pproc = curproc
|
|
249
332
|
rootproc = None
|
|
250
333
|
app_cmdline = curproc.cmdline()
|
|
@@ -253,41 +336,40 @@ def get_workload_app_healthdata(perprocess=True):
|
|
|
253
336
|
if not ppid:
|
|
254
337
|
rootproc = pproc
|
|
255
338
|
continue
|
|
256
|
-
|
|
339
|
+
|
|
257
340
|
tmpproc = psutil.Process(ppid)
|
|
258
341
|
tmpproc_cmdline = tmpproc.cmdline()
|
|
259
342
|
if tmpproc_cmdline == app_cmdline:
|
|
260
|
-
#the pproc has the same cmd line as current proc. the pproc is also related app python process
|
|
343
|
+
# the pproc has the same cmd line as current proc. the pproc is also related app python process
|
|
261
344
|
pproc = tmpproc
|
|
262
|
-
elif any(any(key in p for key in ("python","gunicorn","uwsgi","django")) for p in tmpproc_cmdline):
|
|
263
|
-
#the pproc is still the python process.
|
|
345
|
+
elif any(any(key in p for key in ("python", "gunicorn", "uwsgi", "django")) for p in tmpproc_cmdline):
|
|
346
|
+
# the pproc is still the python process.
|
|
264
347
|
pproc = tmpproc
|
|
265
348
|
else:
|
|
266
349
|
rootproc = pproc
|
|
267
350
|
|
|
268
|
-
|
|
269
|
-
#find all realted processes and its health data
|
|
351
|
+
# find all realted processes and its health data
|
|
270
352
|
rootproc_data = get_process_healthdata(rootproc)
|
|
271
353
|
result = {
|
|
272
354
|
"start_time": rootproc_data["start_time"],
|
|
273
|
-
"cpu_total"
|
|
274
|
-
"cpu_min"
|
|
275
|
-
"cpu_max"
|
|
276
|
-
"pmemory_total"
|
|
277
|
-
"pmemory_min"
|
|
278
|
-
"pmemory_max"
|
|
279
|
-
"vmemory_total"
|
|
280
|
-
"vmemory_min"
|
|
281
|
-
"vmemory_max"
|
|
282
|
-
"processes"
|
|
355
|
+
"cpu_total": rootproc_data["cpu_pcent"],
|
|
356
|
+
"cpu_min": rootproc_data["cpu_pcent"],
|
|
357
|
+
"cpu_max": rootproc_data["cpu_pcent"],
|
|
358
|
+
"pmemory_total": rootproc_data["pmemory"],
|
|
359
|
+
"pmemory_min": rootproc_data["pmemory"],
|
|
360
|
+
"pmemory_max": rootproc_data["pmemory"],
|
|
361
|
+
"vmemory_total": rootproc_data["vmemory"],
|
|
362
|
+
"vmemory_min": rootproc_data["vmemory"],
|
|
363
|
+
"vmemory_max": rootproc_data["vmemory"],
|
|
364
|
+
"processes": 1,
|
|
283
365
|
}
|
|
284
366
|
if perprocess:
|
|
285
367
|
result["process"] = rootproc_data
|
|
286
368
|
result["process"]["children"] = []
|
|
287
369
|
|
|
288
|
-
processes = [(rootproc.children(),result["process"]["children"] if perprocess else None)]
|
|
370
|
+
processes = [(rootproc.children(), result["process"]["children"] if perprocess else None)]
|
|
289
371
|
while processes:
|
|
290
|
-
childproces,childrendatas = processes.pop(0)
|
|
372
|
+
childproces, childrendatas = processes.pop(0)
|
|
291
373
|
for childproc in childproces:
|
|
292
374
|
childproc_data = get_process_healthdata(childproc)
|
|
293
375
|
|
|
@@ -318,98 +400,142 @@ def get_workload_app_healthdata(perprocess=True):
|
|
|
318
400
|
if childproc_children:
|
|
319
401
|
if perprocess:
|
|
320
402
|
childproc_data["children"] = []
|
|
321
|
-
processes.append((childproc_children,childproc_data["children"] if perprocess else None))
|
|
403
|
+
processes.append((childproc_children, childproc_data["children"] if perprocess else None))
|
|
322
404
|
|
|
323
405
|
return result
|
|
324
406
|
|
|
407
|
+
|
|
325
408
|
def get_workload_healthdata():
|
|
409
|
+
"""Compose the full workload health payload.
|
|
410
|
+
|
|
411
|
+
Combines application process data, optional system data, and optional
|
|
412
|
+
volume data into a single response.
|
|
413
|
+
|
|
414
|
+
Returns:
|
|
415
|
+
tuple: (status_code, result_dict) on success; (500, error_message_string) on failure.
|
|
416
|
+
"""
|
|
326
417
|
try:
|
|
327
|
-
result = {
|
|
328
|
-
"resources": get_workload_app_healthdata(HEALTHCHECK_PROCESSDATA_ENABLED)
|
|
329
|
-
}
|
|
418
|
+
result = {"resources": get_workload_app_healthdata(HEALTHCHECK_PROCESSDATA_ENABLED)}
|
|
330
419
|
if HEALTHCHECK_SYSTEMDATA_ENABLED:
|
|
331
420
|
result["system"] = get_workload_system_healthdata()
|
|
332
421
|
|
|
333
422
|
if WORKLOAD_VOLUMES_ENABLED:
|
|
334
423
|
result["volumes"] = get_volumes_healthdata()
|
|
335
424
|
|
|
336
|
-
return (200,result)
|
|
425
|
+
return (200, result)
|
|
337
426
|
except Exception as ex:
|
|
338
|
-
return (500,"{}:{}".format(ex.
|
|
427
|
+
return (500, "{}:{}".format(ex.__class__.__name__, str(ex)))
|
|
428
|
+
|
|
339
429
|
|
|
340
430
|
bearer_token_re = re.compile("^Bearer\\s+(?P<token>\\S+)\\s*$")
|
|
431
|
+
|
|
432
|
+
|
|
341
433
|
def get_auth_bearer(request):
|
|
434
|
+
"""Extract the bearer token from the Authorization header.
|
|
435
|
+
|
|
436
|
+
Args:
|
|
437
|
+
request (django.http.HttpRequest): The incoming request.
|
|
438
|
+
|
|
439
|
+
Returns:
|
|
440
|
+
str or None: The token if present and syntactically valid, otherwise ``None``.
|
|
342
441
|
"""
|
|
343
|
-
|
|
344
|
-
Return True if authenticated; otherwiser return False
|
|
345
|
-
"""
|
|
346
|
-
bearer_auth = request.META.get('HTTP_AUTHORIZATION').strip() if 'HTTP_AUTHORIZATION' in request.META else ''
|
|
442
|
+
bearer_auth = request.META.get("HTTP_AUTHORIZATION", "").strip()
|
|
347
443
|
m = bearer_token_re.search(bearer_auth)
|
|
348
444
|
token = None
|
|
349
445
|
if m:
|
|
350
|
-
token = m.group(
|
|
446
|
+
token = m.group("token")
|
|
351
447
|
return token
|
|
352
448
|
|
|
449
|
+
|
|
353
450
|
key_assignedworkloads = "{}__assignedworkloads__".format(CACHE_PREFIX)
|
|
354
451
|
key_assignedworkloads_lock = "{}lock__".format(key_assignedworkloads)
|
|
355
452
|
|
|
453
|
+
|
|
356
454
|
def str_workloads(workloads):
|
|
357
|
-
|
|
455
|
+
"""Serialize a workloads mapping to a human-readable string.
|
|
358
456
|
|
|
457
|
+
Args:
|
|
458
|
+
workloads (dict): The workloads cache payload.
|
|
359
459
|
|
|
360
|
-
|
|
460
|
+
Returns:
|
|
461
|
+
str: Comma-separated ``host=ip:port(failed_count)`` pairs.
|
|
361
462
|
"""
|
|
362
|
-
|
|
463
|
+
return ",".join(
|
|
464
|
+
[
|
|
465
|
+
"{}={}:{}({})".format(host, data[0][0], data[0][1], data[2]) if host != item_version else "{}={}".format(host, data)
|
|
466
|
+
for host, data in workloads.items()
|
|
467
|
+
]
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def save_workloads(workloads, unreached_servers=None):
|
|
472
|
+
"""Persist the updated workloads mapping to cache with optimistic locking.
|
|
473
|
+
|
|
474
|
+
Uses a spin-lock (``key_workloads_lock``) to avoid concurrent writes.
|
|
475
|
+
Merges in any newly discovered workloads from the current cache before saving.
|
|
476
|
+
|
|
477
|
+
Args:
|
|
478
|
+
workloads (dict): The workloads data to store.
|
|
479
|
+
unreached_servers (list, optional): Server names to exclude from the merge.
|
|
363
480
|
"""
|
|
364
|
-
#save the workloads
|
|
481
|
+
# save the workloads
|
|
365
482
|
logger.debug("Begin to save the changed workloads data({}) to cache.".format(str_workloads(workloads)))
|
|
366
|
-
|
|
483
|
+
max_retries = 1000
|
|
484
|
+
for _ in range(max_retries):
|
|
367
485
|
if cache.add(key_workloads_lock, 1, timeout=1):
|
|
368
|
-
#get the lock
|
|
486
|
+
# get the lock
|
|
369
487
|
try:
|
|
370
488
|
cur_workloads = cache.get(key_workloads)
|
|
371
|
-
if cur_workloads and cur_workloads.get(item_version,0) != workloads[item_version]:
|
|
372
|
-
#workloads data was changed after fetching the workloads data
|
|
373
|
-
#add the new added workloads data
|
|
374
|
-
for k,v in cur_workloads.items():
|
|
489
|
+
if cur_workloads and cur_workloads.get(item_version, 0) != workloads[item_version]:
|
|
490
|
+
# workloads data was changed after fetching the workloads data
|
|
491
|
+
# add the new added workloads data
|
|
492
|
+
for k, v in cur_workloads.items():
|
|
375
493
|
if k == item_version:
|
|
376
494
|
continue
|
|
377
495
|
if k not in workloads and (not unreached_servers or k not in unreached_servers):
|
|
378
496
|
workloads[k] = v
|
|
379
|
-
if cur_workloads.get(item_version,0) == 0:
|
|
497
|
+
if cur_workloads.get(item_version, 0) == 0:
|
|
380
498
|
workloads[item_version] += 1
|
|
381
499
|
else:
|
|
382
500
|
workloads[item_version] = cur_workloads[item_version] + 1
|
|
383
501
|
else:
|
|
384
|
-
#workloads data is not changed.
|
|
502
|
+
# workloads data is not changed.
|
|
385
503
|
workloads[item_version] += 1
|
|
386
504
|
|
|
387
|
-
#save the new workloads data
|
|
388
|
-
cache.set(key_workloads,workloads,timeout=None)
|
|
505
|
+
# save the new workloads data
|
|
506
|
+
cache.set(key_workloads, workloads, timeout=None)
|
|
389
507
|
logger.debug("Successfully save the workloads:{}".format(str_workloads(workloads)))
|
|
390
508
|
return
|
|
391
509
|
finally:
|
|
392
|
-
#release the lock
|
|
510
|
+
# release the lock
|
|
393
511
|
cache.delete(key_workloads_lock)
|
|
394
512
|
else:
|
|
395
|
-
#already locked
|
|
513
|
+
# already locked, wait 100 milliseconds and try again
|
|
396
514
|
time.sleep(0.01)
|
|
397
|
-
|
|
515
|
+
|
|
516
|
+
raise Exception("Failed to acquire cache lock for key_workloads after {} retries".format(max_retries))
|
|
517
|
+
|
|
398
518
|
|
|
399
519
|
def save_assignedworkloads(assignedworkloads):
|
|
520
|
+
"""Persist the assigned-workloads mapping to cache with optimistic locking.
|
|
521
|
+
|
|
522
|
+
Uses a spin-lock (``key_assignedworkloads_lock``). Merges remote
|
|
523
|
+
changes before writing and bumps the version counter.
|
|
524
|
+
|
|
525
|
+
Args:
|
|
526
|
+
assignedworkloads (dict): The assigned-workloads data to store.
|
|
400
527
|
"""
|
|
401
|
-
|
|
402
|
-
"""
|
|
403
|
-
#save the workloads
|
|
528
|
+
# save the workloads
|
|
404
529
|
logger.debug("Begin to save the changed assigned workloads data({}) to cache.".format(assignedworkloads))
|
|
405
|
-
|
|
530
|
+
max_retries = 1000
|
|
531
|
+
for _ in range(max_retries):
|
|
406
532
|
if cache.add(key_assignedworkloads_lock, 1, timeout=1):
|
|
407
|
-
#get the lock
|
|
533
|
+
# get the lock
|
|
408
534
|
try:
|
|
409
535
|
cur_assignedworkloads = cache.get(key_assignedworkloads)
|
|
410
|
-
if cur_assignedworkloads and cur_assignedworkloads.get(item_version,0) != assignedworkloads[item_version]:
|
|
411
|
-
#sync the latest cache data
|
|
412
|
-
for k,v in cur_assignedworkloads.items():
|
|
536
|
+
if cur_assignedworkloads and cur_assignedworkloads.get(item_version, 0) != assignedworkloads[item_version]:
|
|
537
|
+
# sync the latest cache data
|
|
538
|
+
for k, v in cur_assignedworkloads.items():
|
|
413
539
|
if k == item_version:
|
|
414
540
|
continue
|
|
415
541
|
if k not in assignedworkloads:
|
|
@@ -417,55 +543,63 @@ def save_assignedworkloads(assignedworkloads):
|
|
|
417
543
|
elif v != assignedworkloads[k]:
|
|
418
544
|
assignedworkloads[k] = v
|
|
419
545
|
|
|
420
|
-
if cur_assignedworkloads.get(item_version,0) == 0:
|
|
546
|
+
if cur_assignedworkloads.get(item_version, 0) == 0:
|
|
421
547
|
assignedworkloads[item_version] += 1
|
|
422
548
|
else:
|
|
423
549
|
assignedworkloads[item_version] = cur_assignedworkloads[item_version] + 1
|
|
424
550
|
else:
|
|
425
|
-
#workloads data is not changed.
|
|
551
|
+
# workloads data is not changed.
|
|
426
552
|
assignedworkloads[item_version] += 1
|
|
427
553
|
|
|
428
|
-
#save the new workloads data
|
|
429
|
-
cache.set(key_assignedworkloads,assignedworkloads,timeout=None)
|
|
554
|
+
# save the new workloads data
|
|
555
|
+
cache.set(key_assignedworkloads, assignedworkloads, timeout=None)
|
|
430
556
|
logger.debug("Successfully save the assigned workloads:{}".format(assignedworkloads))
|
|
431
557
|
return
|
|
432
558
|
finally:
|
|
433
|
-
#release the lock
|
|
559
|
+
# release the lock
|
|
434
560
|
cache.delete(key_assignedworkloads_lock)
|
|
435
561
|
else:
|
|
436
|
-
#already locked
|
|
562
|
+
# already locked, wait 100 milliseconds and try again
|
|
437
563
|
time.sleep(0.01)
|
|
438
|
-
|
|
564
|
+
|
|
565
|
+
raise Exception("Failed to acquire cache lock for key_assignedworkloads after {} retries".format(max_retries))
|
|
566
|
+
|
|
439
567
|
|
|
440
568
|
def populate_summary_data(datas):
|
|
441
|
-
"""
|
|
442
|
-
|
|
569
|
+
"""Add a ``summary`` key to *datas* with rolled-up statistics.
|
|
570
|
+
|
|
571
|
+
Computes cluster-wide min/max/total for CPU, physical memory and
|
|
572
|
+
virtual memory across all workloads. Also counts running and failed
|
|
573
|
+
workloads.
|
|
574
|
+
|
|
575
|
+
Args:
|
|
576
|
+
datas (dict): Mapping of workload name -> health payload. Modified in-place.
|
|
443
577
|
"""
|
|
444
578
|
summary = {
|
|
445
|
-
"cpu_total":0,
|
|
446
|
-
"cpu_min":None,
|
|
447
|
-
"cpu_max":None,
|
|
448
|
-
"process_cpu_min":None,
|
|
449
|
-
"process_cpu_max":None,
|
|
450
|
-
"pmemory_total":0,
|
|
451
|
-
"pmemory_min":None,
|
|
452
|
-
"pmemory_max":None,
|
|
453
|
-
"process_pmemory_min":None,
|
|
454
|
-
"process_pmemory_max":None,
|
|
455
|
-
"vmemory_total":0,
|
|
456
|
-
"vmemory_min":None,
|
|
457
|
-
"vmemory_max":None,
|
|
458
|
-
"process_vmemory_min":None,
|
|
459
|
-
"process_vmemory_max":None,
|
|
460
|
-
"processes_total":0,
|
|
461
|
-
"workloads_running":0,
|
|
462
|
-
"workloads_failed":0,
|
|
579
|
+
"cpu_total": 0,
|
|
580
|
+
"cpu_min": None,
|
|
581
|
+
"cpu_max": None,
|
|
582
|
+
"process_cpu_min": None,
|
|
583
|
+
"process_cpu_max": None,
|
|
584
|
+
"pmemory_total": 0,
|
|
585
|
+
"pmemory_min": None,
|
|
586
|
+
"pmemory_max": None,
|
|
587
|
+
"process_pmemory_min": None,
|
|
588
|
+
"process_pmemory_max": None,
|
|
589
|
+
"vmemory_total": 0,
|
|
590
|
+
"vmemory_min": None,
|
|
591
|
+
"vmemory_max": None,
|
|
592
|
+
"process_vmemory_min": None,
|
|
593
|
+
"process_vmemory_max": None,
|
|
594
|
+
"processes_total": 0,
|
|
595
|
+
"workloads_running": 0,
|
|
596
|
+
"workloads_failed": 0,
|
|
463
597
|
}
|
|
464
598
|
if settings.DEBUG:
|
|
465
599
|
summary["currentworkload"] = registerhostname
|
|
466
600
|
|
|
467
|
-
for
|
|
468
|
-
if isinstance(serverdata,str):
|
|
601
|
+
for _, serverdata in datas.items():
|
|
602
|
+
if isinstance(serverdata, str):
|
|
469
603
|
summary["workloads_failed"] += 1
|
|
470
604
|
continue
|
|
471
605
|
summary["processes_total"] += serverdata["resources"]["processes"]
|
|
@@ -506,28 +640,41 @@ def populate_summary_data(datas):
|
|
|
506
640
|
|
|
507
641
|
datas["summary"] = summary
|
|
508
642
|
|
|
643
|
+
|
|
509
644
|
workload_healthcheck_url = None
|
|
510
|
-
headers={"Authorization":None,"Accept": "application/json"}
|
|
645
|
+
headers = {"Authorization": None, "Accept": "application/json"}
|
|
646
|
+
|
|
511
647
|
|
|
512
648
|
def harvest_healthdata(request):
|
|
513
|
-
|
|
649
|
+
"""Collect health data from all registered workloads.
|
|
650
|
+
|
|
651
|
+
Iterates the workloads cache, contacts each peer via HTTP for its
|
|
652
|
+
local health data, and removes peers that have exceeded the failure threshold.
|
|
514
653
|
|
|
654
|
+
Args:
|
|
655
|
+
request (django.http.HttpRequest): The incoming request (used to derive the Host header for internal calls).
|
|
656
|
+
|
|
657
|
+
Returns:
|
|
658
|
+
tuple: (workloads_dict, servers_res_dict)
|
|
659
|
+
"""
|
|
660
|
+
global secret
|
|
515
661
|
global workload_healthcheck_url
|
|
662
|
+
|
|
516
663
|
if not workload_healthcheck_url:
|
|
517
|
-
workload_healthcheck_url = reverse(
|
|
664
|
+
workload_healthcheck_url = reverse("healthcheck:workload_healthdata")
|
|
518
665
|
|
|
519
|
-
workloads = cache.get(key_workloads) or {item_version:0}
|
|
666
|
+
workloads = cache.get(key_workloads) or {item_version: 0}
|
|
520
667
|
workloads_changed = False
|
|
521
668
|
logger.debug("Get the workloads from cache :{}".format(str_workloads(workloads)))
|
|
522
669
|
|
|
523
670
|
if registerhostname not in workloads:
|
|
524
671
|
secret = generate_secret()
|
|
525
|
-
workloads[registerhostname] = [[ip,PORT],secret,0]
|
|
672
|
+
workloads[registerhostname] = [[ip, PORT], secret, 0]
|
|
526
673
|
workloads_changed = True
|
|
527
674
|
|
|
528
675
|
servers_res = {}
|
|
529
676
|
unreached_servers = []
|
|
530
|
-
#havest health data from all workloads
|
|
677
|
+
# havest health data from all workloads
|
|
531
678
|
for servername, serverdata in workloads.items():
|
|
532
679
|
if servername == item_version:
|
|
533
680
|
continue
|
|
@@ -535,75 +682,82 @@ def harvest_healthdata(request):
|
|
|
535
682
|
servers_res[servername] = get_workload_healthdata()
|
|
536
683
|
continue
|
|
537
684
|
|
|
538
|
-
serverip,port = serverdata[0]
|
|
685
|
+
serverip, port = serverdata[0]
|
|
539
686
|
headers["Authorization"] = "Bearer {}".format(serverdata[1])
|
|
540
687
|
headers["host"] = request.get_host()
|
|
541
|
-
url = "http://{}:{}{}".format(serverip,port,workload_healthcheck_url)
|
|
688
|
+
url = "http://{}:{}{}".format(serverip, port, workload_healthcheck_url)
|
|
542
689
|
try:
|
|
543
|
-
res = requests.get(url,headers=headers)
|
|
690
|
+
res = requests.get(url, headers=headers, timeout=5)
|
|
544
691
|
except Exception as ex:
|
|
545
|
-
#the server is offline, don't add the data to servers_res
|
|
692
|
+
# the server is offline, don't add the data to servers_res
|
|
546
693
|
workloads_changed = True
|
|
547
694
|
serverdata[2] += 1
|
|
548
695
|
if serverdata[2] >= WORKLOAD_FAILED_THRESHOLD:
|
|
549
|
-
#continuous failed times is greater than WORKLOAD_FAILED_THRESHOLD.
|
|
696
|
+
# continuous failed times is greater than WORKLOAD_FAILED_THRESHOLD.
|
|
550
697
|
unreached_servers.append(servername)
|
|
551
|
-
servers_res[servername] = (-1,"{1}:{2},url={0}".format(url,ex.__class__.__name__,str(ex)))
|
|
698
|
+
servers_res[servername] = (-1, "{1}:{2},url={0}".format(url, ex.__class__.__name__, str(ex)))
|
|
552
699
|
continue
|
|
553
|
-
if res.status_code in (502,503,504):
|
|
554
|
-
#the server is offline, don't add the data to servers_res
|
|
700
|
+
if res.status_code in (502, 503, 504):
|
|
701
|
+
# the server is offline, don't add the data to servers_res
|
|
555
702
|
workloads_changed = True
|
|
556
703
|
serverdata[2] += 1
|
|
557
704
|
if serverdata[2] >= WORKLOAD_FAILED_THRESHOLD:
|
|
558
|
-
#continuous failed times is greater than WORKLOAD_FAILED_THRESHOLD.
|
|
705
|
+
# continuous failed times is greater than WORKLOAD_FAILED_THRESHOLD.
|
|
559
706
|
unreached_servers.append(servername)
|
|
560
|
-
servers_res[servername] = (res.status_code,"{1}:{2},url={0}".format(url,res.status_code,res.text))
|
|
707
|
+
servers_res[servername] = (res.status_code, "{1}:{2},url={0}".format(url, res.status_code, res.text))
|
|
561
708
|
elif res.status_code == 200:
|
|
562
|
-
#the server is in good health, add the health data to servers_res
|
|
563
|
-
servers_res[servername] = (200,res.json())
|
|
709
|
+
# the server is in good health, add the health data to servers_res
|
|
710
|
+
servers_res[servername] = (200, res.json())
|
|
564
711
|
if serverdata[2] > 0:
|
|
565
712
|
serverdata[2] -= 1
|
|
566
713
|
workloads_changed = True
|
|
567
714
|
elif res.status_code == 599:
|
|
568
|
-
#the server is in good health, add the health data to servers_res
|
|
715
|
+
# the server is in good health, add the health data to servers_res
|
|
569
716
|
try:
|
|
570
717
|
data = res.json()
|
|
571
718
|
if data["status"] == 401:
|
|
572
|
-
#authentication error, caused by different workload.
|
|
719
|
+
# authentication error, caused by different workload.
|
|
573
720
|
workloads_changed = True
|
|
574
721
|
unreached_servers.append(servername)
|
|
575
|
-
servers_res[servername] = (503,"{1}:{2},url={0}".format(url,data["status"],data["message"]))
|
|
722
|
+
servers_res[servername] = (503, "{1}:{2},url={0}".format(url, data["status"], data["message"]))
|
|
576
723
|
else:
|
|
577
|
-
servers_res[servername] = (500,"{1}:{2}. url={0}".format(url,data["status"],data["message"]))
|
|
578
|
-
except:
|
|
724
|
+
servers_res[servername] = (500, "{1}:{2}. url={0}".format(url, data["status"], data["message"]))
|
|
725
|
+
except Exception:
|
|
579
726
|
workloads_changed = True
|
|
580
727
|
unreached_servers.append(servername)
|
|
581
|
-
servers_res[servername] = (503,"Web server is offline.url={0}".format(url))
|
|
728
|
+
servers_res[servername] = (503, "Web server is offline.url={0}".format(url))
|
|
582
729
|
|
|
583
730
|
else:
|
|
584
|
-
#unexpected error, caused by different workload
|
|
731
|
+
# unexpected error, caused by different workload
|
|
585
732
|
workloads_changed = True
|
|
586
733
|
unreached_servers.append(servername)
|
|
587
|
-
servers_res[servername] = (500,"{1}:{2},url={0}".format(url,res.status_code,res.text))
|
|
734
|
+
servers_res[servername] = (500, "{1}:{2},url={0}".format(url, res.status_code, res.text))
|
|
588
735
|
|
|
589
736
|
for servername in unreached_servers:
|
|
590
737
|
del workloads[servername]
|
|
591
738
|
|
|
592
|
-
logger.debug("healthdata harvest result :workloads={}, resources={}".format(workloads,servers_res))
|
|
739
|
+
logger.debug("healthdata harvest result :workloads={}, resources={}".format(workloads, servers_res))
|
|
593
740
|
|
|
594
741
|
if workloads_changed:
|
|
595
|
-
save_workloads(workloads,unreached_servers)
|
|
742
|
+
save_workloads(workloads, unreached_servers)
|
|
743
|
+
|
|
744
|
+
return (workloads, servers_res)
|
|
596
745
|
|
|
597
|
-
return (workloads,servers_res)
|
|
598
746
|
|
|
599
|
-
OFFLINE_STATUSCODE_LIST = (502,503,504,401,403
|
|
747
|
+
OFFLINE_STATUSCODE_LIST = (502, 503, 504, 401, 403, -1)
|
|
600
748
|
if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
601
|
-
#has a fixed number of workloads and it is a deployment
|
|
749
|
+
# has a fixed number of workloads and it is a deployment
|
|
602
750
|
WORKLOADNAMES = [get_workloadname(index) for index in range(WORKLOADS)]
|
|
751
|
+
|
|
603
752
|
def healthdata_view(request):
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
753
|
+
"""Django view: Return aggregated health data for the whole cluster.
|
|
754
|
+
|
|
755
|
+
Deployment-with-fixed-workloads variant: assigns named workload slots
|
|
756
|
+
to available servers, handling failover and re-assignment.
|
|
757
|
+
"""
|
|
758
|
+
# process the workloads which are alreasy assigned a workload name
|
|
759
|
+
workloads, servers_res = harvest_healthdata(request)
|
|
760
|
+
assignedworkloads = cache.get(key_assignedworkloads) or {item_version: 1}
|
|
607
761
|
logger.debug("Get assigned workloads:{}".format(assignedworkloads))
|
|
608
762
|
datas = {}
|
|
609
763
|
index = 0
|
|
@@ -611,25 +765,25 @@ if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
|
611
765
|
for workloadname in WORKLOADNAMES:
|
|
612
766
|
servername = assignedworkloads.get(workloadname)
|
|
613
767
|
if not servername:
|
|
614
|
-
#workloadname is not assined to a server
|
|
768
|
+
# workloadname is not assined to a server
|
|
615
769
|
reassign_workloads += 1
|
|
616
770
|
continue
|
|
617
771
|
|
|
618
|
-
#workload name is assigned to a server
|
|
619
|
-
if servername not in servers_res
|
|
620
|
-
#the server is not available
|
|
772
|
+
# workload name is assigned to a server
|
|
773
|
+
if servername not in servers_res:
|
|
774
|
+
# the server is not available
|
|
621
775
|
reassign_workloads += 1
|
|
622
776
|
continue
|
|
623
777
|
|
|
624
778
|
datas[servername] = servers_res[servername]
|
|
625
779
|
if servers_res[servername][0] in OFFLINE_STATUSCODE_LIST:
|
|
626
|
-
#Related workload is offline, need to reassign another workload
|
|
780
|
+
# Related workload is offline, need to reassign another workload
|
|
627
781
|
reassign_workloads += 1
|
|
628
782
|
del servers_res[servername]
|
|
629
783
|
|
|
630
784
|
assignedworkloads_changed = False
|
|
631
785
|
if len(WORKLOADNAMES) != len(assignedworkloads):
|
|
632
|
-
#remove the unexisted workloads from assignedworkloads.
|
|
786
|
+
# remove the unexisted workloads from assignedworkloads.
|
|
633
787
|
for key in [k for k in assignedworkloads.keys()]:
|
|
634
788
|
if key == item_version:
|
|
635
789
|
continue
|
|
@@ -638,35 +792,35 @@ if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
|
638
792
|
del assignedworkloads[key]
|
|
639
793
|
|
|
640
794
|
if reassign_workloads > 0:
|
|
641
|
-
#Some workloads are not assigned a workload name or are not available
|
|
642
|
-
#Using the following to replace the exisint one with new one if possible
|
|
643
|
-
#Step 1: Replace the unavailable server with a new one
|
|
644
|
-
#Step 2: Assign the new server to the missing assignedworkloads(missed in the assignedworkloads before)
|
|
795
|
+
# Some workloads are not assigned a workload name or are not available
|
|
796
|
+
# Using the following to replace the exisint one with new one if possible
|
|
797
|
+
# Step 1: Replace the unavailable server with a new one
|
|
798
|
+
# Step 2: Assign the new server to the missing assignedworkloads(missed in the assignedworkloads before)
|
|
645
799
|
step = 0
|
|
646
800
|
while reassign_workloads > 0:
|
|
647
801
|
step += 1
|
|
648
802
|
for workloadname in WORKLOADNAMES:
|
|
649
803
|
servername = assignedworkloads.get(workloadname)
|
|
650
804
|
if servername in datas and datas[servername][0] not in OFFLINE_STATUSCODE_LIST:
|
|
651
|
-
#related server is online.no need to reassign
|
|
805
|
+
# related server is online.no need to reassign
|
|
652
806
|
continue
|
|
653
807
|
elif step == 1:
|
|
654
|
-
#step 1 only reassign the already assigned workloads
|
|
808
|
+
# step 1 only reassign the already assigned workloads
|
|
655
809
|
if workloadname not in assignedworkloads:
|
|
656
810
|
continue
|
|
657
811
|
replacedservername = None
|
|
658
|
-
for name,res in servers_res.items():
|
|
812
|
+
for name, res in servers_res.items():
|
|
659
813
|
if res[0] == 200:
|
|
660
|
-
#found a good one, choose it
|
|
814
|
+
# found a good one, choose it
|
|
661
815
|
replacedservername = name
|
|
662
816
|
break
|
|
663
817
|
elif res[0] in OFFLINE_STATUSCODE_LIST:
|
|
664
818
|
continue
|
|
665
819
|
elif not replacedservername:
|
|
666
|
-
#fond a available one, but has some issues,choose it if can't find a good one
|
|
820
|
+
# fond a available one, but has some issues,choose it if can't find a good one
|
|
667
821
|
replacedservername = name
|
|
668
822
|
|
|
669
|
-
logger.debug("Replaced {1} with {2} for workload({0})".format(workloadname,servername,replacedservername))
|
|
823
|
+
logger.debug("Replaced {1} with {2} for workload({0})".format(workloadname, servername, replacedservername))
|
|
670
824
|
if replacedservername:
|
|
671
825
|
datas[replacedservername] = servers_res[replacedservername]
|
|
672
826
|
del servers_res[replacedservername]
|
|
@@ -681,22 +835,30 @@ if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
|
681
835
|
break
|
|
682
836
|
|
|
683
837
|
if assignedworkloads_changed:
|
|
684
|
-
#save the workloads
|
|
838
|
+
# save the workloads
|
|
685
839
|
logger.debug("Save the changed running workloads data({}).".format(assignedworkloads))
|
|
686
840
|
save_assignedworkloads(assignedworkloads)
|
|
687
841
|
|
|
688
|
-
#map the healthdata result to workload. and remove status code
|
|
842
|
+
# map the healthdata result to workload. and remove status code
|
|
689
843
|
result = OrderedDict()
|
|
690
844
|
for workloadname in WORKLOADNAMES:
|
|
691
845
|
servername = assignedworkloads.get(workloadname)
|
|
692
846
|
if not servername:
|
|
693
847
|
if settings.DEBUG:
|
|
694
|
-
result[workloadname] =
|
|
848
|
+
result[workloadname] = (
|
|
849
|
+
"Can't find an available host for this non-assigned host.registered workloads: {0}, assigned workloads:{1}".format(
|
|
850
|
+
str_workloads(workloads), assignedworkloads
|
|
851
|
+
)
|
|
852
|
+
)
|
|
695
853
|
else:
|
|
696
854
|
result[workloadname] = "Can't find an available host for this non-assigned host."
|
|
697
855
|
elif servername not in datas:
|
|
698
856
|
if settings.DEBUG:
|
|
699
|
-
result[workloadname] =
|
|
857
|
+
result[workloadname] = (
|
|
858
|
+
"Can't find an available host for this assigned offline host({2}).registered workloads: {0}, assigned workloads:{1}".format(
|
|
859
|
+
str_workloads(workloads), assignedworkloads, servername
|
|
860
|
+
)
|
|
861
|
+
)
|
|
700
862
|
else:
|
|
701
863
|
result[workloadname] = "Can't find an available host for this assigned offline host({0}).".format(servername)
|
|
702
864
|
|
|
@@ -704,8 +866,7 @@ if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
|
704
866
|
result[workloadname] = datas[servername][1]
|
|
705
867
|
result[workloadname]["hostname"] = servername
|
|
706
868
|
else:
|
|
707
|
-
result[workloadname] = "{}: {}".format(servername,datas[servername][1])
|
|
708
|
-
|
|
869
|
+
result[workloadname] = "{}: {}".format(servername, datas[servername][1])
|
|
709
870
|
|
|
710
871
|
datas.clear()
|
|
711
872
|
|
|
@@ -714,13 +875,18 @@ if WORKLOADS > 0 and WORKLOAD_DEPLOYMENT:
|
|
|
714
875
|
return JsonResponse(result)
|
|
715
876
|
|
|
716
877
|
elif WORKLOADS > 0 and not WORKLOAD_DEPLOYMENT:
|
|
717
|
-
WORKLOADNAMES = [get_workloadname(index) for index in range(1,WORKLOADS + 1,1)]
|
|
878
|
+
WORKLOADNAMES = [get_workloadname(index) for index in range(1, WORKLOADS + 1, 1)]
|
|
879
|
+
|
|
718
880
|
def healthdata_view(request):
|
|
719
|
-
|
|
881
|
+
"""Django view: Return aggregated health data for the whole cluster.
|
|
882
|
+
|
|
883
|
+
StatefulSet variant: direct 1:1 mapping of workload names to servers.
|
|
884
|
+
"""
|
|
885
|
+
workloads, servers_res = harvest_healthdata(request)
|
|
720
886
|
|
|
721
887
|
result = OrderedDict()
|
|
722
888
|
for servername in WORKLOADNAMES:
|
|
723
|
-
if
|
|
889
|
+
if servername in servers_res:
|
|
724
890
|
result[servername] = servers_res[servername][1]
|
|
725
891
|
else:
|
|
726
892
|
result[servername] = "Workload is offline.workloads={}".format(str_workloads(workloads))
|
|
@@ -729,8 +895,13 @@ elif WORKLOADS > 0 and not WORKLOAD_DEPLOYMENT:
|
|
|
729
895
|
|
|
730
896
|
return JsonResponse(result)
|
|
731
897
|
elif WORKLOAD_DEPLOYMENT:
|
|
898
|
+
|
|
732
899
|
def healthdata_view(request):
|
|
733
|
-
|
|
900
|
+
"""Django view: Return aggregated health data for the whole cluster.
|
|
901
|
+
|
|
902
|
+
Deployment-without-fixed-count variant: lists only currently available servers.
|
|
903
|
+
"""
|
|
904
|
+
workloads, servers_res = harvest_healthdata(request)
|
|
734
905
|
|
|
735
906
|
result = OrderedDict()
|
|
736
907
|
for servername, serverdata in servers_res.items():
|
|
@@ -742,25 +913,38 @@ elif WORKLOAD_DEPLOYMENT:
|
|
|
742
913
|
|
|
743
914
|
return JsonResponse(result)
|
|
744
915
|
else:
|
|
745
|
-
def healthdata_view(request):
|
|
746
|
-
workloads,servers_res = harvest_healthdata(request)
|
|
747
916
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
917
|
+
def healthdata_view(request):
|
|
918
|
+
"""Django view: Return aggregated health data for the whole cluster.
|
|
919
|
+
|
|
920
|
+
Dynamic-workload variant: returns health data for workloads indexed
|
|
921
|
+
from 0 up to the last currently available workload.
|
|
922
|
+
"""
|
|
923
|
+
workloads, servers_res = harvest_healthdata(request)
|
|
924
|
+
|
|
925
|
+
# get all workload names
|
|
926
|
+
workloadnames = [k for k in workloads.keys() if k != item_version]
|
|
927
|
+
|
|
928
|
+
# find the name of the last available workload
|
|
929
|
+
workloadnames.sort(key=lambda d: int(d[8:]))
|
|
930
|
+
last_workloadname = next(
|
|
931
|
+
(
|
|
932
|
+
name
|
|
933
|
+
for name in reversed(workloadnames)
|
|
934
|
+
if servers_res.get(name, [503, "Workload is Offline"])[0] not in OFFLINE_STATUSCODE_LIST
|
|
935
|
+
),
|
|
936
|
+
None,
|
|
937
|
+
)
|
|
754
938
|
|
|
755
939
|
result = OrderedDict()
|
|
756
940
|
if last_workloadname:
|
|
757
|
-
#find the index of the last available workload
|
|
941
|
+
# find the index of the last available workload
|
|
758
942
|
last_workloadindex = int(last_workloadname[8:])
|
|
759
943
|
|
|
760
|
-
#Return the healthdata of the workloads whose index is from 0 to last_workloadindex(include)
|
|
944
|
+
# Return the healthdata of the workloads whose index is from 0 to last_workloadindex(include)
|
|
761
945
|
for i in range(last_workloadindex + 1):
|
|
762
946
|
servername = get_workloadname(i)
|
|
763
|
-
serverdata = servers_res.get(servername,[503,"Workload is offline"])
|
|
947
|
+
serverdata = servers_res.get(servername, [503, "Workload is offline"])
|
|
764
948
|
result[servername] = serverdata[1]
|
|
765
949
|
|
|
766
950
|
populate_summary_data(result)
|
|
@@ -769,42 +953,59 @@ else:
|
|
|
769
953
|
|
|
770
954
|
|
|
771
955
|
def workload_healthdata_view(request):
|
|
956
|
+
"""Django view: Return local workload health data with bearer auth.
|
|
957
|
+
|
|
958
|
+
Validates the ``Authorization: Bearer <token>`` header against the
|
|
959
|
+
workload's cached secret before returning ``get_workload_healthdata()`` output.
|
|
960
|
+
|
|
961
|
+
Returns:
|
|
962
|
+
JsonResponse: 200 with health payload, or 599 with auth/error details.
|
|
963
|
+
"""
|
|
772
964
|
global secret
|
|
773
965
|
try:
|
|
774
966
|
token = get_auth_bearer(request)
|
|
775
967
|
if not token:
|
|
776
|
-
return JsonResponse({"status":401,"message":"Missing access token."},status=599)
|
|
777
|
-
|
|
968
|
+
return JsonResponse({"status": 401, "message": "Missing access token."}, status=599)
|
|
969
|
+
|
|
778
970
|
if not secret or secret != token:
|
|
779
971
|
workloads = cache.get(key_workloads)
|
|
780
972
|
data = workloads.get(registerhostname)
|
|
781
973
|
if data:
|
|
782
974
|
secret = data[1]
|
|
783
|
-
|
|
975
|
+
|
|
784
976
|
if secret != token:
|
|
785
|
-
return JsonResponse({"status":401,"message":"Access token doesn't match."},status=599)
|
|
786
|
-
|
|
787
|
-
statuscode,data = get_workload_healthdata()
|
|
977
|
+
return JsonResponse({"status": 401, "message": "Access token doesn't match."}, status=599)
|
|
978
|
+
|
|
979
|
+
statuscode, data = get_workload_healthdata()
|
|
788
980
|
if statuscode == 200:
|
|
789
981
|
return JsonResponse(data)
|
|
790
982
|
else:
|
|
791
|
-
return JsonResponse({"status":statuscode,"message":data},status=599)
|
|
983
|
+
return JsonResponse({"status": statuscode, "message": data}, status=599)
|
|
792
984
|
except Exception as ex:
|
|
793
|
-
return JsonResponse({"status":500,"message":"{}:{}".format(ex.__class__.__name__,str(ex))},status=599)
|
|
985
|
+
return JsonResponse({"status": 500, "message": "{}:{}".format(ex.__class__.__name__, str(ex))}, status=599)
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def register_healthcheck_urls():
|
|
989
|
+
"""Inject healthcheck URL patterns into the root URLconf at runtime.
|
|
794
990
|
|
|
795
|
-
|
|
796
|
-
|
|
991
|
+
Dynamically imports ``settings.ROOT_URLCONF`` and appends the
|
|
992
|
+
healthcheck and workload-healthdata routes. Guards against
|
|
993
|
+
duplicate registration if called more than once.
|
|
994
|
+
"""
|
|
995
|
+
# Add urls
|
|
797
996
|
rootconf_module = importlib.import_module(settings.ROOT_URLCONF)
|
|
798
997
|
if not rootconf_module:
|
|
799
998
|
raise Exception("Failed to load module '{}'".format(settings.ROOT_URLCONF))
|
|
800
|
-
|
|
999
|
+
|
|
801
1000
|
if HEALTHCHECK_ENABLED:
|
|
802
1001
|
urlpatterns = [
|
|
803
|
-
|
|
804
|
-
|
|
1002
|
+
path("healthcheck/healthdata", healthdata_view, name="healthdata"),
|
|
1003
|
+
path("workload/healthcheck/healthdata", workload_healthdata_view, name="workload_healthdata"),
|
|
805
1004
|
]
|
|
806
1005
|
else:
|
|
807
1006
|
urlpatterns = []
|
|
808
1007
|
|
|
809
|
-
rootconf_module
|
|
810
|
-
|
|
1008
|
+
if not getattr(rootconf_module, "_healthcheck_urls_registered", False):
|
|
1009
|
+
# Guard against re-registration:
|
|
1010
|
+
rootconf_module.urlpatterns.append(path("", include((urlpatterns, "healthcheck"), namespace="healthcheck")))
|
|
1011
|
+
rootconf_module._healthcheck_urls_registered = True
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
from django.urls import path
|
|
2
|
-
from django.conf import settings
|
|
3
|
-
|
|
4
|
-
from . import healthcheck
|
|
5
|
-
|
|
6
|
-
app_name = 'healthcheck'
|
|
7
|
-
|
|
8
|
-
if healthcheck.HEALTHCHECK_ENABLED:
|
|
9
|
-
urlpatterns = [
|
|
10
|
-
path('healthcheck/healthdata', healthcheck.healthdata_view,name="healthdata"),
|
|
11
|
-
path('healthcheck/workload_healthdata', healthcheck.workload_healthdata_view,name="workload_healthdata")
|
|
12
|
-
]
|
|
13
|
-
else:
|
|
14
|
-
urlpatterns = []
|
|
15
|
-
|
|
16
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|