platform-plugin-aspects 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- platform_plugin_aspects/__init__.py +11 -0
- platform_plugin_aspects/apps.py +77 -0
- platform_plugin_aspects/conf/locale/babel_mako.cfg +10 -0
- platform_plugin_aspects/conf/locale/config.yaml +85 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +99 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +70 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/extensions/__init__.py +0 -0
- platform_plugin_aspects/extensions/filters.py +108 -0
- platform_plugin_aspects/management/__init__.py +0 -0
- platform_plugin_aspects/management/commands/__init__.py +0 -0
- platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +214 -0
- platform_plugin_aspects/management/commands/load_test_tracking_events.py +310 -0
- platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +440 -0
- platform_plugin_aspects/settings/__init__.py +0 -0
- platform_plugin_aspects/settings/common.py +147 -0
- platform_plugin_aspects/settings/production.py +38 -0
- platform_plugin_aspects/signals.py +305 -0
- platform_plugin_aspects/sinks/__init__.py +11 -0
- platform_plugin_aspects/sinks/base_sink.py +381 -0
- platform_plugin_aspects/sinks/course_enrollment_sink.py +20 -0
- platform_plugin_aspects/sinks/course_overview_sink.py +286 -0
- platform_plugin_aspects/sinks/external_id_sink.py +20 -0
- platform_plugin_aspects/sinks/serializers.py +274 -0
- platform_plugin_aspects/sinks/tag_sink.py +47 -0
- platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
- platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
- platform_plugin_aspects/static/css/superset.css +53 -0
- platform_plugin_aspects/static/html/superset.html +39 -0
- platform_plugin_aspects/static/html/superset_student.html +5 -0
- platform_plugin_aspects/static/js/embed_dashboard.js +94 -0
- platform_plugin_aspects/static/js/install_required.js +11 -0
- platform_plugin_aspects/static/js/superset.js +38 -0
- platform_plugin_aspects/tasks.py +64 -0
- platform_plugin_aspects/templates/instructor_dashboard/aspects.html +22 -0
- platform_plugin_aspects/urls.py +39 -0
- platform_plugin_aspects/utils.py +359 -0
- platform_plugin_aspects/views.py +296 -0
- platform_plugin_aspects/waffle.py +5 -0
- platform_plugin_aspects/xblock.py +239 -0
- platform_plugin_aspects-2.0.1.dist-info/METADATA +335 -0
- platform_plugin_aspects-2.0.1.dist-info/RECORD +51 -0
- platform_plugin_aspects-2.0.1.dist-info/WHEEL +5 -0
- platform_plugin_aspects-2.0.1.dist-info/entry_points.txt +8 -0
- platform_plugin_aspects-2.0.1.dist-info/licenses/LICENSE +201 -0
- platform_plugin_aspects-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Monitors the load test tracking script and saves output for later analysis.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import datetime
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
from textwrap import dedent
|
|
11
|
+
from time import sleep
|
|
12
|
+
from typing import Any, Union
|
|
13
|
+
|
|
14
|
+
import redis
|
|
15
|
+
import requests
|
|
16
|
+
from django.conf import settings
|
|
17
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
18
|
+
|
|
19
|
+
from platform_plugin_aspects.sinks.base_sink import ClickHouseAuth
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
import confluent_kafka
|
|
23
|
+
except ImportError:
|
|
24
|
+
confluent_kafka = None
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger("tracking_event_loadtest_monitor")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Monitor:
|
|
30
|
+
"""
|
|
31
|
+
Manages the configuration and state of the load test monitor.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
run_id = None
|
|
35
|
+
|
|
36
|
+
def __init__(self, sleep_time: float, backend: str):
|
|
37
|
+
self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
|
|
38
|
+
self.ch_auth = ClickHouseAuth(
|
|
39
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
|
|
40
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
|
|
41
|
+
)
|
|
42
|
+
self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
|
|
43
|
+
self.ch_xapi_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
44
|
+
"xapi_database", "xapi"
|
|
45
|
+
)
|
|
46
|
+
self.ch_xapi_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
47
|
+
"xapi_table", "xapi_events_all"
|
|
48
|
+
)
|
|
49
|
+
self.ch_stats_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
50
|
+
"stats_table", "load_test_stats"
|
|
51
|
+
)
|
|
52
|
+
self.ch_runs_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
53
|
+
"runs_table", "load_test_runs"
|
|
54
|
+
)
|
|
55
|
+
self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
|
|
56
|
+
"timeout_secs"
|
|
57
|
+
]
|
|
58
|
+
self.sleep_time = sleep_time
|
|
59
|
+
self.backend = backend
|
|
60
|
+
|
|
61
|
+
def check_for_run_id(self) -> Union[str, None]:
|
|
62
|
+
"""
|
|
63
|
+
Return a run id for any unfinished run started in the last minute.
|
|
64
|
+
"""
|
|
65
|
+
query = f"""
|
|
66
|
+
SELECT run_id
|
|
67
|
+
FROM {self.ch_runs_table} runs
|
|
68
|
+
WHERE event_type = 'start'
|
|
69
|
+
AND run_id NOT IN (
|
|
70
|
+
SELECT run_id
|
|
71
|
+
FROM {self.ch_runs_table}
|
|
72
|
+
WHERE event_type = 'end'
|
|
73
|
+
)
|
|
74
|
+
AND dateDiff('second', now(), timestamp) <= 60
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
response = requests.post(
|
|
78
|
+
url=self.ch_url,
|
|
79
|
+
auth=self.ch_auth,
|
|
80
|
+
params={"database": self.ch_database, "query": query},
|
|
81
|
+
timeout=self.ch_timeout_secs,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
response.raise_for_status()
|
|
85
|
+
|
|
86
|
+
# ClickHouse will respond with the ID and a newline or an empty string
|
|
87
|
+
return response.text.strip()
|
|
88
|
+
|
|
89
|
+
def wait_for_start(self) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Wait for a run to start, tested by checking ClickHouse for new run ids.
|
|
92
|
+
"""
|
|
93
|
+
while True:
|
|
94
|
+
self.run_id = self.check_for_run_id()
|
|
95
|
+
|
|
96
|
+
if self.run_id:
|
|
97
|
+
log.info(f"Found run id {self.run_id}! Starting monitor.")
|
|
98
|
+
return
|
|
99
|
+
|
|
100
|
+
log.info("No run id from the last 60 seconds found...") # pragma: no cover
|
|
101
|
+
sleep(2) # pragma: no cover
|
|
102
|
+
|
|
103
|
+
def test_has_ended(self) -> bool:
|
|
104
|
+
"""
|
|
105
|
+
Return True if the current run has finished.
|
|
106
|
+
"""
|
|
107
|
+
query = f"""
|
|
108
|
+
SELECT run_id
|
|
109
|
+
FROM {self.ch_runs_table} runs
|
|
110
|
+
WHERE run_id = '{self.run_id}'
|
|
111
|
+
AND event_type = 'end'
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
response = requests.post(
|
|
115
|
+
url=self.ch_url,
|
|
116
|
+
auth=self.ch_auth,
|
|
117
|
+
params={"database": self.ch_database, "query": query},
|
|
118
|
+
timeout=self.ch_timeout_secs,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
response.raise_for_status()
|
|
122
|
+
|
|
123
|
+
# ClickHouse will respond with the ID and a newline or an empty string
|
|
124
|
+
return response.text.strip() == self.run_id
|
|
125
|
+
|
|
126
|
+
def run(self) -> None:
|
|
127
|
+
"""
|
|
128
|
+
Wait for a new test to start, then run the monitor until killed.
|
|
129
|
+
"""
|
|
130
|
+
collect_redis_bus = self.backend == "redis_bus"
|
|
131
|
+
collect_celery = self.backend == "celery"
|
|
132
|
+
collect_kafka_bus = self.backend == "kafka_bus"
|
|
133
|
+
collect_vector = self.backend == "vector"
|
|
134
|
+
|
|
135
|
+
log.info("Waiting for test to start...")
|
|
136
|
+
self.wait_for_start()
|
|
137
|
+
|
|
138
|
+
# Once the test has ended we will wait for the backend to drain the
|
|
139
|
+
# backlog before exiting. This tracks the "in between" state.
|
|
140
|
+
shutting_down = False
|
|
141
|
+
|
|
142
|
+
while True:
|
|
143
|
+
start = datetime.datetime.now()
|
|
144
|
+
log.info(f"----------- {start} --------")
|
|
145
|
+
lag = None
|
|
146
|
+
|
|
147
|
+
current_stats = {"clickhouse": self.get_clickhouse_stats()}
|
|
148
|
+
lag = -1
|
|
149
|
+
if collect_redis_bus:
|
|
150
|
+
current_stats["redis_bus"] = self.get_redis_bus_stats()
|
|
151
|
+
lag = current_stats["redis_bus"]["lag"]
|
|
152
|
+
elif collect_celery:
|
|
153
|
+
current_stats["celery"] = self.get_celery_stats()
|
|
154
|
+
lag = current_stats["celery"]["lag"]
|
|
155
|
+
elif collect_kafka_bus:
|
|
156
|
+
current_stats["kafka_bus"] = self.get_kafka_bus_stats()
|
|
157
|
+
lag = current_stats["kafka_bus"]["lag"]
|
|
158
|
+
elif collect_vector:
|
|
159
|
+
current_stats["vector"] = self.get_vector_stats()
|
|
160
|
+
lag = current_stats["vector"]["lag"]
|
|
161
|
+
|
|
162
|
+
self.store_stats(current_stats)
|
|
163
|
+
|
|
164
|
+
if not shutting_down and self.test_has_ended(): # pragma: no cover
|
|
165
|
+
shutting_down = True
|
|
166
|
+
log.info(
|
|
167
|
+
"----- Test has ended, waiting for events to drain off or force end with CTRL-C"
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
if shutting_down and not lag:
|
|
171
|
+
log.info("----- Test has ended, events are drained. Ending monitor!")
|
|
172
|
+
break
|
|
173
|
+
|
|
174
|
+
# Try to keep our collection cadence to exactly what was asked
|
|
175
|
+
# otherwise
|
|
176
|
+
check_duration = datetime.datetime.now() - start
|
|
177
|
+
|
|
178
|
+
if check_duration.total_seconds() >= self.sleep_time:
|
|
179
|
+
log.warning(
|
|
180
|
+
f"It took {check_duration} to collect and store stats, this is greater than the sleep time!"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
next_sleep = self.sleep_time - check_duration.total_seconds()
|
|
184
|
+
sleep(next_sleep)
|
|
185
|
+
|
|
186
|
+
def store_stats(self, current_stats: dict) -> None:
|
|
187
|
+
"""
|
|
188
|
+
Send the results for this iteration to ClickHouse.
|
|
189
|
+
"""
|
|
190
|
+
stats = json.dumps(current_stats)
|
|
191
|
+
|
|
192
|
+
insert = f"""INSERT INTO {self.ch_stats_table} (run_id, stats) FORMAT CSV"""
|
|
193
|
+
|
|
194
|
+
output = io.StringIO()
|
|
195
|
+
writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
|
|
196
|
+
writer.writerow((self.run_id, stats))
|
|
197
|
+
|
|
198
|
+
response = requests.post(
|
|
199
|
+
url=self.ch_url,
|
|
200
|
+
auth=self.ch_auth,
|
|
201
|
+
params={"database": self.ch_database, "query": insert},
|
|
202
|
+
data=output.getvalue().encode("utf-8"),
|
|
203
|
+
timeout=self.ch_timeout_secs,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
response.raise_for_status()
|
|
207
|
+
|
|
208
|
+
def get_clickhouse_stats(self):
|
|
209
|
+
"""
|
|
210
|
+
Get the current state of ClickHouse for this iteration.
|
|
211
|
+
"""
|
|
212
|
+
select = f"""
|
|
213
|
+
SELECT
|
|
214
|
+
count(*) as ttl_count,
|
|
215
|
+
max(emission_time) as most_recent,
|
|
216
|
+
date_diff('second', max(emission_time), now()) as lag_seconds
|
|
217
|
+
FROM {self.ch_xapi_table}
|
|
218
|
+
FINAL
|
|
219
|
+
FORMAT JSON
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
response = requests.post(
|
|
223
|
+
url=self.ch_url,
|
|
224
|
+
auth=self.ch_auth,
|
|
225
|
+
params={"database": self.ch_xapi_database, "query": select},
|
|
226
|
+
timeout=self.ch_timeout_secs,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
response.raise_for_status()
|
|
230
|
+
|
|
231
|
+
resp = response.json()["data"][0]
|
|
232
|
+
log.info(f"Clickhouse lag seconds: {resp['lag_seconds']}")
|
|
233
|
+
|
|
234
|
+
return {
|
|
235
|
+
"total_rows": resp["ttl_count"],
|
|
236
|
+
"most_recent_event": resp["most_recent"],
|
|
237
|
+
"lag_seconds": resp["lag_seconds"],
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
def get_celery_stats(self):
|
|
241
|
+
"""
|
|
242
|
+
Get the current state of Celery for this iteration.
|
|
243
|
+
"""
|
|
244
|
+
r = redis.Redis.from_url(settings.BROKER_URL)
|
|
245
|
+
lms_queue = r.llen("edx.lms.core.default")
|
|
246
|
+
cms_queue = r.llen("edx.cms.core.default")
|
|
247
|
+
|
|
248
|
+
log.info(f"Celery queues: LMS {lms_queue}, CMS {cms_queue}")
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
"lms_queue_length": lms_queue,
|
|
252
|
+
"cms_queue_length": cms_queue,
|
|
253
|
+
"lag": lms_queue + cms_queue,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
def get_redis_bus_stats(self):
|
|
257
|
+
"""
|
|
258
|
+
Get the current state of redis for this iteration.
|
|
259
|
+
"""
|
|
260
|
+
r = redis.Redis.from_url(settings.EVENT_BUS_REDIS_CONNECTION_URL)
|
|
261
|
+
info = r.xinfo_stream("openedx-analytics", full=True)
|
|
262
|
+
|
|
263
|
+
lag = 0
|
|
264
|
+
|
|
265
|
+
try:
|
|
266
|
+
for g in info["groups"]:
|
|
267
|
+
lag += g["lag"]
|
|
268
|
+
# Older versions of redis don't have "lag".
|
|
269
|
+
except KeyError: # pragma: no cover
|
|
270
|
+
pass
|
|
271
|
+
|
|
272
|
+
consumer_stats = {
|
|
273
|
+
"total_events": info["length"],
|
|
274
|
+
"lag": lag,
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
log.info(f"Redis bus queue length: {consumer_stats['lag']}")
|
|
278
|
+
|
|
279
|
+
return consumer_stats
|
|
280
|
+
|
|
281
|
+
def get_kafka_bus_stats(self):
|
|
282
|
+
"""
|
|
283
|
+
Get the current state of ClickHouse for this iteration.
|
|
284
|
+
"""
|
|
285
|
+
if not confluent_kafka: # pragma: no cover
|
|
286
|
+
raise CommandError(
|
|
287
|
+
"Trying to monitor Kafka bus, but confluent_kafka is not installed"
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
brokers = settings.EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS
|
|
291
|
+
topic = f"{settings.EVENT_BUS_TOPIC_PREFIX}-analytics"
|
|
292
|
+
group = "analytics-service"
|
|
293
|
+
|
|
294
|
+
# This consumer will not join the group, but the group.id is required by
|
|
295
|
+
# committed() to know which group to get offsets for.
|
|
296
|
+
consumer = confluent_kafka.Consumer(
|
|
297
|
+
{"bootstrap.servers": brokers, "group.id": group}
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
# Get the topic's partitions
|
|
301
|
+
metadata = consumer.list_topics(topic, timeout=10)
|
|
302
|
+
|
|
303
|
+
if metadata.topics[topic].error is not None: # pragma: no cover
|
|
304
|
+
log.info(metadata.topics[topic].error)
|
|
305
|
+
|
|
306
|
+
partitions = [
|
|
307
|
+
confluent_kafka.TopicPartition(topic, p)
|
|
308
|
+
for p in metadata.topics[topic].partitions
|
|
309
|
+
]
|
|
310
|
+
committed = consumer.committed(partitions, timeout=10)
|
|
311
|
+
|
|
312
|
+
consumer_stats = {
|
|
313
|
+
"topic": topic,
|
|
314
|
+
"partitions": [],
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
total_lag = 0
|
|
318
|
+
|
|
319
|
+
for partition in committed:
|
|
320
|
+
# Get the partitions low and high watermark offsets.
|
|
321
|
+
low, high = consumer.get_watermark_offsets(
|
|
322
|
+
partition, timeout=10, cached=False
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
if high < 0: # pragma: no cover
|
|
326
|
+
lag = 0
|
|
327
|
+
elif partition.offset < 0: # pragma: no cover
|
|
328
|
+
# No committed offset, show total message count as lag.
|
|
329
|
+
# The actual message count may be lower due to compaction
|
|
330
|
+
# and record deletions.
|
|
331
|
+
lag = high - low
|
|
332
|
+
else:
|
|
333
|
+
lag = high - partition.offset
|
|
334
|
+
|
|
335
|
+
log.info(f"{partition.topic} [{partition.partition}] Lag: {lag}")
|
|
336
|
+
total_lag += lag
|
|
337
|
+
|
|
338
|
+
consumer_stats["partitions"].append(
|
|
339
|
+
{
|
|
340
|
+
"partition": partition.partition,
|
|
341
|
+
"lag": lag,
|
|
342
|
+
}
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
consumer.close()
|
|
346
|
+
consumer_stats["lag"] = total_lag
|
|
347
|
+
return consumer_stats
|
|
348
|
+
|
|
349
|
+
def _call_vector_graphql(self):
|
|
350
|
+
"""
|
|
351
|
+
Make the actual GraphQL call to the Vector API.
|
|
352
|
+
"""
|
|
353
|
+
# These values are hard coded in tutor local, K8s changes TBD
|
|
354
|
+
url = "http://vector:8686/graphql"
|
|
355
|
+
query = """
|
|
356
|
+
{
|
|
357
|
+
sinks(filter:{componentId:{equals:"clickhouse_xapi"}}) {
|
|
358
|
+
edges {
|
|
359
|
+
node {
|
|
360
|
+
...on Sink {
|
|
361
|
+
componentId,
|
|
362
|
+
componentType,
|
|
363
|
+
metrics {
|
|
364
|
+
receivedEventsTotal {timestamp, receivedEventsTotal},
|
|
365
|
+
sentEventsTotal {timestamp, sentEventsTotal}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
"""
|
|
373
|
+
r = requests.post(url, json={"query": query}, timeout=10)
|
|
374
|
+
r.raise_for_status()
|
|
375
|
+
return r.json()["data"]["sinks"]["edges"][0]["node"]["metrics"]
|
|
376
|
+
|
|
377
|
+
def get_vector_stats(self):
|
|
378
|
+
"""
|
|
379
|
+
Get the current state of Vector for this iteration.
|
|
380
|
+
"""
|
|
381
|
+
metrics = self._call_vector_graphql()
|
|
382
|
+
|
|
383
|
+
# These will be null until events start arriving
|
|
384
|
+
received = (
|
|
385
|
+
metrics["receivedEventsTotal"]["receivedEventsTotal"]
|
|
386
|
+
if metrics["receivedEventsTotal"]
|
|
387
|
+
else 0.0
|
|
388
|
+
)
|
|
389
|
+
sent = (
|
|
390
|
+
metrics["sentEventsTotal"]["sentEventsTotal"]
|
|
391
|
+
if metrics["sentEventsTotal"]
|
|
392
|
+
else 0.0
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
rtn = {"events_received": received, "events_sent": sent, "lag": received - sent}
|
|
396
|
+
|
|
397
|
+
log.info(
|
|
398
|
+
f"Vector received: {rtn['events_received']} sent: {rtn['events_sent']} lag: {rtn.get('lag')}"
|
|
399
|
+
)
|
|
400
|
+
return rtn
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class Command(BaseCommand):
|
|
404
|
+
"""
|
|
405
|
+
Dump objects to a ClickHouse instance.
|
|
406
|
+
|
|
407
|
+
Example:
|
|
408
|
+
tutor local run lms ./manage.py lms monitor_load_test_tracking --sleep_time 5 --backend redis_bus
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
help = dedent(__doc__).strip()
|
|
412
|
+
|
|
413
|
+
def add_arguments(self, parser: Any) -> None:
|
|
414
|
+
parser.add_argument(
|
|
415
|
+
"--sleep_time",
|
|
416
|
+
type=float,
|
|
417
|
+
default=10,
|
|
418
|
+
help="Fractional number of seconds to sleep between gathering data.",
|
|
419
|
+
)
|
|
420
|
+
parser.add_argument(
|
|
421
|
+
"--backend",
|
|
422
|
+
choices=["redis_bus", "kafka_bus", "celery", "vector"],
|
|
423
|
+
default="celery",
|
|
424
|
+
help="Backend used to send events to ClickHouse",
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
def handle(self, *_, **options):
|
|
428
|
+
"""
|
|
429
|
+
Creates users and triggers events for them as configured above.
|
|
430
|
+
"""
|
|
431
|
+
start = datetime.datetime.now()
|
|
432
|
+
log.info(
|
|
433
|
+
f"Starting monitor for {options['backend']} with sleep of {options['sleep_time']} seconds"
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
monitor = Monitor(options["sleep_time"], options["backend"])
|
|
437
|
+
monitor.run()
|
|
438
|
+
|
|
439
|
+
end = datetime.datetime.now()
|
|
440
|
+
log.info(f"Monitored from {start} to {end} (duration {end - start}).")
|
|
File without changes
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Common Django settings for eox_hooks project.
|
|
3
|
+
For more information on this file, see
|
|
4
|
+
https://docs.djangoproject.com/en/2.22/topics/settings/
|
|
5
|
+
For the full list of settings and their values, see
|
|
6
|
+
https://docs.djangoproject.com/en/2.22/ref/settings/
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from platform_plugin_aspects import ROOT_DIRECTORY
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Make '_' a no-op so we can scrape strings for translation, because
|
|
13
|
+
# `django.utils.translation.ugettext_noop` cannot be imported in this file
|
|
14
|
+
def _(text):
|
|
15
|
+
return text
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def plugin_settings(settings):
|
|
19
|
+
"""
|
|
20
|
+
Set of plugin settings used by the Open Edx platform.
|
|
21
|
+
More info: https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/plugins/README.rst
|
|
22
|
+
"""
|
|
23
|
+
settings.MAKO_TEMPLATE_DIRS_BASE.append(ROOT_DIRECTORY / "templates")
|
|
24
|
+
settings.SUPERSET_CONFIG = {
|
|
25
|
+
"internal_service_url": "http://superset:8088",
|
|
26
|
+
"username": "superset",
|
|
27
|
+
"password": "superset",
|
|
28
|
+
}
|
|
29
|
+
settings.ASPECTS_INSTRUCTOR_DASHBOARDS = [
|
|
30
|
+
{
|
|
31
|
+
"name": _("Course Dashboard"),
|
|
32
|
+
"slug": "course-dashboard",
|
|
33
|
+
"uuid": "c0e64194-33d1-4d5a-8c10-4f51530c5ee9",
|
|
34
|
+
"allow_translations": True,
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": _("Individual Learner Dashboard"),
|
|
38
|
+
"slug": "individual-learner",
|
|
39
|
+
"uuid": "abae8a25-1ba4-4653-81bd-d3937a162a11",
|
|
40
|
+
"allow_translations": True,
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"name": _("At-Risk Learners Dashboard"),
|
|
44
|
+
"slug": "learner-groups",
|
|
45
|
+
"uuid": "8661d20c-cee6-4245-9fcc-610daea5fd24",
|
|
46
|
+
"allow_translations": True,
|
|
47
|
+
},
|
|
48
|
+
]
|
|
49
|
+
settings.SUPERSET_SHOW_INSTRUCTOR_DASHBOARD_LINK = True
|
|
50
|
+
settings.SUPERSET_EXTRA_FILTERS_FORMAT = []
|
|
51
|
+
settings.SUPERSET_DASHBOARD_LOCALES = ["en"]
|
|
52
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG = {
|
|
53
|
+
# URL to a running ClickHouse server's HTTP interface. ex: https://foo.openedx.org:8443/ or
|
|
54
|
+
# http://foo.openedx.org:8123/ . Note that we only support the ClickHouse HTTP interface
|
|
55
|
+
# to avoid pulling in more dependencies to the platform than necessary.
|
|
56
|
+
"url": "http://clickhouse:8123",
|
|
57
|
+
"username": "ch_cms",
|
|
58
|
+
"password": "password",
|
|
59
|
+
"database": "event_sink",
|
|
60
|
+
"timeout_secs": 5,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS = [
|
|
64
|
+
"user_profile",
|
|
65
|
+
"external_id",
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
settings.EVENT_SINK_CLICKHOUSE_MODEL_CONFIG = {
|
|
69
|
+
"auth_user": {
|
|
70
|
+
"module": "django.contrib.auth.models",
|
|
71
|
+
"model": "User",
|
|
72
|
+
},
|
|
73
|
+
"user_profile": {
|
|
74
|
+
"module": "common.djangoapps.student.models",
|
|
75
|
+
"model": "UserProfile",
|
|
76
|
+
},
|
|
77
|
+
"course_overviews": {
|
|
78
|
+
"module": "openedx.core.djangoapps.content.course_overviews.models",
|
|
79
|
+
"model": "CourseOverview",
|
|
80
|
+
},
|
|
81
|
+
"external_id": {
|
|
82
|
+
"module": "openedx.core.djangoapps.external_user_ids.models",
|
|
83
|
+
"model": "ExternalId",
|
|
84
|
+
},
|
|
85
|
+
"course_enrollment": {
|
|
86
|
+
"module": "common.djangoapps.student.models",
|
|
87
|
+
"model": "CourseEnrollment",
|
|
88
|
+
},
|
|
89
|
+
"custom_course_edx": {
|
|
90
|
+
"module": "lms.djangoapps.ccx.models",
|
|
91
|
+
"model": "CustomCourseForEdX",
|
|
92
|
+
},
|
|
93
|
+
"user_preference": {
|
|
94
|
+
"module": "openedx.core.djangoapps.user_api.models",
|
|
95
|
+
"model": "UserPreference",
|
|
96
|
+
},
|
|
97
|
+
"tag": {"module": "openedx_tagging.models.base", "model": "Tag"},
|
|
98
|
+
"taxonomy": {
|
|
99
|
+
"module": "openedx_tagging.models.base",
|
|
100
|
+
"model": "Taxonomy",
|
|
101
|
+
},
|
|
102
|
+
"object_tag": {
|
|
103
|
+
"module": "openedx_tagging.models.base",
|
|
104
|
+
"model": "ObjectTag",
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
# This is on by default here, off by default in tutor-contrib-aspects until we only
|
|
109
|
+
# support versions Sumac and beyond
|
|
110
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = True
|
|
111
|
+
|
|
112
|
+
settings.ASPECTS_IN_CONTEXT_DASHBOARDS = {
|
|
113
|
+
"course": {
|
|
114
|
+
"name": _("Course"),
|
|
115
|
+
"slug": "in-context-course",
|
|
116
|
+
"uuid": "f2880cc1-63e9-48d7-ac3c-d2ff6f6698e2",
|
|
117
|
+
"allow_translations": True,
|
|
118
|
+
"course_filter_ids": ["NATIVE_FILTER-QLQbulmHH"],
|
|
119
|
+
"block_filter_ids": [],
|
|
120
|
+
},
|
|
121
|
+
"sequential": {
|
|
122
|
+
"name": _("Graded Subsection"),
|
|
123
|
+
"slug": "in-context-graded-subsection",
|
|
124
|
+
"uuid": "f0321087-6428-4b97-b32e-2dae7d9cc447",
|
|
125
|
+
"allow_translations": True,
|
|
126
|
+
"course_filter_ids": ["NATIVE_FILTER-oPAuR7ahy"],
|
|
127
|
+
"block_filter_ids": ["NATIVE_FILTER-CBWbI7uLq"],
|
|
128
|
+
},
|
|
129
|
+
"problem": {
|
|
130
|
+
"name": _("Problem"),
|
|
131
|
+
"slug": "in-context-problem",
|
|
132
|
+
"uuid": "98ff33ff-18dd-48f9-8c58-629ae4f4194b",
|
|
133
|
+
"allow_translations": True,
|
|
134
|
+
"course_filter_ids": ["NATIVE_FILTER-29CPcbirK"],
|
|
135
|
+
"block_filter_ids": ["NATIVE_FILTER-TJwItQhUI"],
|
|
136
|
+
},
|
|
137
|
+
"video": {
|
|
138
|
+
"name": _("Video"),
|
|
139
|
+
"slug": "in-context-video",
|
|
140
|
+
"uuid": "bc6510fb-027f-4026-a333-d0c42d3cc35c",
|
|
141
|
+
"allow_translations": True,
|
|
142
|
+
"course_filter_ids": ["NATIVE_FILTER-uaxvZkSAg"],
|
|
143
|
+
"block_filter_ids": ["NATIVE_FILTER-Fse4rzDW0"],
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
settings.IN_CONTEXT_DASHBOARD_COURSE_KEY_COLUMN = "course_key"
|
|
147
|
+
settings.IN_CONTEXT_DASHBOARD_BLOCK_ID_COLUMN = "block_id"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Production Django settings for Aspects project.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def plugin_settings(settings):
|
|
7
|
+
"""
|
|
8
|
+
Set of plugin settings used by the Open Edx platform.
|
|
9
|
+
More info: https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/plugins/README.rst
|
|
10
|
+
"""
|
|
11
|
+
settings.SUPERSET_CONFIG = getattr(settings, "ENV_TOKENS", {}).get(
|
|
12
|
+
"SUPERSET_CONFIG", settings.SUPERSET_CONFIG
|
|
13
|
+
)
|
|
14
|
+
settings.ASPECTS_INSTRUCTOR_DASHBOARDS = getattr(settings, "ENV_TOKENS", {}).get(
|
|
15
|
+
"ASPECTS_INSTRUCTOR_DASHBOARDS", settings.ASPECTS_INSTRUCTOR_DASHBOARDS
|
|
16
|
+
)
|
|
17
|
+
settings.SUPERSET_EXTRA_FILTERS_FORMAT = getattr(settings, "ENV_TOKENS", {}).get(
|
|
18
|
+
"SUPERSET_EXTRA_FILTERS_FORMAT", settings.SUPERSET_EXTRA_FILTERS_FORMAT
|
|
19
|
+
)
|
|
20
|
+
settings.SUPERSET_DASHBOARD_LOCALES = getattr(settings, "ENV_TOKENS", {}).get(
|
|
21
|
+
"SUPERSET_DASHBOARD_LOCALES", settings.SUPERSET_DASHBOARD_LOCALES
|
|
22
|
+
)
|
|
23
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG = settings.ENV_TOKENS.get(
|
|
24
|
+
"EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG",
|
|
25
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG,
|
|
26
|
+
)
|
|
27
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS = settings.ENV_TOKENS.get(
|
|
28
|
+
"EVENT_SINK_CLICKHOUSE_PII_MODELS",
|
|
29
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS,
|
|
30
|
+
)
|
|
31
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = settings.ENV_TOKENS.get(
|
|
32
|
+
"ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS",
|
|
33
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS,
|
|
34
|
+
)
|
|
35
|
+
settings.ASPECTS_IN_CONTEXT_DASHBOARDS = settings.ENV_TOKENS.get(
|
|
36
|
+
"ASPECTS_IN_CONTEXT_DASHBOARDS",
|
|
37
|
+
settings.ASPECTS_IN_CONTEXT_DASHBOARDS,
|
|
38
|
+
)
|