platform-plugin-aspects 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/CHANGELOG.rst +9 -0
  2. {platform-plugin-aspects-0.4.0/platform_plugin_aspects.egg-info → platform-plugin-aspects-0.5.0}/PKG-INFO +11 -1
  3. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/__init__.py +1 -1
  4. platform-plugin-aspects-0.5.0/platform_plugin_aspects/management/commands/load_test_tracking_events.py +313 -0
  5. platform-plugin-aspects-0.5.0/platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +438 -0
  6. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/utils.py +1 -1
  7. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0/platform_plugin_aspects.egg-info}/PKG-INFO +11 -1
  8. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/SOURCES.txt +2 -0
  9. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/requires.txt +1 -0
  10. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/requirements/base.in +1 -0
  11. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/LICENSE +0 -0
  12. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/MANIFEST.in +0 -0
  13. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/README.rst +0 -0
  14. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/apps.py +0 -0
  15. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/extensions/__init__.py +0 -0
  16. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/extensions/filters.py +0 -0
  17. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/management/__init__.py +0 -0
  18. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/management/commands/__init__.py +0 -0
  19. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +0 -0
  20. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/settings/__init__.py +0 -0
  21. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/settings/common.py +0 -0
  22. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/settings/production.py +0 -0
  23. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/signals.py +0 -0
  24. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/__init__.py +0 -0
  25. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/base_sink.py +0 -0
  26. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/course_overview_sink.py +0 -0
  27. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/external_id_sink.py +0 -0
  28. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/serializers.py +0 -0
  29. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/user_profile_sink.py +0 -0
  30. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/sinks/user_retire_sink.py +0 -0
  31. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/css/superset.css +0 -0
  32. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/html/superset.html +0 -0
  33. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/html/superset_student.html +0 -0
  34. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/js/embed_dashboard.js +0 -0
  35. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/js/install_required.js +0 -0
  36. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/static/js/superset.js +0 -0
  37. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/tasks.py +0 -0
  38. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/templates/instructor_dashboard/aspects.html +0 -0
  39. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/waffle.py +0 -0
  40. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects/xblock.py +0 -0
  41. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/dependency_links.txt +0 -0
  42. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/entry_points.txt +0 -0
  43. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/not-zip-safe +0 -0
  44. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/platform_plugin_aspects.egg-info/top_level.txt +0 -0
  45. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/requirements/constraints.txt +0 -0
  46. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/setup.cfg +0 -0
  47. {platform-plugin-aspects-0.4.0 → platform-plugin-aspects-0.5.0}/setup.py +0 -0
@@ -16,6 +16,15 @@ Unreleased
16
16
 
17
17
  *
18
18
 
19
+ 0.5.0 - 2024-04-01
20
+ ******************
21
+
22
+ Added
23
+ =====
24
+
25
+ * Load testing and test monitoring scripts.
26
+
27
+
19
28
  0.4.0 - 2024-03-18
20
29
  ******************
21
30
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: platform-plugin-aspects
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Aspects plugins for edx-platform
5
5
  Home-page: https://github.com/openedx/platform-plugin-aspects
6
6
  Author: Open edX Project
@@ -27,6 +27,7 @@ Requires-Dist: edx-opaque-keys
27
27
  Requires-Dist: edx-toggles
28
28
  Requires-Dist: openedx-atlas
29
29
  Requires-Dist: openedx-filters
30
+ Requires-Dist: redis
30
31
  Requires-Dist: requests
31
32
  Requires-Dist: superset-api-client
32
33
  Requires-Dist: web_fragments
@@ -149,6 +150,15 @@ Unreleased
149
150
 
150
151
  *
151
152
 
153
+ 0.5.0 - 2024-04-01
154
+ ******************
155
+
156
+ Added
157
+ =====
158
+
159
+ * Load testing and test monitoring scripts.
160
+
161
+
152
162
  0.4.0 - 2024-03-18
153
163
  ******************
154
164
 
@@ -5,6 +5,6 @@ Aspects plugins for edx-platform.
5
5
  import os
6
6
  from pathlib import Path
7
7
 
8
- __version__ = "0.4.0"
8
+ __version__ = "0.5.0"
9
9
 
10
10
  ROOT_DIRECTORY = Path(os.path.dirname(os.path.abspath(__file__)))
@@ -0,0 +1,313 @@
1
+ """
2
+ Generates tracking events by creating test users and fake activity.
3
+
4
+ This should never be run on a production server as it will generate a lot of
5
+ bad data. It is entirely for benchmarking purposes in load test environments.
6
+ It is also fragile due to reaching into the edx-platform testing internals.
7
+ """
8
+
9
+ import csv
10
+ import io
11
+ import json
12
+ import logging
13
+ import uuid
14
+ from datetime import datetime, timedelta
15
+ from random import choice
16
+ from textwrap import dedent
17
+ from time import sleep
18
+ from typing import Any, List
19
+
20
+ import requests
21
+ from django.conf import settings
22
+ from django.core.management.base import BaseCommand, CommandError
23
+
24
+ from platform_plugin_aspects.sinks.base_sink import ClickHouseAuth
25
+
26
+ # For testing we won't be able to import from edx-platform
27
+ try: # pragma: no cover
28
+ from cms.djangoapps.contentstore.views.course import create_new_course_in_store
29
+ from common.djangoapps.student.helpers import do_create_account
30
+ from common.djangoapps.student.models.course_enrollment import CourseEnrollment
31
+ from openedx.core.djangoapps.user_authn.views.registration_form import (
32
+ AccountCreationForm,
33
+ )
34
+ from xmodule.modulestore import ModuleStoreEnum
35
+
36
+ RUNNING_IN_PLATFORM = True
37
+ except ImportError:
38
+ create_new_course_in_store = None
39
+ do_create_account = None
40
+ CourseEnrollment = None
41
+ AccountCreationForm = None
42
+ ModuleStoreEnum = None
43
+
44
+ RUNNING_IN_PLATFORM = False
45
+
46
+ log = logging.getLogger("tracking_event_loadtest")
47
+
48
+
49
+ class LoadTest:
50
+ """
51
+ Runs the load test and reports results to ClickHouse.
52
+ """
53
+
54
+ users = []
55
+ sent_event_count = 0
56
+ ch_runs_table = "load_test_runs"
57
+
58
+ def __init__(self, num_users: int, username_prefix: str, tags: List[str]):
59
+ self.num_users = num_users
60
+ self.username_prefix = username_prefix
61
+ self.tags = tags
62
+ self.run_id = str(uuid.uuid4())[:6]
63
+
64
+ self.course_shortname = str(uuid.uuid4())[:6]
65
+
66
+ self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
67
+ self.ch_auth = ClickHouseAuth(
68
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
69
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
70
+ )
71
+ self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
72
+ self.ch_xapi_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
73
+ "xapi_database", "xapi"
74
+ )
75
+ self.ch_runs_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
76
+ "runs_table", "load_test_runs"
77
+ )
78
+ self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
79
+ "timeout_secs"
80
+ ]
81
+
82
+ self.instructor = self.create_user(
83
+ username=f"instructor_{self.course_shortname}",
84
+ name="Instructor",
85
+ password="aspects",
86
+ email=f"instructor_{self.course_shortname}@openedx.invalid",
87
+ )
88
+ self.create_course()
89
+ self.record_start()
90
+ self.create_and_enroll_learners(num_users, username_prefix)
91
+
92
+ def create_course(self) -> None:
93
+ """
94
+ Create a course using the CMS API.
95
+ """
96
+ start_date = datetime.now() - timedelta(days=7)
97
+ fields = {
98
+ "start": start_date,
99
+ "display_name": f"Course {self.course_shortname}",
100
+ }
101
+
102
+ log.info(
103
+ f"""Creating course:
104
+ Instructor: {self.instructor.id}
105
+ Org: "OEX"
106
+ Number: "{self.course_shortname}"
107
+ Run: "2024-1"
108
+ Fields: {fields}
109
+ """
110
+ )
111
+
112
+ self.course = create_new_course_in_store(
113
+ ModuleStoreEnum.Type.split,
114
+ self.instructor,
115
+ "OEX",
116
+ self.course_shortname,
117
+ "2024-1",
118
+ fields,
119
+ )
120
+
121
+ log.info(f"Created course {self.course.id}")
122
+
123
+ def record_start(self) -> None:
124
+ """
125
+ Send a start event to ClickHouse.
126
+
127
+ Start and end events are used by the monitor script to know when to
128
+ begin and end monitoring.
129
+ """
130
+ self.record_to_clickhouse(
131
+ "start",
132
+ {
133
+ "tags": self.tags,
134
+ "course_id": str(self.course.id),
135
+ "num_users": self.num_users,
136
+ "username_prefix": self.username_prefix,
137
+ },
138
+ )
139
+
140
+ # Let the monitoring script connect, otherwise we can finish the test before it even
141
+ # knows we've started.
142
+ sleep(5)
143
+
144
+ def record_end(self) -> None:
145
+ """
146
+ Send an end event to ClickHouse.
147
+ """
148
+ self.record_to_clickhouse("end", {"sent_event_count": self.sent_event_count})
149
+
150
+ def record_to_clickhouse(self, event_type, extra) -> None:
151
+ """
152
+ Send the run events to ClickHouse.
153
+ """
154
+ insert = (
155
+ f"INSERT INTO {self.ch_runs_table} (run_id, event_type, extra) FORMAT CSV "
156
+ )
157
+
158
+ output = io.StringIO()
159
+ writer = csv.writer(output)
160
+ writer.writerow((self.run_id, event_type, json.dumps(extra)))
161
+ print(output.getvalue().encode("utf-8"))
162
+
163
+ response = requests.post(
164
+ url=self.ch_url,
165
+ auth=self.ch_auth,
166
+ params={"database": self.ch_database, "query": insert},
167
+ data=output.getvalue().encode("utf-8"),
168
+ timeout=self.ch_timeout_secs,
169
+ )
170
+
171
+ response.raise_for_status()
172
+
173
+ def create_and_enroll_learners(self, num_users, username_prefix):
174
+ """
175
+ Uses create test users and enroll them in our test course.
176
+ """
177
+ log.info(f"Creating {num_users} users prefixed with {username_prefix}.")
178
+
179
+ for _ in range(num_users):
180
+ user_short_name = str(uuid.uuid4())[:6]
181
+ u = self.create_user(
182
+ username=f"{username_prefix}_{user_short_name}",
183
+ name=f"Learner {user_short_name}",
184
+ password="aspects",
185
+ email=f"{user_short_name}@openedx.invalid",
186
+ )
187
+ self.users.append(u)
188
+ e = CourseEnrollment.get_or_create_enrollment(
189
+ user=u, course_key=self.course.id
190
+ )
191
+ e.is_active = True
192
+ e.save()
193
+
194
+ def create_user(self, **user_data):
195
+ """
196
+ Create, activate, and return a user using the edx-platform API.
197
+ """
198
+ account_creation_form = AccountCreationForm(data=user_data, tos_required=False)
199
+
200
+ user, _, _ = do_create_account(account_creation_form)
201
+ user.is_active = True
202
+ user.save()
203
+ return user
204
+
205
+ def trigger_events(
206
+ self, num_events: int, sleep_time: float, run_until_killed: bool
207
+ ) -> None:
208
+ """
209
+ Trigger the appropriate number of events based on configuration.
210
+ """
211
+
212
+ if run_until_killed:
213
+ log.info(f"Creating events until killed with {sleep_time} sleep between!")
214
+ while True:
215
+ self.trigger_event_and_sleep(sleep_time)
216
+ else:
217
+ log.info(f"Creating {num_events} event with {sleep_time} sleep between!")
218
+ for _ in range(num_events):
219
+ self.trigger_event_and_sleep(sleep_time)
220
+
221
+ def trigger_event_and_sleep(self, sleep_time: float) -> None:
222
+ """
223
+ Cause a tracking log to be emitted and sleep the specified amount of time.
224
+ """
225
+ user = choice(self.users)
226
+
227
+ e = CourseEnrollment.get_or_create_enrollment(
228
+ user=user, course_key=self.course.id
229
+ )
230
+
231
+ if e.is_active:
232
+ e.unenroll(user, self.course.id)
233
+ else:
234
+ e.enroll(user, self.course.id) # pragma: no cover
235
+
236
+ self.sent_event_count += 1
237
+ sleep(sleep_time)
238
+
239
+
240
+ class Command(BaseCommand):
241
+ """
242
+ Create tracking log events for load testing purposes.
243
+
244
+ Example:
245
+ tutor local run lms ./manage.py lms load_test_tracking_events --sleep_time 0 --tags celery 1worker
246
+ """
247
+
248
+ help = dedent(__doc__).strip()
249
+
250
+ def add_arguments(self, parser: Any) -> None:
251
+ parser.add_argument(
252
+ "--num_users",
253
+ type=int,
254
+ default=10,
255
+ help="The number of users to create. All events will be generated for these learners.",
256
+ )
257
+ parser.add_argument(
258
+ "--username_prefix",
259
+ type=str,
260
+ default="lt_",
261
+ help="Prefix for the generated user names.",
262
+ )
263
+ parser.add_argument(
264
+ "--num_events",
265
+ type=int,
266
+ default=10,
267
+ help="The number of events to generate. This is ignored if --run_until_killed is set.",
268
+ )
269
+ parser.add_argument(
270
+ "--run_until_killed",
271
+ action="store_true",
272
+ default=False,
273
+ help="If this is set, the process will run endlessly until killed.",
274
+ )
275
+ parser.add_argument(
276
+ "--sleep_time",
277
+ type=float,
278
+ default=0.75,
279
+ help="Fractional number of seconds to sleep between sending events.",
280
+ )
281
+ parser.add_argument(
282
+ "--tags",
283
+ nargs="*",
284
+ help="Tags to help define the run (ex: --tags celery 3workers k8s).",
285
+ )
286
+
287
+ def handle(self, *args, **options):
288
+ """
289
+ Create users and trigger events for them as configured above.
290
+ """
291
+ if not RUNNING_IN_PLATFORM: # pragma: no cover
292
+ raise CommandError("This command must be run in the Open edX LMS or CMS.")
293
+
294
+ start = datetime.now()
295
+ lt = LoadTest(options["num_users"], options["username_prefix"], options["tags"])
296
+
297
+ try:
298
+ lt.trigger_events(
299
+ options["num_events"],
300
+ options["sleep_time"],
301
+ options["run_until_killed"],
302
+ )
303
+ lt.record_end()
304
+ except KeyboardInterrupt:
305
+ log.warning("Killed by keyboard, finishing.")
306
+ lt.record_end()
307
+
308
+ end = datetime.now()
309
+ log.info(f"Sent {lt.sent_event_count} events in {end - start}.")
310
+
311
+ # Wait 5 seconds for Kafka to clear the queue otherwise not all events
312
+ # may be delivered.
313
+ sleep(5)
@@ -0,0 +1,438 @@
1
+ """
2
+ Monitors the load test tracking script and saves output for later analysis.
3
+ """
4
+
5
+ import csv
6
+ import datetime
7
+ import io
8
+ import json
9
+ import logging
10
+ from textwrap import dedent
11
+ from time import sleep
12
+ from typing import Any, Union
13
+
14
+ import redis
15
+ import requests
16
+ from django.conf import settings
17
+ from django.core.management.base import BaseCommand, CommandError
18
+
19
+ from platform_plugin_aspects.sinks.base_sink import ClickHouseAuth
20
+
21
+ try:
22
+ import confluent_kafka
23
+ except ImportError:
24
+ confluent_kafka = None
25
+
26
+ log = logging.getLogger("tracking_event_loadtest_monitor")
27
+
28
+
29
+ class Monitor:
30
+ """
31
+ Manages the configuration and state of the load test monitor.
32
+ """
33
+
34
+ run_id = None
35
+
36
+ def __init__(self, sleep_time: float, backend: str):
37
+ self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
38
+ self.ch_auth = ClickHouseAuth(
39
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
40
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
41
+ )
42
+ self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
43
+ self.ch_xapi_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
44
+ "xapi_database", "xapi"
45
+ )
46
+ self.ch_xapi_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
47
+ "xapi_table", "xapi_events_all"
48
+ )
49
+ self.ch_stats_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
50
+ "stats_table", "load_test_stats"
51
+ )
52
+ self.ch_runs_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
53
+ "runs_table", "load_test_runs"
54
+ )
55
+ self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
56
+ "timeout_secs"
57
+ ]
58
+ self.sleep_time = sleep_time
59
+ self.backend = backend
60
+
61
+ def check_for_run_id(self) -> Union[str, None]:
62
+ """
63
+ Return a run id for any unfinished run started in the last minute.
64
+ """
65
+ query = f"""
66
+ SELECT run_id
67
+ FROM {self.ch_runs_table} runs
68
+ WHERE event_type = 'start'
69
+ AND run_id NOT IN (
70
+ SELECT run_id
71
+ FROM {self.ch_runs_table}
72
+ WHERE event_type = 'end'
73
+ )
74
+ AND dateDiff('second', now(), timestamp) <= 60
75
+ """
76
+
77
+ response = requests.post(
78
+ url=self.ch_url,
79
+ auth=self.ch_auth,
80
+ params={"database": self.ch_database, "query": query},
81
+ timeout=self.ch_timeout_secs,
82
+ )
83
+
84
+ response.raise_for_status()
85
+
86
+ # ClickHouse will respond with the ID and a newline or an empty string
87
+ return response.text.strip()
88
+
89
+ def wait_for_start(self) -> None:
90
+ """
91
+ Wait for a run to start, tested by checking ClickHouse for new run ids.
92
+ """
93
+ while True:
94
+ self.run_id = self.check_for_run_id()
95
+
96
+ if self.run_id:
97
+ log.info(f"Found run id {self.run_id}! Starting monitor.")
98
+ return
99
+
100
+ log.info("No run id from the last 60 seconds found...") # pragma: no cover
101
+ sleep(2) # pragma: no cover
102
+
103
+ def test_has_ended(self) -> bool:
104
+ """
105
+ Return True if the current run has finished.
106
+ """
107
+ query = f"""
108
+ SELECT run_id
109
+ FROM {self.ch_runs_table} runs
110
+ WHERE run_id = '{self.run_id}'
111
+ AND event_type = 'end'
112
+ """
113
+
114
+ response = requests.post(
115
+ url=self.ch_url,
116
+ auth=self.ch_auth,
117
+ params={"database": self.ch_database, "query": query},
118
+ timeout=self.ch_timeout_secs,
119
+ )
120
+
121
+ response.raise_for_status()
122
+
123
+ # ClickHouse will respond with the ID and a newline or an empty string
124
+ return response.text.strip() == self.run_id
125
+
126
+ def run(self) -> None:
127
+ """
128
+ Wait for a new test to start, then run the monitor until killed.
129
+ """
130
+ collect_redis_bus = self.backend == "redis_bus"
131
+ collect_celery = self.backend == "celery"
132
+ collect_kafka_bus = self.backend == "kafka_bus"
133
+ collect_vector = self.backend == "vector"
134
+
135
+ log.info("Waiting for test to start...")
136
+ self.wait_for_start()
137
+
138
+ # Once the test has ended we will wait for the backend to drain the
139
+ # backlog before exiting. This tracks the "in between" state.
140
+ shutting_down = False
141
+
142
+ while True:
143
+ start = datetime.datetime.now()
144
+ log.info(f"----------- {start} --------")
145
+
146
+ current_stats = {"clickhouse": self.get_clickhouse_stats()}
147
+ if collect_redis_bus:
148
+ current_stats["redis_bus"] = self.get_redis_bus_stats()
149
+ lag = current_stats["redis_bus"]["lag"]
150
+ elif collect_celery:
151
+ current_stats["celery"] = self.get_celery_stats()
152
+ lag = current_stats["celery"]["lag"]
153
+ elif collect_kafka_bus:
154
+ current_stats["kafka_bus"] = self.get_kafka_bus_stats()
155
+ lag = current_stats["kafka_bus"]["lag"]
156
+ elif collect_vector:
157
+ current_stats["vector"] = self.get_vector_stats()
158
+ lag = current_stats["vector"]["lag"]
159
+
160
+ self.store_stats(current_stats)
161
+
162
+ if not shutting_down and self.test_has_ended(): # pragma: no cover
163
+ shutting_down = True
164
+ log.info(
165
+ "----- Test has ended, waiting for events to drain off or force end with CTRL-C"
166
+ )
167
+
168
+ if shutting_down and not lag:
169
+ log.info("----- Test has ended, events are drained. Ending monitor!")
170
+ break
171
+
172
+ # Try to keep our collection cadence to exactly what was asked
173
+ # otherwise
174
+ check_duration = datetime.datetime.now() - start
175
+
176
+ if check_duration.total_seconds() >= self.sleep_time:
177
+ log.warning(
178
+ f"It took {check_duration} to collect and store stats, this is greater than the sleep time!"
179
+ )
180
+
181
+ next_sleep = self.sleep_time - check_duration.total_seconds()
182
+ sleep(next_sleep)
183
+
184
+ def store_stats(self, current_stats: dict) -> None:
185
+ """
186
+ Send the results for this iteration to ClickHouse.
187
+ """
188
+ stats = json.dumps(current_stats)
189
+
190
+ insert = f"""INSERT INTO {self.ch_stats_table} (run_id, stats) FORMAT CSV"""
191
+
192
+ output = io.StringIO()
193
+ writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
194
+ writer.writerow((self.run_id, stats))
195
+
196
+ response = requests.post(
197
+ url=self.ch_url,
198
+ auth=self.ch_auth,
199
+ params={"database": self.ch_database, "query": insert},
200
+ data=output.getvalue().encode("utf-8"),
201
+ timeout=self.ch_timeout_secs,
202
+ )
203
+
204
+ response.raise_for_status()
205
+
206
+ def get_clickhouse_stats(self):
207
+ """
208
+ Get the current state of ClickHouse for this iteration.
209
+ """
210
+ select = f"""
211
+ SELECT
212
+ count(*) as ttl_count,
213
+ max(emission_time) as most_recent,
214
+ date_diff('second', max(emission_time), now()) as lag_seconds
215
+ FROM {self.ch_xapi_table}
216
+ FINAL
217
+ FORMAT JSON
218
+ """
219
+
220
+ response = requests.post(
221
+ url=self.ch_url,
222
+ auth=self.ch_auth,
223
+ params={"database": self.ch_xapi_database, "query": select},
224
+ timeout=self.ch_timeout_secs,
225
+ )
226
+
227
+ response.raise_for_status()
228
+
229
+ resp = response.json()["data"][0]
230
+ log.info(f"Clickhouse lag seconds: {resp['lag_seconds']}")
231
+
232
+ return {
233
+ "total_rows": resp["ttl_count"],
234
+ "most_recent_event": resp["most_recent"],
235
+ "lag_seconds": resp["lag_seconds"],
236
+ }
237
+
238
+ def get_celery_stats(self):
239
+ """
240
+ Get the current state of Celery for this iteration.
241
+ """
242
+ r = redis.Redis.from_url(settings.BROKER_URL)
243
+ lms_queue = r.llen("edx.lms.core.default")
244
+ cms_queue = r.llen("edx.cms.core.default")
245
+
246
+ log.info(f"Celery queues: LMS {lms_queue}, CMS {cms_queue}")
247
+
248
+ return {
249
+ "lms_queue_length": lms_queue,
250
+ "cms_queue_length": cms_queue,
251
+ "lag": lms_queue + cms_queue,
252
+ }
253
+
254
+ def get_redis_bus_stats(self):
255
+ """
256
+ Get the current state of redis for this iteration.
257
+ """
258
+ r = redis.Redis.from_url(settings.EVENT_BUS_REDIS_CONNECTION_URL)
259
+ info = r.xinfo_stream("openedx-analytics", full=True)
260
+
261
+ lag = 0
262
+
263
+ try:
264
+ for g in info["groups"]:
265
+ lag += g["lag"]
266
+ # Older versions of redis don't have "lag".
267
+ except KeyError: # pragma: no cover
268
+ pass
269
+
270
+ consumer_stats = {
271
+ "total_events": info["length"],
272
+ "lag": lag,
273
+ }
274
+
275
+ log.info(f"Redis bus queue length: {consumer_stats['lag']}")
276
+
277
+ return consumer_stats
278
+
279
+ def get_kafka_bus_stats(self):
280
+ """
281
+ Get the current state of ClickHouse for this iteration.
282
+ """
283
+ if not confluent_kafka: # pragma: no cover
284
+ raise CommandError(
285
+ "Trying to monitor Kafka bus, but confluent_kafka is not installed"
286
+ )
287
+
288
+ brokers = settings.EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS
289
+ topic = f"{settings.EVENT_BUS_TOPIC_PREFIX}-analytics"
290
+ group = "analytics-service"
291
+
292
+ # This consumer will not join the group, but the group.id is required by
293
+ # committed() to know which group to get offsets for.
294
+ consumer = confluent_kafka.Consumer(
295
+ {"bootstrap.servers": brokers, "group.id": group}
296
+ )
297
+
298
+ # Get the topic's partitions
299
+ metadata = consumer.list_topics(topic, timeout=10)
300
+
301
+ if metadata.topics[topic].error is not None: # pragma: no cover
302
+ log.info(metadata.topics[topic].error)
303
+
304
+ partitions = [
305
+ confluent_kafka.TopicPartition(topic, p)
306
+ for p in metadata.topics[topic].partitions
307
+ ]
308
+ committed = consumer.committed(partitions, timeout=10)
309
+
310
+ consumer_stats = {
311
+ "topic": topic,
312
+ "partitions": [],
313
+ }
314
+
315
+ total_lag = 0
316
+
317
+ for partition in committed:
318
+ # Get the partitions low and high watermark offsets.
319
+ low, high = consumer.get_watermark_offsets(
320
+ partition, timeout=10, cached=False
321
+ )
322
+
323
+ if high < 0: # pragma: no cover
324
+ lag = 0
325
+ elif partition.offset < 0: # pragma: no cover
326
+ # No committed offset, show total message count as lag.
327
+ # The actual message count may be lower due to compaction
328
+ # and record deletions.
329
+ lag = high - low
330
+ else:
331
+ lag = high - partition.offset
332
+
333
+ log.info(f"{partition.topic} [{partition.partition}] Lag: {lag}")
334
+ total_lag += lag
335
+
336
+ consumer_stats["partitions"].append(
337
+ {
338
+ "partition": partition.partition,
339
+ "lag": lag,
340
+ }
341
+ )
342
+
343
+ consumer.close()
344
+ consumer_stats["lag"] = total_lag
345
+ return consumer_stats
346
+
347
+ def _call_vector_graphql(self):
348
+ """
349
+ Make the actual GraphQL call to the Vector API.
350
+ """
351
+ # These values are hard coded in tutor local, K8s changes TBD
352
+ url = "http://vector:8686/graphql"
353
+ query = """
354
+ {
355
+ sinks(filter:{componentId:{equals:"clickhouse_xapi"}}) {
356
+ edges {
357
+ node {
358
+ ...on Sink {
359
+ componentId,
360
+ componentType,
361
+ metrics {
362
+ receivedEventsTotal {timestamp, receivedEventsTotal},
363
+ sentEventsTotal {timestamp, sentEventsTotal}
364
+ }
365
+ }
366
+ }
367
+ }
368
+ }
369
+ }
370
+ """
371
+ r = requests.post(url, json={"query": query}, timeout=10)
372
+ r.raise_for_status()
373
+ return r.json()["data"]["sinks"]["edges"][0]["node"]["metrics"]
374
+
375
+ def get_vector_stats(self):
376
+ """
377
+ Get the current state of Vector for this iteration.
378
+ """
379
+ metrics = self._call_vector_graphql()
380
+
381
+ # These will be null until events start arriving
382
+ received = (
383
+ metrics["receivedEventsTotal"]["receivedEventsTotal"]
384
+ if metrics["receivedEventsTotal"]
385
+ else 0.0
386
+ )
387
+ sent = (
388
+ metrics["sentEventsTotal"]["sentEventsTotal"]
389
+ if metrics["sentEventsTotal"]
390
+ else 0.0
391
+ )
392
+
393
+ rtn = {"events_received": received, "events_sent": sent, "lag": received - sent}
394
+
395
+ log.info(
396
+ f"Vector received: {rtn['events_received']} sent: {rtn['events_sent']} lag: {rtn.get('lag')}"
397
+ )
398
+ return rtn
399
+
400
+
401
+ class Command(BaseCommand):
402
+ """
403
+ Dump objects to a ClickHouse instance.
404
+
405
+ Example:
406
+ tutor local run lms ./manage.py lms monitor_load_test_tracking --sleep_time 5 --backend redis_bus
407
+ """
408
+
409
+ help = dedent(__doc__).strip()
410
+
411
+ def add_arguments(self, parser: Any) -> None:
412
+ parser.add_argument(
413
+ "--sleep_time",
414
+ type=float,
415
+ default=10,
416
+ help="Fractional number of seconds to sleep between gathering data.",
417
+ )
418
+ parser.add_argument(
419
+ "--backend",
420
+ choices=["redis_bus", "kafka_bus", "celery", "vector"],
421
+ default="celery",
422
+ help="Backend used to send events to ClickHouse",
423
+ )
424
+
425
+ def handle(self, *_, **options):
426
+ """
427
+ Creates users and triggers events for them as configured above.
428
+ """
429
+ start = datetime.datetime.now()
430
+ log.info(
431
+ f"Starting monitor for {options['backend']} with sleep of {options['sleep_time']} seconds"
432
+ )
433
+
434
+ monitor = Monitor(options["sleep_time"], options["backend"])
435
+ monitor.run()
436
+
437
+ end = datetime.datetime.now()
438
+ log.info(f"Monitored from {start} to {end} (duration {end - start}).")
@@ -14,7 +14,7 @@ from xblock.reference.user_service import XBlockUser
14
14
 
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
- if settings.DEBUG:
17
+ if settings.DEBUG: # pragma: no cover
18
18
  os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
19
19
 
20
20
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: platform-plugin-aspects
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Aspects plugins for edx-platform
5
5
  Home-page: https://github.com/openedx/platform-plugin-aspects
6
6
  Author: Open edX Project
@@ -27,6 +27,7 @@ Requires-Dist: edx-opaque-keys
27
27
  Requires-Dist: edx-toggles
28
28
  Requires-Dist: openedx-atlas
29
29
  Requires-Dist: openedx-filters
30
+ Requires-Dist: redis
30
31
  Requires-Dist: requests
31
32
  Requires-Dist: superset-api-client
32
33
  Requires-Dist: web_fragments
@@ -149,6 +150,15 @@ Unreleased
149
150
 
150
151
  *
151
152
 
153
+ 0.5.0 - 2024-04-01
154
+ ******************
155
+
156
+ Added
157
+ =====
158
+
159
+ * Load testing and test monitoring scripts.
160
+
161
+
152
162
  0.4.0 - 2024-03-18
153
163
  ******************
154
164
 
@@ -23,6 +23,8 @@ platform_plugin_aspects/extensions/filters.py
23
23
  platform_plugin_aspects/management/__init__.py
24
24
  platform_plugin_aspects/management/commands/__init__.py
25
25
  platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py
26
+ platform_plugin_aspects/management/commands/load_test_tracking_events.py
27
+ platform_plugin_aspects/management/commands/monitor_load_test_tracking.py
26
28
  platform_plugin_aspects/settings/__init__.py
27
29
  platform_plugin_aspects/settings/common.py
28
30
  platform_plugin_aspects/settings/production.py
@@ -8,6 +8,7 @@ edx-opaque-keys
8
8
  edx-toggles
9
9
  openedx-atlas
10
10
  openedx-filters
11
+ redis
11
12
  requests
12
13
  superset-api-client
13
14
  web_fragments
@@ -9,6 +9,7 @@ web_fragments
9
9
  django_crum
10
10
  celery # Asynchronous task execution library
11
11
  Django # Web application framework
12
+ redis
12
13
  requests # HTTP request library
13
14
  edx-django-utils # Django utilities, we use caching and monitoring
14
15
  edx-opaque-keys # Parsing library for course and usage keys