platform-plugin-aspects 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- platform_plugin_aspects/__init__.py +11 -0
- platform_plugin_aspects/apps.py +77 -0
- platform_plugin_aspects/conf/locale/babel_mako.cfg +10 -0
- platform_plugin_aspects/conf/locale/config.yaml +85 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +99 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +70 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/extensions/__init__.py +0 -0
- platform_plugin_aspects/extensions/filters.py +108 -0
- platform_plugin_aspects/management/__init__.py +0 -0
- platform_plugin_aspects/management/commands/__init__.py +0 -0
- platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +214 -0
- platform_plugin_aspects/management/commands/load_test_tracking_events.py +310 -0
- platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +440 -0
- platform_plugin_aspects/settings/__init__.py +0 -0
- platform_plugin_aspects/settings/common.py +147 -0
- platform_plugin_aspects/settings/production.py +38 -0
- platform_plugin_aspects/signals.py +305 -0
- platform_plugin_aspects/sinks/__init__.py +11 -0
- platform_plugin_aspects/sinks/base_sink.py +381 -0
- platform_plugin_aspects/sinks/course_enrollment_sink.py +20 -0
- platform_plugin_aspects/sinks/course_overview_sink.py +286 -0
- platform_plugin_aspects/sinks/external_id_sink.py +20 -0
- platform_plugin_aspects/sinks/serializers.py +274 -0
- platform_plugin_aspects/sinks/tag_sink.py +47 -0
- platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
- platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
- platform_plugin_aspects/static/css/superset.css +53 -0
- platform_plugin_aspects/static/html/superset.html +39 -0
- platform_plugin_aspects/static/html/superset_student.html +5 -0
- platform_plugin_aspects/static/js/embed_dashboard.js +94 -0
- platform_plugin_aspects/static/js/install_required.js +11 -0
- platform_plugin_aspects/static/js/superset.js +38 -0
- platform_plugin_aspects/tasks.py +64 -0
- platform_plugin_aspects/templates/instructor_dashboard/aspects.html +22 -0
- platform_plugin_aspects/urls.py +39 -0
- platform_plugin_aspects/utils.py +359 -0
- platform_plugin_aspects/views.py +296 -0
- platform_plugin_aspects/waffle.py +5 -0
- platform_plugin_aspects/xblock.py +239 -0
- platform_plugin_aspects-2.0.1.dist-info/METADATA +335 -0
- platform_plugin_aspects-2.0.1.dist-info/RECORD +51 -0
- platform_plugin_aspects-2.0.1.dist-info/WHEEL +5 -0
- platform_plugin_aspects-2.0.1.dist-info/entry_points.txt +8 -0
- platform_plugin_aspects-2.0.1.dist-info/licenses/LICENSE +201 -0
- platform_plugin_aspects-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Management command for exporting the modulestore ClickHouse.
|
|
3
|
+
|
|
4
|
+
Example usages (see usage for more options):
|
|
5
|
+
|
|
6
|
+
# Dump all objects published since last dump.
|
|
7
|
+
# Use connection parameters from `settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`:
|
|
8
|
+
python manage.py cms dump_objects_to_clickhouse --object user_profile
|
|
9
|
+
|
|
10
|
+
# Specify certain objects instead of dumping all of them.
|
|
11
|
+
# Use connection parameters from `settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`.
|
|
12
|
+
python manage.py cms dump_objects_to_clickhouse --object user_profile --objects 123 124 125
|
|
13
|
+
|
|
14
|
+
# Dump a limited number of objects to prevent stress on production systems
|
|
15
|
+
python manage.py cms dump_objects_to_clickhouse --limit 1000
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
import time
|
|
20
|
+
from textwrap import dedent
|
|
21
|
+
|
|
22
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
23
|
+
|
|
24
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def dump_target_objects_to_clickhouse(
|
|
30
|
+
sink=None,
|
|
31
|
+
start_pk=None,
|
|
32
|
+
object_ids=None,
|
|
33
|
+
objects_to_skip=None,
|
|
34
|
+
force=False,
|
|
35
|
+
limit=None,
|
|
36
|
+
batch_size=1000,
|
|
37
|
+
sleep_time=10,
|
|
38
|
+
):
|
|
39
|
+
"""
|
|
40
|
+
Iterates through a list of objects in the ORN, serializes them to csv,
|
|
41
|
+
then submits tasks to post them to ClickHouse.
|
|
42
|
+
|
|
43
|
+
Arguments:
|
|
44
|
+
force: serialize the objects even if they've been recently
|
|
45
|
+
serialized
|
|
46
|
+
|
|
47
|
+
Returns: two lists--one of the objects that had dump jobs queued for them
|
|
48
|
+
and one of objects that did not.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
count = 0
|
|
52
|
+
skipped_objects = []
|
|
53
|
+
objects_to_submit = []
|
|
54
|
+
|
|
55
|
+
for obj, should_be_dumped, reason in sink.fetch_target_items(
|
|
56
|
+
start_pk, object_ids, objects_to_skip, force, batch_size
|
|
57
|
+
):
|
|
58
|
+
if not should_be_dumped:
|
|
59
|
+
skipped_objects.append(obj.pk)
|
|
60
|
+
log.info(f"{sink.model}: Skipping object {obj.pk}, reason: '{reason}'")
|
|
61
|
+
else:
|
|
62
|
+
objects_to_submit.append(obj)
|
|
63
|
+
if len(objects_to_submit) % batch_size == 0:
|
|
64
|
+
count += len(objects_to_submit)
|
|
65
|
+
sink.dump(objects_to_submit, many=True)
|
|
66
|
+
objects_to_submit = []
|
|
67
|
+
log.info(f"Last ID: {obj.pk}")
|
|
68
|
+
time.sleep(sleep_time)
|
|
69
|
+
|
|
70
|
+
if limit and count == limit:
|
|
71
|
+
log.info(
|
|
72
|
+
f"Limit of {limit} eligible objects has been reached, quitting!"
|
|
73
|
+
)
|
|
74
|
+
break
|
|
75
|
+
|
|
76
|
+
if objects_to_submit:
|
|
77
|
+
sink.dump(objects_to_submit, many=True)
|
|
78
|
+
count += len(objects_to_submit)
|
|
79
|
+
log.info(f"Last ID: {objects_to_submit[-1].pk}")
|
|
80
|
+
|
|
81
|
+
log.info(f"Dumped {count} objects to ClickHouse")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Command(BaseCommand):
|
|
85
|
+
"""
|
|
86
|
+
Dump objects to a ClickHouse instance.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
help = dedent(__doc__).strip()
|
|
90
|
+
|
|
91
|
+
def add_arguments(self, parser):
|
|
92
|
+
parser.add_argument(
|
|
93
|
+
"--url",
|
|
94
|
+
type=str,
|
|
95
|
+
help="the URL of the ClickHouse server",
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"--username",
|
|
99
|
+
type=str,
|
|
100
|
+
help="the username of the ClickHouse user",
|
|
101
|
+
)
|
|
102
|
+
parser.add_argument(
|
|
103
|
+
"--password",
|
|
104
|
+
type=str,
|
|
105
|
+
help="the password of the ClickHouse user",
|
|
106
|
+
)
|
|
107
|
+
parser.add_argument(
|
|
108
|
+
"--database",
|
|
109
|
+
type=str,
|
|
110
|
+
help="the database in ClickHouse to connect to",
|
|
111
|
+
)
|
|
112
|
+
parser.add_argument(
|
|
113
|
+
"--timeout_secs",
|
|
114
|
+
type=int,
|
|
115
|
+
help="timeout for ClickHouse requests, in seconds",
|
|
116
|
+
)
|
|
117
|
+
parser.add_argument(
|
|
118
|
+
"--object",
|
|
119
|
+
type=str,
|
|
120
|
+
help="the type of object to dump",
|
|
121
|
+
)
|
|
122
|
+
parser.add_argument(
|
|
123
|
+
"--start_pk",
|
|
124
|
+
type=int,
|
|
125
|
+
help="the primary key to start at",
|
|
126
|
+
default=None,
|
|
127
|
+
)
|
|
128
|
+
parser.add_argument(
|
|
129
|
+
"--ids",
|
|
130
|
+
metavar="KEY",
|
|
131
|
+
type=str,
|
|
132
|
+
nargs="*",
|
|
133
|
+
help="keys of objects to serialize; if omitted all objects in system are serialized",
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument(
|
|
136
|
+
"--ids_to_skip",
|
|
137
|
+
metavar="KEY",
|
|
138
|
+
type=str,
|
|
139
|
+
nargs="*",
|
|
140
|
+
help="keys of objects to NOT to serialize",
|
|
141
|
+
)
|
|
142
|
+
parser.add_argument(
|
|
143
|
+
"--force",
|
|
144
|
+
action="store_true",
|
|
145
|
+
help="dump all objects regardless of when they were last published",
|
|
146
|
+
)
|
|
147
|
+
parser.add_argument(
|
|
148
|
+
"--limit",
|
|
149
|
+
type=int,
|
|
150
|
+
help="maximum number of objects to dump, cannot be used with '--ids' or '--force'",
|
|
151
|
+
)
|
|
152
|
+
parser.add_argument(
|
|
153
|
+
"--batch_size",
|
|
154
|
+
type=int,
|
|
155
|
+
default=10000,
|
|
156
|
+
help="number of objects to dump in a single batch",
|
|
157
|
+
)
|
|
158
|
+
parser.add_argument(
|
|
159
|
+
"--sleep_time",
|
|
160
|
+
type=int,
|
|
161
|
+
default=1,
|
|
162
|
+
help="number of seconds to sleep between batches",
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
def handle(self, *args, **options):
|
|
166
|
+
"""
|
|
167
|
+
Iterates through each objects, serializes and saves them to clickhouse.
|
|
168
|
+
"""
|
|
169
|
+
connection_overrides = {
|
|
170
|
+
key: options[key]
|
|
171
|
+
for key in ["url", "username", "password", "database", "timeout_secs"]
|
|
172
|
+
if options[key]
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
ids = options["ids"] if options["ids"] else []
|
|
176
|
+
ids_to_skip = options["ids_to_skip"] if options["ids_to_skip"] else []
|
|
177
|
+
|
|
178
|
+
if options["limit"] is not None and int(options["limit"]) < 1:
|
|
179
|
+
message = "'limit' must be greater than 0!"
|
|
180
|
+
log.error(message)
|
|
181
|
+
raise CommandError(message)
|
|
182
|
+
|
|
183
|
+
if options["limit"] and options["force"]:
|
|
184
|
+
message = (
|
|
185
|
+
"The 'limit' option cannot be used with 'force' as running the "
|
|
186
|
+
"command repeatedly will result in the same objects being dumped every time."
|
|
187
|
+
)
|
|
188
|
+
log.error(message)
|
|
189
|
+
raise CommandError(message)
|
|
190
|
+
|
|
191
|
+
if options["object"] is None:
|
|
192
|
+
message = "You must specify an object type to dump!"
|
|
193
|
+
log.error(message)
|
|
194
|
+
raise CommandError(message)
|
|
195
|
+
|
|
196
|
+
Sink = ModelBaseSink.get_sink_by_model_name(options["object"])
|
|
197
|
+
|
|
198
|
+
if not Sink.is_enabled():
|
|
199
|
+
log.info(
|
|
200
|
+
f"Sink for {options['object']} is disabled, no data will be dumped."
|
|
201
|
+
)
|
|
202
|
+
return
|
|
203
|
+
|
|
204
|
+
sink = Sink(connection_overrides, log)
|
|
205
|
+
dump_target_objects_to_clickhouse(
|
|
206
|
+
sink,
|
|
207
|
+
options["start_pk"],
|
|
208
|
+
[object_id.strip() for object_id in ids],
|
|
209
|
+
[object_id.strip() for object_id in ids_to_skip],
|
|
210
|
+
options["force"],
|
|
211
|
+
options["limit"],
|
|
212
|
+
options["batch_size"],
|
|
213
|
+
options["sleep_time"],
|
|
214
|
+
)
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generates tracking events by creating test users and fake activity.
|
|
3
|
+
|
|
4
|
+
This should never be run on a production server as it will generate a lot of
|
|
5
|
+
bad data. It is entirely for benchmarking purposes in load test environments.
|
|
6
|
+
It is also fragile due to reaching into the edx-platform testing internals.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import csv
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import uuid
|
|
14
|
+
from datetime import datetime, timedelta
|
|
15
|
+
from random import choice
|
|
16
|
+
from textwrap import dedent
|
|
17
|
+
from time import sleep
|
|
18
|
+
from typing import Any, List
|
|
19
|
+
|
|
20
|
+
import requests
|
|
21
|
+
from django.conf import settings
|
|
22
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
23
|
+
|
|
24
|
+
from platform_plugin_aspects.sinks.base_sink import ClickHouseAuth
|
|
25
|
+
|
|
26
|
+
# For testing we won't be able to import from edx-platform
|
|
27
|
+
try: # pragma: no cover
|
|
28
|
+
from cms.djangoapps.contentstore.views.course import create_new_course_in_store
|
|
29
|
+
from common.djangoapps.student.helpers import do_create_account
|
|
30
|
+
from common.djangoapps.student.models.course_enrollment import CourseEnrollment
|
|
31
|
+
from openedx.core.djangoapps.user_authn.views.registration_form import (
|
|
32
|
+
AccountCreationForm,
|
|
33
|
+
)
|
|
34
|
+
from xmodule.modulestore import ModuleStoreEnum
|
|
35
|
+
|
|
36
|
+
RUNNING_IN_PLATFORM = True
|
|
37
|
+
except ImportError:
|
|
38
|
+
create_new_course_in_store = None
|
|
39
|
+
do_create_account = None
|
|
40
|
+
CourseEnrollment = None
|
|
41
|
+
AccountCreationForm = None
|
|
42
|
+
ModuleStoreEnum = None
|
|
43
|
+
|
|
44
|
+
RUNNING_IN_PLATFORM = False
|
|
45
|
+
|
|
46
|
+
log = logging.getLogger("tracking_event_loadtest")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class LoadTest:
|
|
50
|
+
"""
|
|
51
|
+
Runs the load test and reports results to ClickHouse.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
users = []
|
|
55
|
+
sent_event_count = 0
|
|
56
|
+
ch_runs_table = "load_test_runs"
|
|
57
|
+
|
|
58
|
+
def __init__(self, num_users: int, username_prefix: str, tags: List[str]):
|
|
59
|
+
self.num_users = num_users
|
|
60
|
+
self.username_prefix = username_prefix
|
|
61
|
+
self.tags = tags
|
|
62
|
+
self.run_id = str(uuid.uuid4())[:6]
|
|
63
|
+
|
|
64
|
+
self.course_shortname = str(uuid.uuid4())[:6]
|
|
65
|
+
|
|
66
|
+
self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
|
|
67
|
+
self.ch_auth = ClickHouseAuth(
|
|
68
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
|
|
69
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
|
|
70
|
+
)
|
|
71
|
+
self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
|
|
72
|
+
self.ch_xapi_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
73
|
+
"xapi_database", "xapi"
|
|
74
|
+
)
|
|
75
|
+
self.ch_runs_table = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG.get(
|
|
76
|
+
"runs_table", "load_test_runs"
|
|
77
|
+
)
|
|
78
|
+
self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
|
|
79
|
+
"timeout_secs"
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
self.instructor = self.create_user(
|
|
83
|
+
username=f"instructor_{self.course_shortname}",
|
|
84
|
+
name="Instructor",
|
|
85
|
+
password="aspects",
|
|
86
|
+
email=f"instructor_{self.course_shortname}@openedx.invalid",
|
|
87
|
+
)
|
|
88
|
+
self.create_course()
|
|
89
|
+
self.record_start()
|
|
90
|
+
self.create_and_enroll_learners(num_users, username_prefix)
|
|
91
|
+
|
|
92
|
+
def create_course(self) -> None:
|
|
93
|
+
"""
|
|
94
|
+
Create a course using the CMS API.
|
|
95
|
+
"""
|
|
96
|
+
start_date = datetime.now() - timedelta(days=7)
|
|
97
|
+
fields = {
|
|
98
|
+
"start": start_date,
|
|
99
|
+
"display_name": f"Course {self.course_shortname}",
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
log.info(f"""Creating course:
|
|
103
|
+
Instructor: {self.instructor.id}
|
|
104
|
+
Org: "OEX"
|
|
105
|
+
Number: "{self.course_shortname}"
|
|
106
|
+
Run: "2024-1"
|
|
107
|
+
Fields: {fields}
|
|
108
|
+
""")
|
|
109
|
+
|
|
110
|
+
self.course = create_new_course_in_store(
|
|
111
|
+
ModuleStoreEnum.Type.split,
|
|
112
|
+
self.instructor,
|
|
113
|
+
"OEX",
|
|
114
|
+
self.course_shortname,
|
|
115
|
+
"2024-1",
|
|
116
|
+
fields,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
log.info(f"Created course {self.course.id}")
|
|
120
|
+
|
|
121
|
+
def record_start(self) -> None:
|
|
122
|
+
"""
|
|
123
|
+
Send a start event to ClickHouse.
|
|
124
|
+
|
|
125
|
+
Start and end events are used by the monitor script to know when to
|
|
126
|
+
begin and end monitoring.
|
|
127
|
+
"""
|
|
128
|
+
self.record_to_clickhouse(
|
|
129
|
+
"start",
|
|
130
|
+
{
|
|
131
|
+
"tags": self.tags,
|
|
132
|
+
"course_id": str(self.course.id),
|
|
133
|
+
"num_users": self.num_users,
|
|
134
|
+
"username_prefix": self.username_prefix,
|
|
135
|
+
},
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Let the monitoring script connect, otherwise we can finish the test before it even
|
|
139
|
+
# knows we've started.
|
|
140
|
+
sleep(5)
|
|
141
|
+
|
|
142
|
+
def record_end(self) -> None:
|
|
143
|
+
"""
|
|
144
|
+
Send an end event to ClickHouse.
|
|
145
|
+
"""
|
|
146
|
+
self.record_to_clickhouse("end", {"sent_event_count": self.sent_event_count})
|
|
147
|
+
|
|
148
|
+
def record_to_clickhouse(self, event_type, extra) -> None:
|
|
149
|
+
"""
|
|
150
|
+
Send the run events to ClickHouse.
|
|
151
|
+
"""
|
|
152
|
+
insert = (
|
|
153
|
+
f"INSERT INTO {self.ch_runs_table} (run_id, event_type, extra) FORMAT CSV "
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
output = io.StringIO()
|
|
157
|
+
writer = csv.writer(output)
|
|
158
|
+
writer.writerow((self.run_id, event_type, json.dumps(extra)))
|
|
159
|
+
|
|
160
|
+
response = requests.post(
|
|
161
|
+
url=self.ch_url,
|
|
162
|
+
auth=self.ch_auth,
|
|
163
|
+
params={"database": self.ch_database, "query": insert},
|
|
164
|
+
data=output.getvalue().encode("utf-8"),
|
|
165
|
+
timeout=self.ch_timeout_secs,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
response.raise_for_status()
|
|
169
|
+
|
|
170
|
+
def create_and_enroll_learners(self, num_users, username_prefix):
|
|
171
|
+
"""
|
|
172
|
+
Uses create test users and enroll them in our test course.
|
|
173
|
+
"""
|
|
174
|
+
log.info(f"Creating {num_users} users prefixed with {username_prefix}.")
|
|
175
|
+
|
|
176
|
+
for _ in range(num_users):
|
|
177
|
+
user_short_name = str(uuid.uuid4())[:6]
|
|
178
|
+
u = self.create_user(
|
|
179
|
+
username=f"{username_prefix}_{user_short_name}",
|
|
180
|
+
name=f"Learner {user_short_name}",
|
|
181
|
+
password="aspects",
|
|
182
|
+
email=f"{user_short_name}@openedx.invalid",
|
|
183
|
+
)
|
|
184
|
+
self.users.append(u)
|
|
185
|
+
e = CourseEnrollment.get_or_create_enrollment(
|
|
186
|
+
user=u, course_key=self.course.id
|
|
187
|
+
)
|
|
188
|
+
e.is_active = True
|
|
189
|
+
e.save()
|
|
190
|
+
|
|
191
|
+
def create_user(self, **user_data):
|
|
192
|
+
"""
|
|
193
|
+
Create, activate, and return a user using the edx-platform API.
|
|
194
|
+
"""
|
|
195
|
+
account_creation_form = AccountCreationForm(data=user_data, tos_required=False)
|
|
196
|
+
|
|
197
|
+
user, _, _ = do_create_account(account_creation_form)
|
|
198
|
+
user.is_active = True
|
|
199
|
+
user.save()
|
|
200
|
+
return user
|
|
201
|
+
|
|
202
|
+
def trigger_events(
|
|
203
|
+
self, num_events: int, sleep_time: float, run_until_killed: bool
|
|
204
|
+
) -> None:
|
|
205
|
+
"""
|
|
206
|
+
Trigger the appropriate number of events based on configuration.
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
if run_until_killed:
|
|
210
|
+
log.info(f"Creating events until killed with {sleep_time} sleep between!")
|
|
211
|
+
while True:
|
|
212
|
+
self.trigger_event_and_sleep(sleep_time)
|
|
213
|
+
else:
|
|
214
|
+
log.info(f"Creating {num_events} event with {sleep_time} sleep between!")
|
|
215
|
+
for _ in range(num_events):
|
|
216
|
+
self.trigger_event_and_sleep(sleep_time)
|
|
217
|
+
|
|
218
|
+
def trigger_event_and_sleep(self, sleep_time: float) -> None:
|
|
219
|
+
"""
|
|
220
|
+
Cause a tracking log to be emitted and sleep the specified amount of time.
|
|
221
|
+
"""
|
|
222
|
+
user = choice(self.users)
|
|
223
|
+
|
|
224
|
+
e = CourseEnrollment.get_or_create_enrollment(
|
|
225
|
+
user=user, course_key=self.course.id
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
if e.is_active:
|
|
229
|
+
e.unenroll(user, self.course.id)
|
|
230
|
+
else:
|
|
231
|
+
e.enroll(user, self.course.id) # pragma: no cover
|
|
232
|
+
|
|
233
|
+
self.sent_event_count += 1
|
|
234
|
+
sleep(sleep_time)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class Command(BaseCommand):
|
|
238
|
+
"""
|
|
239
|
+
Create tracking log events for load testing purposes.
|
|
240
|
+
|
|
241
|
+
Example:
|
|
242
|
+
tutor local run lms ./manage.py lms load_test_tracking_events --sleep_time 0 --tags celery 1worker
|
|
243
|
+
"""
|
|
244
|
+
|
|
245
|
+
help = dedent(__doc__).strip()
|
|
246
|
+
|
|
247
|
+
def add_arguments(self, parser: Any) -> None:
|
|
248
|
+
parser.add_argument(
|
|
249
|
+
"--num_users",
|
|
250
|
+
type=int,
|
|
251
|
+
default=10,
|
|
252
|
+
help="The number of users to create. All events will be generated for these learners.",
|
|
253
|
+
)
|
|
254
|
+
parser.add_argument(
|
|
255
|
+
"--username_prefix",
|
|
256
|
+
type=str,
|
|
257
|
+
default="lt_",
|
|
258
|
+
help="Prefix for the generated user names.",
|
|
259
|
+
)
|
|
260
|
+
parser.add_argument(
|
|
261
|
+
"--num_events",
|
|
262
|
+
type=int,
|
|
263
|
+
default=10,
|
|
264
|
+
help="The number of events to generate. This is ignored if --run_until_killed is set.",
|
|
265
|
+
)
|
|
266
|
+
parser.add_argument(
|
|
267
|
+
"--run_until_killed",
|
|
268
|
+
action="store_true",
|
|
269
|
+
default=False,
|
|
270
|
+
help="If this is set, the process will run endlessly until killed.",
|
|
271
|
+
)
|
|
272
|
+
parser.add_argument(
|
|
273
|
+
"--sleep_time",
|
|
274
|
+
type=float,
|
|
275
|
+
default=0.75,
|
|
276
|
+
help="Fractional number of seconds to sleep between sending events.",
|
|
277
|
+
)
|
|
278
|
+
parser.add_argument(
|
|
279
|
+
"--tags",
|
|
280
|
+
nargs="*",
|
|
281
|
+
help="Tags to help define the run (ex: --tags celery 3workers k8s).",
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
def handle(self, *args, **options):
|
|
285
|
+
"""
|
|
286
|
+
Create users and trigger events for them as configured above.
|
|
287
|
+
"""
|
|
288
|
+
if not RUNNING_IN_PLATFORM: # pragma: no cover
|
|
289
|
+
raise CommandError("This command must be run in the Open edX LMS or CMS.")
|
|
290
|
+
|
|
291
|
+
start = datetime.now()
|
|
292
|
+
lt = LoadTest(options["num_users"], options["username_prefix"], options["tags"])
|
|
293
|
+
|
|
294
|
+
try:
|
|
295
|
+
lt.trigger_events(
|
|
296
|
+
options["num_events"],
|
|
297
|
+
options["sleep_time"],
|
|
298
|
+
options["run_until_killed"],
|
|
299
|
+
)
|
|
300
|
+
lt.record_end()
|
|
301
|
+
except KeyboardInterrupt:
|
|
302
|
+
log.warning("Killed by keyboard, finishing.")
|
|
303
|
+
lt.record_end()
|
|
304
|
+
|
|
305
|
+
end = datetime.now()
|
|
306
|
+
log.info(f"Sent {lt.sent_event_count} events in {end - start}.")
|
|
307
|
+
|
|
308
|
+
# Wait 5 seconds for Kafka to clear the queue otherwise not all events
|
|
309
|
+
# may be delivered.
|
|
310
|
+
sleep(5)
|