platform-plugin-aspects 1.1.0__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {platform_plugin_aspects-1.1.0/platform_plugin_aspects.egg-info → platform_plugin_aspects-1.1.2}/PKG-INFO +114 -67
- platform_plugin_aspects-1.1.2/README.rst +283 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/__init__.py +1 -1
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/load_test_tracking_events.py +2 -4
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/common.py +4 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/production.py +4 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/utils.py +15 -18
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/views.py +8 -1
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/xblock.py +2 -4
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2/platform_plugin_aspects.egg-info}/PKG-INFO +114 -67
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/setup.py +1 -0
- platform_plugin_aspects-1.1.0/README.rst +0 -236
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/LICENSE +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/MANIFEST.in +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/apps.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/extensions/__init__.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/extensions/filters.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/__init__.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/__init__.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/__init__.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/signals.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/__init__.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/base_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/course_enrollment_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/course_overview_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/external_id_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/serializers.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/tag_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/user_profile_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/user_retire_sink.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/css/superset.css +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/html/superset.html +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/html/superset_student.html +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/embed_dashboard.js +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/install_required.js +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/superset.js +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/tasks.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/templates/instructor_dashboard/aspects.html +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/urls.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/waffle.py +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/SOURCES.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/dependency_links.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/entry_points.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/not-zip-safe +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/requires.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/top_level.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/requirements/base.in +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/requirements/constraints.txt +0 -0
- {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: platform-plugin-aspects
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: Aspects plugins for edx-platform
|
|
5
5
|
Home-page: https://github.com/openedx/platform-plugin-aspects
|
|
6
6
|
Author: Open edX Project
|
|
@@ -52,15 +52,15 @@ Platform Plugin Aspects
|
|
|
52
52
|
Purpose
|
|
53
53
|
*******
|
|
54
54
|
|
|
55
|
-
This repository holds various Aspects plugins for the Open edX platform including the
|
|
56
|
-
event sinks that move data from the LMS to ClickHouse and the
|
|
55
|
+
This repository holds various Aspects plugins for the Open edX platform, including the
|
|
56
|
+
event sinks that move data from the LMS to ClickHouse and the embedding of Superset
|
|
57
57
|
dashboards back into the platform.
|
|
58
58
|
|
|
59
59
|
Version Compatibility
|
|
60
60
|
*********************
|
|
61
61
|
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
- ``platform-plugin-aspects`` version 1.x: Nutmeg to Sumac
|
|
63
|
+
- ``platform-plugin-aspects`` version 2.x: Redwood (with in-context metrics turned off), or Sumac and above
|
|
64
64
|
|
|
65
65
|
Sinks
|
|
66
66
|
*****
|
|
@@ -71,40 +71,82 @@ events are emitted by the Open edX platform via `Open edX events`_ or Django sig
|
|
|
71
71
|
Available Sinks
|
|
72
72
|
===============
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
74
|
+
Below are the existing sink names and their corresponding object names (as needed for the
|
|
75
|
+
``dump_data_to_clickhouse`` command below.
|
|
76
|
+
|
|
77
|
+
- ``CourseOverviewSink`` - Listens for the ``COURSE_PUBLISHED`` event and stores the
|
|
78
|
+
course structure, including ordering data through ``XBlockSink`` in ClickHouse. Object name:
|
|
79
|
+
``course_overviews``
|
|
80
|
+
- ``ExternalIdSink`` - Listens for the ``post_save`` Django signal on the ``ExternalId``
|
|
81
|
+
model and stores the external id data in ClickHouse. This model stores the relationships
|
|
82
|
+
between users and their xAPI unique identifiers. Object name: ``external_id``
|
|
83
|
+
- ``UserProfile`` - Listens for the ``post_save`` Django signal on the ``UserProfile``
|
|
84
|
+
model and stores the user profile data in ClickHouse. Object name: ``user_profile``
|
|
85
|
+
- ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
|
|
86
|
+
the course enrollment data. Object name: ``course_enrollment``
|
|
87
|
+
- ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
|
|
88
|
+
remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
|
|
84
89
|
|
|
85
90
|
Commands
|
|
86
91
|
========
|
|
87
92
|
|
|
88
93
|
In addition to being an event listener, this package provides the following commands:
|
|
89
94
|
|
|
90
|
-
-
|
|
91
|
-
Allows bootstrapping a new data platform or backfilling lost or missing data.
|
|
95
|
+
- ``dump_data_to_clickhouse`` - This command allows bulk export of the data from the Sinks.
|
|
96
|
+
Allows bootstrapping a new data platform or backfilling lost or missing data. Each sink object
|
|
97
|
+
is dumped individually. For large dumps, you can use the ``--batch_size`` and ``--sleep_time`` to control
|
|
98
|
+
how much load is placed on your LMS / Studio servers. Examples:
|
|
92
99
|
|
|
93
|
-
|
|
100
|
+
Dump any courses that the system thinks are out of date (last publish time is newer than the
|
|
101
|
+
last dump time in ClickHouse):
|
|
94
102
|
|
|
95
|
-
|
|
103
|
+
.. code-block:: bash
|
|
104
|
+
|
|
105
|
+
python manage.py cms dump_data_to_clickhouse --object course_overviews
|
|
106
|
+
|
|
107
|
+
The ``force`` option will dump all objects, regardless of the data ClickHouse currently has
|
|
108
|
+
so this command will push all course data for all courses:
|
|
109
|
+
|
|
110
|
+
.. code-block:: bash
|
|
111
|
+
|
|
112
|
+
python manage.py cms dump_data_to_clickhouse --object course_overviews --force
|
|
113
|
+
|
|
114
|
+
These commands will dump the user data Aspects uses when PII is turned on:
|
|
115
|
+
|
|
116
|
+
.. code-block:: bash
|
|
117
|
+
|
|
118
|
+
python manage.py cms dump_data_to_clickhouse --object external_id
|
|
119
|
+
python manage.py cms dump_data_to_clickhouse --object user_profile
|
|
120
|
+
|
|
121
|
+
To reduce server load, this command will dump 1000 user profiles at a time, with a 5-second
|
|
122
|
+
sleep in between:
|
|
123
|
+
|
|
124
|
+
.. code-block:: bash
|
|
125
|
+
|
|
126
|
+
python manage.py cms dump_data_to_clickhouse --object user_profile --batch_size 1000 --sleep_time 5
|
|
127
|
+
|
|
128
|
+
There are many more options that can be used for different circumstances. Please refer to
|
|
129
|
+
the commands help for more information. There is also a Tutor command that wraps this, so
|
|
130
|
+
that you don't need to get shell on a container to execute this command. More information on
|
|
131
|
+
that can be found in the `Aspects backfill documentation`_.
|
|
132
|
+
|
|
133
|
+
- ``load_test_tracking_events`` - This command allows loading test tracking events into
|
|
96
134
|
ClickHouse. This is useful for testing the ClickHouse connection to measure the performance of the
|
|
97
|
-
different data pipelines such as Vector, Event Bus (Redis and Kafka), and Celery.
|
|
135
|
+
different data pipelines, such as Vector, Event Bus (Redis and Kafka), and Celery.
|
|
98
136
|
|
|
99
|
-
Do not use this command in production as it will generate a large amount of data
|
|
137
|
+
**Do not use this command in production**, as it will generate a large amount of data
|
|
100
138
|
and will slow down the system.
|
|
101
139
|
|
|
102
|
-
|
|
140
|
+
.. code-block:: bash
|
|
141
|
+
|
|
142
|
+
python manage.py cms load_test_tracking_events
|
|
103
143
|
|
|
104
|
-
-
|
|
144
|
+
- ``monitor_load_test_tracking`` - Monitors the load test tracking script and saves
|
|
105
145
|
output for later analysis.
|
|
106
146
|
|
|
107
|
-
|
|
147
|
+
.. code-block:: bash
|
|
148
|
+
|
|
149
|
+
python manage.py cms monitor_load_test_tracking
|
|
108
150
|
|
|
109
151
|
Instructor Dashboard Integration
|
|
110
152
|
================================
|
|
@@ -120,10 +162,10 @@ Please see the Open edX documentation for `guidance on Python development <https
|
|
|
120
162
|
Deploying
|
|
121
163
|
*********
|
|
122
164
|
|
|
123
|
-
The
|
|
165
|
+
The ``Platform Plugin Aspects`` component is a django plugin that doesn't
|
|
124
166
|
need independent deployment. Therefore, its setup is reasonably straightforward.
|
|
125
167
|
First, it needs to be added to your service requirements, and then it will be
|
|
126
|
-
installed alongside requirements of the service.
|
|
168
|
+
installed alongside the requirements of the service.
|
|
127
169
|
|
|
128
170
|
Configuration
|
|
129
171
|
*************
|
|
@@ -135,72 +177,79 @@ The Instructor Dashboard integration uses the `Open edX Filters`_. To learn more
|
|
|
135
177
|
the filters, see the `Open edX Filters`_ documentation. Make sure to configure the
|
|
136
178
|
superset pipeline into the filter as follows:
|
|
137
179
|
|
|
138
|
-
|
|
180
|
+
.. code-block:: python
|
|
139
181
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
182
|
+
OPEN_EDX_FILTERS_CONFIG = {
|
|
183
|
+
"org.openedx.learning.instructor.dashboard.render.started.v1": {
|
|
184
|
+
"fail_silently": False,
|
|
185
|
+
"pipeline": [
|
|
186
|
+
"platform_plugin_superset.extensions.filters.AddSupersetTab",
|
|
187
|
+
]
|
|
188
|
+
},
|
|
189
|
+
}
|
|
148
190
|
|
|
149
|
-
-
|
|
191
|
+
- ``SUPERSET_CONFIG`` - This setting is used to configure the Superset Embedded SDK.
|
|
150
192
|
The configuration is a dictionary that contains the following keys:
|
|
151
193
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
194
|
+
- ``internal_service_url`` - The URL of the Superset instance (useful in development, omit in production).
|
|
195
|
+
- ``service_url`` - The URL of the Superset instance.
|
|
196
|
+
- ``username`` - The username of the Superset user.
|
|
197
|
+
- ``password`` - The password of the Superset user.
|
|
156
198
|
|
|
157
|
-
-
|
|
199
|
+
- ``ASPECTS_INSTRUCTOR_DASHBOARDS`` - This setting is used to configure the dashboards
|
|
158
200
|
that will be displayed in the Instructor Dashboard. The configuration is a list of
|
|
159
201
|
dictionaries that contains the following keys:
|
|
160
202
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
is translated in `Aspects`_.
|
|
203
|
+
- ``name`` - The name of the dashboard.
|
|
204
|
+
- ``slug`` - The slug of the dashboard.
|
|
205
|
+
- ``uuid`` - The UUID of the dashboard.
|
|
206
|
+
- ``allow_translations`` - A boolean value that determines if the dashboard is translated in `Aspects`_.
|
|
166
207
|
|
|
167
|
-
-
|
|
208
|
+
- ``SUPERSET_EXTRA_FILTERS_FORMAT`` - This setting is used to configure the extra filters
|
|
168
209
|
that will be applied to the dashboards. The configuration is a list of strings that
|
|
169
210
|
can be formatted with the following variables:
|
|
170
211
|
|
|
171
|
-
|
|
172
|
-
|
|
212
|
+
- ``user`` - The user object.
|
|
213
|
+
- ``course`` - The course object.
|
|
173
214
|
|
|
174
|
-
-
|
|
215
|
+
- ``SUPERSET_DASHBOARD_LOCALES`` - This setting is used to configure the available locales
|
|
175
216
|
for the dashboards. The configuration is a list of supported locales by `Aspects`_.
|
|
176
217
|
|
|
218
|
+
- ``ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS`` - This setting turns on and off the in-context
|
|
219
|
+
metrics feature. It must be turned off in Open edX releases before Sumac, and when using
|
|
220
|
+
``tutor-contrib-aspects`` before v2.2.0, as those dashboards will not exist, causing errors in
|
|
221
|
+
the embedded Instructor Dashboards.
|
|
222
|
+
|
|
223
|
+
- ``ASPECTS_IN_CONTEXT_DASHBOARDS`` - This setting mirrors the ``ASPECTS_INSTRUCTOR_DASHBOARDS``, but
|
|
224
|
+
with additional keys used for filtering the boards to specific courses and blocks.
|
|
225
|
+
|
|
177
226
|
Event Sink Configuration
|
|
178
227
|
========================
|
|
179
228
|
|
|
180
|
-
-
|
|
229
|
+
- ``EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`` - This setting is used to configure the ClickHouse
|
|
181
230
|
connection. The configuration is a dictionary that contains the following keys:
|
|
182
231
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
232
|
+
- ``url`` - The host of the ClickHouse instance.
|
|
233
|
+
- ``database`` - The database name.
|
|
234
|
+
- ``username`` - The username of the ClickHouse user.
|
|
235
|
+
- ``password`` - The password of the ClickHouse user.
|
|
236
|
+
- ``timeout_secs`` - The timeout in seconds for the ClickHouse connection.
|
|
188
237
|
|
|
189
|
-
-
|
|
238
|
+
- ``EVENT_SINK_CLICKHOUSE_PII_MODELS`` - This setting is used to configure the models that
|
|
190
239
|
contain PII information. The configuration is a list of strings that contain the
|
|
191
240
|
table names where the PII information is stored.
|
|
192
241
|
|
|
193
|
-
-
|
|
242
|
+
- ``EVENT_SINK_CLICKHOUSE_MODEL_CONFIG`` - This setting is used to provide compatibility
|
|
194
243
|
with multiple Open edX models. The configuration is a dictionary that contains the
|
|
195
|
-
following a key per model that contains a dictionary with the following keys:
|
|
244
|
+
following: a key per model that contains a dictionary with the following keys:
|
|
196
245
|
|
|
197
|
-
|
|
198
|
-
|
|
246
|
+
- ``module`` - The module path of the model.
|
|
247
|
+
- ``model`` - The model class name.
|
|
199
248
|
|
|
200
249
|
Event Sinks are disabled by default. To enable them, you need to enable the following
|
|
201
|
-
waffle flag:
|
|
250
|
+
waffle flag: ``event_sink_clickhouse.{{model_name}}.enabled``, where model_name is the name
|
|
202
251
|
of the model that you want to enable. Or, you can enable them via settings by setting
|
|
203
|
-
|
|
252
|
+
``EVENT_SINK_CLICKHOUSE_{{model_name}}_ENABLED`` to ``True``.
|
|
204
253
|
|
|
205
254
|
|
|
206
255
|
Getting Help
|
|
@@ -209,17 +258,15 @@ Getting Help
|
|
|
209
258
|
Documentation
|
|
210
259
|
=============
|
|
211
260
|
|
|
212
|
-
|
|
261
|
+
Start by going through `the documentation`_. If you need more help see below.
|
|
213
262
|
|
|
214
263
|
.. _the documentation: https://docs.openedx.org/projects/platform-plugin-aspects
|
|
215
264
|
|
|
216
|
-
(TODO: `Set up documentation <https://openedx.atlassian.net/wiki/spaces/DOC/pages/21627535/Publish+Documentation+on+Read+the+Docs>`_)
|
|
217
|
-
|
|
218
265
|
More Help
|
|
219
266
|
=========
|
|
220
267
|
|
|
221
268
|
If you're having trouble, we have discussion forums at
|
|
222
|
-
https://discuss.openedx.org where you can connect with others in the
|
|
269
|
+
https://discuss.openedx.org, where you can connect with others in the
|
|
223
270
|
community.
|
|
224
271
|
|
|
225
272
|
Our real-time conversations are on Slack. You can request a `Slack
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
Platform Plugin Aspects
|
|
2
|
+
#######################
|
|
3
|
+
|
|
4
|
+
Purpose
|
|
5
|
+
*******
|
|
6
|
+
|
|
7
|
+
This repository holds various Aspects plugins for the Open edX platform, including the
|
|
8
|
+
event sinks that move data from the LMS to ClickHouse and the embedding of Superset
|
|
9
|
+
dashboards back into the platform.
|
|
10
|
+
|
|
11
|
+
Version Compatibility
|
|
12
|
+
*********************
|
|
13
|
+
|
|
14
|
+
- ``platform-plugin-aspects`` version 1.x: Nutmeg to Sumac
|
|
15
|
+
- ``platform-plugin-aspects`` version 2.x: Redwood (with in-context metrics turned off), or Sumac and above
|
|
16
|
+
|
|
17
|
+
Sinks
|
|
18
|
+
*****
|
|
19
|
+
|
|
20
|
+
Sinks are components that listen for events and store them in ClickHouse. The
|
|
21
|
+
events are emitted by the Open edX platform via `Open edX events`_ or Django signals.
|
|
22
|
+
|
|
23
|
+
Available Sinks
|
|
24
|
+
===============
|
|
25
|
+
|
|
26
|
+
Below are the existing sink names and their corresponding object names (as needed for the
|
|
27
|
+
``dump_data_to_clickhouse`` command below.
|
|
28
|
+
|
|
29
|
+
- ``CourseOverviewSink`` - Listens for the ``COURSE_PUBLISHED`` event and stores the
|
|
30
|
+
course structure, including ordering data through ``XBlockSink`` in ClickHouse. Object name:
|
|
31
|
+
``course_overviews``
|
|
32
|
+
- ``ExternalIdSink`` - Listens for the ``post_save`` Django signal on the ``ExternalId``
|
|
33
|
+
model and stores the external id data in ClickHouse. This model stores the relationships
|
|
34
|
+
between users and their xAPI unique identifiers. Object name: ``external_id``
|
|
35
|
+
- ``UserProfile`` - Listens for the ``post_save`` Django signal on the ``UserProfile``
|
|
36
|
+
model and stores the user profile data in ClickHouse. Object name: ``user_profile``
|
|
37
|
+
- ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
|
|
38
|
+
the course enrollment data. Object name: ``course_enrollment``
|
|
39
|
+
- ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
|
|
40
|
+
remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
|
|
41
|
+
|
|
42
|
+
Commands
|
|
43
|
+
========
|
|
44
|
+
|
|
45
|
+
In addition to being an event listener, this package provides the following commands:
|
|
46
|
+
|
|
47
|
+
- ``dump_data_to_clickhouse`` - This command allows bulk export of the data from the Sinks.
|
|
48
|
+
Allows bootstrapping a new data platform or backfilling lost or missing data. Each sink object
|
|
49
|
+
is dumped individually. For large dumps, you can use the ``--batch_size`` and ``--sleep_time`` to control
|
|
50
|
+
how much load is placed on your LMS / Studio servers. Examples:
|
|
51
|
+
|
|
52
|
+
Dump any courses that the system thinks are out of date (last publish time is newer than the
|
|
53
|
+
last dump time in ClickHouse):
|
|
54
|
+
|
|
55
|
+
.. code-block:: bash
|
|
56
|
+
|
|
57
|
+
python manage.py cms dump_data_to_clickhouse --object course_overviews
|
|
58
|
+
|
|
59
|
+
The ``force`` option will dump all objects, regardless of the data ClickHouse currently has
|
|
60
|
+
so this command will push all course data for all courses:
|
|
61
|
+
|
|
62
|
+
.. code-block:: bash
|
|
63
|
+
|
|
64
|
+
python manage.py cms dump_data_to_clickhouse --object course_overviews --force
|
|
65
|
+
|
|
66
|
+
These commands will dump the user data Aspects uses when PII is turned on:
|
|
67
|
+
|
|
68
|
+
.. code-block:: bash
|
|
69
|
+
|
|
70
|
+
python manage.py cms dump_data_to_clickhouse --object external_id
|
|
71
|
+
python manage.py cms dump_data_to_clickhouse --object user_profile
|
|
72
|
+
|
|
73
|
+
To reduce server load, this command will dump 1000 user profiles at a time, with a 5-second
|
|
74
|
+
sleep in between:
|
|
75
|
+
|
|
76
|
+
.. code-block:: bash
|
|
77
|
+
|
|
78
|
+
python manage.py cms dump_data_to_clickhouse --object user_profile --batch_size 1000 --sleep_time 5
|
|
79
|
+
|
|
80
|
+
There are many more options that can be used for different circumstances. Please refer to
|
|
81
|
+
the commands help for more information. There is also a Tutor command that wraps this, so
|
|
82
|
+
that you don't need to get shell on a container to execute this command. More information on
|
|
83
|
+
that can be found in the `Aspects backfill documentation`_.
|
|
84
|
+
|
|
85
|
+
- ``load_test_tracking_events`` - This command allows loading test tracking events into
|
|
86
|
+
ClickHouse. This is useful for testing the ClickHouse connection to measure the performance of the
|
|
87
|
+
different data pipelines, such as Vector, Event Bus (Redis and Kafka), and Celery.
|
|
88
|
+
|
|
89
|
+
**Do not use this command in production**, as it will generate a large amount of data
|
|
90
|
+
and will slow down the system.
|
|
91
|
+
|
|
92
|
+
.. code-block:: bash
|
|
93
|
+
|
|
94
|
+
python manage.py cms load_test_tracking_events
|
|
95
|
+
|
|
96
|
+
- ``monitor_load_test_tracking`` - Monitors the load test tracking script and saves
|
|
97
|
+
output for later analysis.
|
|
98
|
+
|
|
99
|
+
.. code-block:: bash
|
|
100
|
+
|
|
101
|
+
python manage.py cms monitor_load_test_tracking
|
|
102
|
+
|
|
103
|
+
Instructor Dashboard Integration
|
|
104
|
+
================================
|
|
105
|
+
|
|
106
|
+
Dashboards from `Aspects`_ are integrated into the Instructor Dashboard via `Superset Embedded SDK`_.
|
|
107
|
+
See `Configuration`_ for more details.
|
|
108
|
+
|
|
109
|
+
Getting Started with Development
|
|
110
|
+
********************************
|
|
111
|
+
|
|
112
|
+
Please see the Open edX documentation for `guidance on Python development <https://docs.openedx.org/en/latest/developers/how-tos/get-ready-for-python-dev.html>`_ in this repo.
|
|
113
|
+
|
|
114
|
+
Deploying
|
|
115
|
+
*********
|
|
116
|
+
|
|
117
|
+
The ``Platform Plugin Aspects`` component is a django plugin that doesn't
|
|
118
|
+
need independent deployment. Therefore, its setup is reasonably straightforward.
|
|
119
|
+
First, it needs to be added to your service requirements, and then it will be
|
|
120
|
+
installed alongside the requirements of the service.
|
|
121
|
+
|
|
122
|
+
Configuration
|
|
123
|
+
*************
|
|
124
|
+
|
|
125
|
+
Instructor Dashboard Configuration
|
|
126
|
+
==================================
|
|
127
|
+
|
|
128
|
+
The Instructor Dashboard integration uses the `Open edX Filters`_. To learn more about
|
|
129
|
+
the filters, see the `Open edX Filters`_ documentation. Make sure to configure the
|
|
130
|
+
superset pipeline into the filter as follows:
|
|
131
|
+
|
|
132
|
+
.. code-block:: python
|
|
133
|
+
|
|
134
|
+
OPEN_EDX_FILTERS_CONFIG = {
|
|
135
|
+
"org.openedx.learning.instructor.dashboard.render.started.v1": {
|
|
136
|
+
"fail_silently": False,
|
|
137
|
+
"pipeline": [
|
|
138
|
+
"platform_plugin_superset.extensions.filters.AddSupersetTab",
|
|
139
|
+
]
|
|
140
|
+
},
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
- ``SUPERSET_CONFIG`` - This setting is used to configure the Superset Embedded SDK.
|
|
144
|
+
The configuration is a dictionary that contains the following keys:
|
|
145
|
+
|
|
146
|
+
- ``internal_service_url`` - The URL of the Superset instance (useful in development, omit in production).
|
|
147
|
+
- ``service_url`` - The URL of the Superset instance.
|
|
148
|
+
- ``username`` - The username of the Superset user.
|
|
149
|
+
- ``password`` - The password of the Superset user.
|
|
150
|
+
|
|
151
|
+
- ``ASPECTS_INSTRUCTOR_DASHBOARDS`` - This setting is used to configure the dashboards
|
|
152
|
+
that will be displayed in the Instructor Dashboard. The configuration is a list of
|
|
153
|
+
dictionaries that contains the following keys:
|
|
154
|
+
|
|
155
|
+
- ``name`` - The name of the dashboard.
|
|
156
|
+
- ``slug`` - The slug of the dashboard.
|
|
157
|
+
- ``uuid`` - The UUID of the dashboard.
|
|
158
|
+
- ``allow_translations`` - A boolean value that determines if the dashboard is translated in `Aspects`_.
|
|
159
|
+
|
|
160
|
+
- ``SUPERSET_EXTRA_FILTERS_FORMAT`` - This setting is used to configure the extra filters
|
|
161
|
+
that will be applied to the dashboards. The configuration is a list of strings that
|
|
162
|
+
can be formatted with the following variables:
|
|
163
|
+
|
|
164
|
+
- ``user`` - The user object.
|
|
165
|
+
- ``course`` - The course object.
|
|
166
|
+
|
|
167
|
+
- ``SUPERSET_DASHBOARD_LOCALES`` - This setting is used to configure the available locales
|
|
168
|
+
for the dashboards. The configuration is a list of supported locales by `Aspects`_.
|
|
169
|
+
|
|
170
|
+
- ``ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS`` - This setting turns on and off the in-context
|
|
171
|
+
metrics feature. It must be turned off in Open edX releases before Sumac, and when using
|
|
172
|
+
``tutor-contrib-aspects`` before v2.2.0, as those dashboards will not exist, causing errors in
|
|
173
|
+
the embedded Instructor Dashboards.
|
|
174
|
+
|
|
175
|
+
- ``ASPECTS_IN_CONTEXT_DASHBOARDS`` - This setting mirrors the ``ASPECTS_INSTRUCTOR_DASHBOARDS``, but
|
|
176
|
+
with additional keys used for filtering the boards to specific courses and blocks.
|
|
177
|
+
|
|
178
|
+
Event Sink Configuration
|
|
179
|
+
========================
|
|
180
|
+
|
|
181
|
+
- ``EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`` - This setting is used to configure the ClickHouse
|
|
182
|
+
connection. The configuration is a dictionary that contains the following keys:
|
|
183
|
+
|
|
184
|
+
- ``url`` - The host of the ClickHouse instance.
|
|
185
|
+
- ``database`` - The database name.
|
|
186
|
+
- ``username`` - The username of the ClickHouse user.
|
|
187
|
+
- ``password`` - The password of the ClickHouse user.
|
|
188
|
+
- ``timeout_secs`` - The timeout in seconds for the ClickHouse connection.
|
|
189
|
+
|
|
190
|
+
- ``EVENT_SINK_CLICKHOUSE_PII_MODELS`` - This setting is used to configure the models that
|
|
191
|
+
contain PII information. The configuration is a list of strings that contain the
|
|
192
|
+
table names where the PII information is stored.
|
|
193
|
+
|
|
194
|
+
- ``EVENT_SINK_CLICKHOUSE_MODEL_CONFIG`` - This setting is used to provide compatibility
|
|
195
|
+
with multiple Open edX models. The configuration is a dictionary that contains the
|
|
196
|
+
following: a key per model that contains a dictionary with the following keys:
|
|
197
|
+
|
|
198
|
+
- ``module`` - The module path of the model.
|
|
199
|
+
- ``model`` - The model class name.
|
|
200
|
+
|
|
201
|
+
Event Sinks are disabled by default. To enable them, you need to enable the following
|
|
202
|
+
waffle flag: ``event_sink_clickhouse.{{model_name}}.enabled``, where model_name is the name
|
|
203
|
+
of the model that you want to enable. Or, you can enable them via settings by setting
|
|
204
|
+
``EVENT_SINK_CLICKHOUSE_{{model_name}}_ENABLED`` to ``True``.
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
Getting Help
|
|
208
|
+
************
|
|
209
|
+
|
|
210
|
+
Documentation
|
|
211
|
+
=============
|
|
212
|
+
|
|
213
|
+
Start by going through `the documentation`_. If you need more help see below.
|
|
214
|
+
|
|
215
|
+
.. _the documentation: https://docs.openedx.org/projects/platform-plugin-aspects
|
|
216
|
+
|
|
217
|
+
More Help
|
|
218
|
+
=========
|
|
219
|
+
|
|
220
|
+
If you're having trouble, we have discussion forums at
|
|
221
|
+
https://discuss.openedx.org, where you can connect with others in the
|
|
222
|
+
community.
|
|
223
|
+
|
|
224
|
+
Our real-time conversations are on Slack. You can request a `Slack
|
|
225
|
+
invitation`_, then join our `community Slack workspace`_.
|
|
226
|
+
|
|
227
|
+
For anything non-trivial, the best path is to open an issue in this
|
|
228
|
+
repository with as many details about the issue you are facing as you
|
|
229
|
+
can provide.
|
|
230
|
+
|
|
231
|
+
https://github.com/openedx/platform-plugin-aspects/issues
|
|
232
|
+
|
|
233
|
+
For more information about these options, see the `Getting Help <https://openedx.org/getting-help>`__ page.
|
|
234
|
+
|
|
235
|
+
.. _Slack invitation: https://openedx.org/slack
|
|
236
|
+
.. _community Slack workspace: https://openedx.slack.com/
|
|
237
|
+
|
|
238
|
+
License
|
|
239
|
+
*******
|
|
240
|
+
|
|
241
|
+
Please see `LICENSE.txt <LICENSE.txt>`_ for details.
|
|
242
|
+
|
|
243
|
+
Contributing
|
|
244
|
+
************
|
|
245
|
+
|
|
246
|
+
Contributions are very welcome.
|
|
247
|
+
Please read `How To Contribute <https://openedx.org/r/how-to-contribute>`_ for details.
|
|
248
|
+
|
|
249
|
+
This project is currently accepting all types of contributions, bug fixes,
|
|
250
|
+
security fixes, maintenance work, or new features. However, please make sure
|
|
251
|
+
to have a discussion about your new feature idea with the maintainers prior to
|
|
252
|
+
beginning development to maximize the chances of your change being accepted.
|
|
253
|
+
You can start a conversation by creating a new issue on this repo summarizing
|
|
254
|
+
your idea.
|
|
255
|
+
|
|
256
|
+
The Open edX Code of Conduct
|
|
257
|
+
****************************
|
|
258
|
+
|
|
259
|
+
All community members are expected to follow the `Open edX Code of Conduct`_.
|
|
260
|
+
|
|
261
|
+
.. _Open edX Code of Conduct: https://openedx.org/code-of-conduct/
|
|
262
|
+
|
|
263
|
+
People
|
|
264
|
+
******
|
|
265
|
+
|
|
266
|
+
The assigned maintainers for this component and other project details may be
|
|
267
|
+
found in `Backstage`_. Backstage pulls this data from the ``catalog-info.yaml``
|
|
268
|
+
file in this repo.
|
|
269
|
+
|
|
270
|
+
.. _Backstage: https://backstage.openedx.org/catalog/default/component/platform-plugin-aspects
|
|
271
|
+
|
|
272
|
+
Reporting Security Issues
|
|
273
|
+
*************************
|
|
274
|
+
|
|
275
|
+
Please do not report security issues in public. Please email security@openedx.org.
|
|
276
|
+
|
|
277
|
+
.. _Open edX events: https://github.com/openedx/openedx-events
|
|
278
|
+
.. _Edx Platform: https://github.com/openedx/edx-platform
|
|
279
|
+
.. _ClickHouse: https://clickhouse.com
|
|
280
|
+
.. _Aspects: https://docs.openedx.org/projects/openedx-aspects/en/latest/index.html
|
|
281
|
+
.. _Superset Embedded SDK: https://www.npmjs.com/package/@superset-ui/embedded-sdk
|
|
282
|
+
.. _Open edX Filters: https://docs.openedx.org/projects/openedx-filters/en/latest/
|
|
283
|
+
.. _Aspects backfill documentation: https://docs.openedx.org/projects/openedx-aspects/en/latest/technical_documentation/how-tos/backfill.html
|
|
@@ -99,15 +99,13 @@ class LoadTest:
|
|
|
99
99
|
"display_name": f"Course {self.course_shortname}",
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
-
log.info(
|
|
103
|
-
f"""Creating course:
|
|
102
|
+
log.info(f"""Creating course:
|
|
104
103
|
Instructor: {self.instructor.id}
|
|
105
104
|
Org: "OEX"
|
|
106
105
|
Number: "{self.course_shortname}"
|
|
107
106
|
Run: "2024-1"
|
|
108
107
|
Fields: {fields}
|
|
109
|
-
"""
|
|
110
|
-
)
|
|
108
|
+
""")
|
|
111
109
|
|
|
112
110
|
self.course = create_new_course_in_store(
|
|
113
111
|
ModuleStoreEnum.Type.split,
|
|
@@ -105,6 +105,10 @@ def plugin_settings(settings):
|
|
|
105
105
|
},
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
# This is on by default here, off by default in tutor-contrib-aspects until we only
|
|
109
|
+
# support versions Sumac and beyond
|
|
110
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = True
|
|
111
|
+
|
|
108
112
|
settings.ASPECTS_IN_CONTEXT_DASHBOARDS = {
|
|
109
113
|
"course": {
|
|
110
114
|
"name": _("Course"),
|
|
@@ -28,6 +28,10 @@ def plugin_settings(settings):
|
|
|
28
28
|
"EVENT_SINK_CLICKHOUSE_PII_MODELS",
|
|
29
29
|
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS,
|
|
30
30
|
)
|
|
31
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = settings.ENV_TOKENS.get(
|
|
32
|
+
"ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS",
|
|
33
|
+
settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS,
|
|
34
|
+
)
|
|
31
35
|
settings.ASPECTS_IN_CONTEXT_DASHBOARDS = settings.ENV_TOKENS.get(
|
|
32
36
|
"ASPECTS_IN_CONTEXT_DASHBOARDS",
|
|
33
37
|
settings.ASPECTS_IN_CONTEXT_DASHBOARDS,
|
{platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/utils.py
RENAMED
|
@@ -155,26 +155,23 @@ def generate_guest_token(user, course, dashboards, filters) -> str:
|
|
|
155
155
|
token = response.json().get("token")
|
|
156
156
|
return token
|
|
157
157
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
) from err
|
|
158
|
+
# We manually log the error message here because ImproperlyConfigured is eaten up the stack
|
|
159
|
+
except HTTPError as exc:
|
|
160
|
+
err = f"""Unable to fetch Superset guest token, mostly likely due to invalid
|
|
161
|
+
settings.SUPERSET_CONFIG or because one of the dashboard UUIDs requested does not
|
|
162
|
+
exist in Superset.\n
|
|
163
|
+
{exc.response.status_code} {exc.response.json()} for url:
|
|
164
|
+
{exc.response.url}, data: {data}\n\n dashboards: {dashboards}"""
|
|
165
|
+
|
|
166
|
+
logger.error(err)
|
|
167
|
+
raise ImproperlyConfigured(err) from exc
|
|
169
168
|
|
|
170
169
|
except Exception as exc:
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
)
|
|
177
|
-
) from exc
|
|
170
|
+
err = f"""Error trying to fetch Superset guest token, mostly likely due to invalid
|
|
171
|
+
settings.SUPERSET_CONFIG or because one of the dashboard UUIDs requested does not
|
|
172
|
+
exist in Superset. data: {data} \n\n exception: {exc}"""
|
|
173
|
+
logger.error(err)
|
|
174
|
+
raise ImproperlyConfigured(err) from exc
|
|
178
175
|
|
|
179
176
|
|
|
180
177
|
def _fix_service_url(url: str) -> str:
|