platform-plugin-aspects 1.1.0__tar.gz → 1.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {platform_plugin_aspects-1.1.0/platform_plugin_aspects.egg-info → platform_plugin_aspects-1.1.2}/PKG-INFO +114 -67
  2. platform_plugin_aspects-1.1.2/README.rst +283 -0
  3. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/__init__.py +1 -1
  4. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/load_test_tracking_events.py +2 -4
  5. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/common.py +4 -0
  6. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/production.py +4 -0
  7. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/utils.py +15 -18
  8. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/views.py +8 -1
  9. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/xblock.py +2 -4
  10. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2/platform_plugin_aspects.egg-info}/PKG-INFO +114 -67
  11. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/setup.py +1 -0
  12. platform_plugin_aspects-1.1.0/README.rst +0 -236
  13. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/LICENSE +0 -0
  14. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/MANIFEST.in +0 -0
  15. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/apps.py +0 -0
  16. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/extensions/__init__.py +0 -0
  17. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/extensions/filters.py +0 -0
  18. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/__init__.py +0 -0
  19. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/__init__.py +0 -0
  20. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +0 -0
  21. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +0 -0
  22. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/settings/__init__.py +0 -0
  23. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/signals.py +0 -0
  24. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/__init__.py +0 -0
  25. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/base_sink.py +0 -0
  26. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/course_enrollment_sink.py +0 -0
  27. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/course_overview_sink.py +0 -0
  28. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/external_id_sink.py +0 -0
  29. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/serializers.py +0 -0
  30. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/tag_sink.py +0 -0
  31. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/user_profile_sink.py +0 -0
  32. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/sinks/user_retire_sink.py +0 -0
  33. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/css/superset.css +0 -0
  34. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/html/superset.html +0 -0
  35. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/html/superset_student.html +0 -0
  36. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/embed_dashboard.js +0 -0
  37. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/install_required.js +0 -0
  38. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/static/js/superset.js +0 -0
  39. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/tasks.py +0 -0
  40. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/templates/instructor_dashboard/aspects.html +0 -0
  41. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/urls.py +0 -0
  42. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects/waffle.py +0 -0
  43. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/SOURCES.txt +0 -0
  44. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/dependency_links.txt +0 -0
  45. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/entry_points.txt +0 -0
  46. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/not-zip-safe +0 -0
  47. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/requires.txt +0 -0
  48. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/platform_plugin_aspects.egg-info/top_level.txt +0 -0
  49. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/requirements/base.in +0 -0
  50. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/requirements/constraints.txt +0 -0
  51. {platform_plugin_aspects-1.1.0 → platform_plugin_aspects-1.1.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: platform-plugin-aspects
3
- Version: 1.1.0
3
+ Version: 1.1.2
4
4
  Summary: Aspects plugins for edx-platform
5
5
  Home-page: https://github.com/openedx/platform-plugin-aspects
6
6
  Author: Open edX Project
@@ -52,15 +52,15 @@ Platform Plugin Aspects
52
52
  Purpose
53
53
  *******
54
54
 
55
- This repository holds various Aspects plugins for the Open edX platform including the
56
- event sinks that move data from the LMS to ClickHouse and the embbedding of Superset
55
+ This repository holds various Aspects plugins for the Open edX platform, including the
56
+ event sinks that move data from the LMS to ClickHouse and the embedding of Superset
57
57
  dashboards back into the platform.
58
58
 
59
59
  Version Compatibility
60
60
  *********************
61
61
 
62
- `platform-plugin-aspects` version 1.x: Nutmeg to Sumac
63
- `platform-plugin-aspects` version 2.x: Redwood and above
62
+ - ``platform-plugin-aspects`` version 1.x: Nutmeg to Sumac
63
+ - ``platform-plugin-aspects`` version 2.x: Redwood (with in-context metrics turned off), or Sumac and above
64
64
 
65
65
  Sinks
66
66
  *****
@@ -71,40 +71,82 @@ events are emitted by the Open edX platform via `Open edX events`_ or Django sig
71
71
  Available Sinks
72
72
  ===============
73
73
 
74
- - `CourseOverviewSink` - Listens for the `COURSE_PUBLISHED` event and stores the
75
- course structure data through `XBlockSink` in ClickHouse.
76
- - `ExternalIdSink` - Listens for the `post_save` Django signal on the `ExternalId`
77
- model and stores the external id data in ClickHouse.
78
- - `UserProfile` - Listens for the `post_save` Django signal on the `UserProfile`
79
- model and stores the user profile data in ClickHouse.
80
- - `UserRetirementSink` - Listen for the `USER_RETIRE_LMS_MISC` Django signal and
81
- remove the user PII information from ClickHouse.
82
- - `CourseEnrollmentSink` - Listen for the `ENROLL_STATUS_CHANGE` event and stores
83
- the course enrollment data.
74
+ Below are the existing sink names and their corresponding object names (as needed for the
75
+ ``dump_data_to_clickhouse`` command below.
76
+
77
+ - ``CourseOverviewSink`` - Listens for the ``COURSE_PUBLISHED`` event and stores the
78
+ course structure, including ordering data through ``XBlockSink`` in ClickHouse. Object name:
79
+ ``course_overviews``
80
+ - ``ExternalIdSink`` - Listens for the ``post_save`` Django signal on the ``ExternalId``
81
+ model and stores the external id data in ClickHouse. This model stores the relationships
82
+ between users and their xAPI unique identifiers. Object name: ``external_id``
83
+ - ``UserProfile`` - Listens for the ``post_save`` Django signal on the ``UserProfile``
84
+ model and stores the user profile data in ClickHouse. Object name: ``user_profile``
85
+ - ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
86
+ the course enrollment data. Object name: ``course_enrollment``
87
+ - ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
88
+ remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
84
89
 
85
90
  Commands
86
91
  ========
87
92
 
88
93
  In addition to being an event listener, this package provides the following commands:
89
94
 
90
- - `dump_data_to_clickhouse` - This command allows bulk export of the data from the Sinks.
91
- Allows bootstrapping a new data platform or backfilling lost or missing data. More information can be found in the `Aspects backfill documentation`_.
95
+ - ``dump_data_to_clickhouse`` - This command allows bulk export of the data from the Sinks.
96
+ Allows bootstrapping a new data platform or backfilling lost or missing data. Each sink object
97
+ is dumped individually. For large dumps, you can use the ``--batch_size`` and ``--sleep_time`` to control
98
+ how much load is placed on your LMS / Studio servers. Examples:
92
99
 
93
- ``python manage.py cms dump_data_to_clickhouse``
100
+ Dump any courses that the system thinks are out of date (last publish time is newer than the
101
+ last dump time in ClickHouse):
94
102
 
95
- - `load_test_tracking_events` - This command allows loading test tracking events into
103
+ .. code-block:: bash
104
+
105
+ python manage.py cms dump_data_to_clickhouse --object course_overviews
106
+
107
+ The ``force`` option will dump all objects, regardless of the data ClickHouse currently has
108
+ so this command will push all course data for all courses:
109
+
110
+ .. code-block:: bash
111
+
112
+ python manage.py cms dump_data_to_clickhouse --object course_overviews --force
113
+
114
+ These commands will dump the user data Aspects uses when PII is turned on:
115
+
116
+ .. code-block:: bash
117
+
118
+ python manage.py cms dump_data_to_clickhouse --object external_id
119
+ python manage.py cms dump_data_to_clickhouse --object user_profile
120
+
121
+ To reduce server load, this command will dump 1000 user profiles at a time, with a 5-second
122
+ sleep in between:
123
+
124
+ .. code-block:: bash
125
+
126
+ python manage.py cms dump_data_to_clickhouse --object user_profile --batch_size 1000 --sleep_time 5
127
+
128
+ There are many more options that can be used for different circumstances. Please refer to
129
+ the commands help for more information. There is also a Tutor command that wraps this, so
130
+ that you don't need to get shell on a container to execute this command. More information on
131
+ that can be found in the `Aspects backfill documentation`_.
132
+
133
+ - ``load_test_tracking_events`` - This command allows loading test tracking events into
96
134
  ClickHouse. This is useful for testing the ClickHouse connection to measure the performance of the
97
- different data pipelines such as Vector, Event Bus (Redis and Kafka), and Celery.
135
+ different data pipelines, such as Vector, Event Bus (Redis and Kafka), and Celery.
98
136
 
99
- Do not use this command in production as it will generate a large amount of data
137
+ **Do not use this command in production**, as it will generate a large amount of data
100
138
  and will slow down the system.
101
139
 
102
- ``python manage.py cms load_test_tracking_events``
140
+ .. code-block:: bash
141
+
142
+ python manage.py cms load_test_tracking_events
103
143
 
104
- - `monitor_load_test_tracking` - Monitors the load test tracking script and saves
144
+ - ``monitor_load_test_tracking`` - Monitors the load test tracking script and saves
105
145
  output for later analysis.
106
146
 
107
- ``python manage.py cms monitor_load_test_tracking``
147
+ .. code-block:: bash
148
+
149
+ python manage.py cms monitor_load_test_tracking
108
150
 
109
151
  Instructor Dashboard Integration
110
152
  ================================
@@ -120,10 +162,10 @@ Please see the Open edX documentation for `guidance on Python development <https
120
162
  Deploying
121
163
  *********
122
164
 
123
- The `Platform Plugin Aspects` component is a django plugin which doesn't
165
+ The ``Platform Plugin Aspects`` component is a django plugin that doesn't
124
166
  need independent deployment. Therefore, its setup is reasonably straightforward.
125
167
  First, it needs to be added to your service requirements, and then it will be
126
- installed alongside requirements of the service.
168
+ installed alongside the requirements of the service.
127
169
 
128
170
  Configuration
129
171
  *************
@@ -135,72 +177,79 @@ The Instructor Dashboard integration uses the `Open edX Filters`_. To learn more
135
177
  the filters, see the `Open edX Filters`_ documentation. Make sure to configure the
136
178
  superset pipeline into the filter as follows:
137
179
 
138
- .. code-block:: python
180
+ .. code-block:: python
139
181
 
140
- OPEN_EDX_FILTERS_CONFIG = {
141
- "org.openedx.learning.instructor.dashboard.render.started.v1": {
142
- "fail_silently": False,
143
- "pipeline": [
144
- "platform_plugin_superset.extensions.filters.AddSupersetTab",
145
- ]
146
- },
147
- }
182
+ OPEN_EDX_FILTERS_CONFIG = {
183
+ "org.openedx.learning.instructor.dashboard.render.started.v1": {
184
+ "fail_silently": False,
185
+ "pipeline": [
186
+ "platform_plugin_superset.extensions.filters.AddSupersetTab",
187
+ ]
188
+ },
189
+ }
148
190
 
149
- - `SUPERSET_CONFIG` - This setting is used to configure the Superset Embedded SDK.
191
+ - ``SUPERSET_CONFIG`` - This setting is used to configure the Superset Embedded SDK.
150
192
  The configuration is a dictionary that contains the following keys:
151
193
 
152
- - `internal_service_url` - The URL of the Superset instance (useful in development, omit in production).
153
- - `service_url` - The URL of the Superset instance.
154
- - `username` - The username of the Superset user.
155
- - `password` - The password of the Superset user.
194
+ - ``internal_service_url`` - The URL of the Superset instance (useful in development, omit in production).
195
+ - ``service_url`` - The URL of the Superset instance.
196
+ - ``username`` - The username of the Superset user.
197
+ - ``password`` - The password of the Superset user.
156
198
 
157
- - `ASPECTS_INSTRUCTOR_DASHBOARDS` - This setting is used to configure the dashboards
199
+ - ``ASPECTS_INSTRUCTOR_DASHBOARDS`` - This setting is used to configure the dashboards
158
200
  that will be displayed in the Instructor Dashboard. The configuration is a list of
159
201
  dictionaries that contains the following keys:
160
202
 
161
- - `name` - The name of the dashboard.
162
- - `slug` - The slug of the dashboard.
163
- - `uuid` - The UUID of the dashboard.
164
- - `allow_translations` - A boolean value that determines if the dashboard
165
- is translated in `Aspects`_.
203
+ - ``name`` - The name of the dashboard.
204
+ - ``slug`` - The slug of the dashboard.
205
+ - ``uuid`` - The UUID of the dashboard.
206
+ - ``allow_translations`` - A boolean value that determines if the dashboard is translated in `Aspects`_.
166
207
 
167
- - `SUPERSET_EXTRA_FILTERS_FORMAT` - This setting is used to configure the extra filters
208
+ - ``SUPERSET_EXTRA_FILTERS_FORMAT`` - This setting is used to configure the extra filters
168
209
  that will be applied to the dashboards. The configuration is a list of strings that
169
210
  can be formatted with the following variables:
170
211
 
171
- - `user` - The user object.
172
- - `course` - The course object.
212
+ - ``user`` - The user object.
213
+ - ``course`` - The course object.
173
214
 
174
- - `SUPERSET_DASHBOARD_LOCALES` - This setting is used to configure the available locales
215
+ - ``SUPERSET_DASHBOARD_LOCALES`` - This setting is used to configure the available locales
175
216
  for the dashboards. The configuration is a list of supported locales by `Aspects`_.
176
217
 
218
+ - ``ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS`` - This setting turns on and off the in-context
219
+ metrics feature. It must be turned off in Open edX releases before Sumac, and when using
220
+ ``tutor-contrib-aspects`` before v2.2.0, as those dashboards will not exist, causing errors in
221
+ the embedded Instructor Dashboards.
222
+
223
+ - ``ASPECTS_IN_CONTEXT_DASHBOARDS`` - This setting mirrors the ``ASPECTS_INSTRUCTOR_DASHBOARDS``, but
224
+ with additional keys used for filtering the boards to specific courses and blocks.
225
+
177
226
  Event Sink Configuration
178
227
  ========================
179
228
 
180
- - `EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG` - This setting is used to configure the ClickHouse
229
+ - ``EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`` - This setting is used to configure the ClickHouse
181
230
  connection. The configuration is a dictionary that contains the following keys:
182
231
 
183
- - `url` - The host of the ClickHouse instance.
184
- - `database` - The database name.
185
- - `username` - The username of the ClickHouse user.
186
- - `password` - The password of the ClickHouse user.
187
- - `timeout_secs` - The timeout in seconds for the ClickHouse connection.
232
+ - ``url`` - The host of the ClickHouse instance.
233
+ - ``database`` - The database name.
234
+ - ``username`` - The username of the ClickHouse user.
235
+ - ``password`` - The password of the ClickHouse user.
236
+ - ``timeout_secs`` - The timeout in seconds for the ClickHouse connection.
188
237
 
189
- - `EVENT_SINK_CLICKHOUSE_PII_MODELS` - This setting is used to configure the models that
238
+ - ``EVENT_SINK_CLICKHOUSE_PII_MODELS`` - This setting is used to configure the models that
190
239
  contain PII information. The configuration is a list of strings that contain the
191
240
  table names where the PII information is stored.
192
241
 
193
- - `EVENT_SINK_CLICKHOUSE_MODEL_CONFIG` - This setting is used to provide compatibility
242
+ - ``EVENT_SINK_CLICKHOUSE_MODEL_CONFIG`` - This setting is used to provide compatibility
194
243
  with multiple Open edX models. The configuration is a dictionary that contains the
195
- following a key per model that contains a dictionary with the following keys:
244
+ following: a key per model that contains a dictionary with the following keys:
196
245
 
197
- - `module` - The module path of the model.
198
- - `model` - The model class name.
246
+ - ``module`` - The module path of the model.
247
+ - ``model`` - The model class name.
199
248
 
200
249
  Event Sinks are disabled by default. To enable them, you need to enable the following
201
- waffle flag: `event_sink_clickhouse.{{model_name}}.enabled` where model name is the name
250
+ waffle flag: ``event_sink_clickhouse.{{model_name}}.enabled``, where model_name is the name
202
251
  of the model that you want to enable. Or, you can enable them via settings by setting
203
- `EVENT_SINK_CLICKHOUSE_{{model_name}}_ENABLED` to `True`.
252
+ ``EVENT_SINK_CLICKHOUSE_{{model_name}}_ENABLED`` to ``True``.
204
253
 
205
254
 
206
255
  Getting Help
@@ -209,17 +258,15 @@ Getting Help
209
258
  Documentation
210
259
  =============
211
260
 
212
- PLACEHOLDER: Start by going through `the documentation`_. If you need more help see below.
261
+ Start by going through `the documentation`_. If you need more help see below.
213
262
 
214
263
  .. _the documentation: https://docs.openedx.org/projects/platform-plugin-aspects
215
264
 
216
- (TODO: `Set up documentation <https://openedx.atlassian.net/wiki/spaces/DOC/pages/21627535/Publish+Documentation+on+Read+the+Docs>`_)
217
-
218
265
  More Help
219
266
  =========
220
267
 
221
268
  If you're having trouble, we have discussion forums at
222
- https://discuss.openedx.org where you can connect with others in the
269
+ https://discuss.openedx.org, where you can connect with others in the
223
270
  community.
224
271
 
225
272
  Our real-time conversations are on Slack. You can request a `Slack
@@ -0,0 +1,283 @@
1
+ Platform Plugin Aspects
2
+ #######################
3
+
4
+ Purpose
5
+ *******
6
+
7
+ This repository holds various Aspects plugins for the Open edX platform, including the
8
+ event sinks that move data from the LMS to ClickHouse and the embedding of Superset
9
+ dashboards back into the platform.
10
+
11
+ Version Compatibility
12
+ *********************
13
+
14
+ - ``platform-plugin-aspects`` version 1.x: Nutmeg to Sumac
15
+ - ``platform-plugin-aspects`` version 2.x: Redwood (with in-context metrics turned off), or Sumac and above
16
+
17
+ Sinks
18
+ *****
19
+
20
+ Sinks are components that listen for events and store them in ClickHouse. The
21
+ events are emitted by the Open edX platform via `Open edX events`_ or Django signals.
22
+
23
+ Available Sinks
24
+ ===============
25
+
26
+ Below are the existing sink names and their corresponding object names (as needed for the
27
+ ``dump_data_to_clickhouse`` command below.
28
+
29
+ - ``CourseOverviewSink`` - Listens for the ``COURSE_PUBLISHED`` event and stores the
30
+ course structure, including ordering data through ``XBlockSink`` in ClickHouse. Object name:
31
+ ``course_overviews``
32
+ - ``ExternalIdSink`` - Listens for the ``post_save`` Django signal on the ``ExternalId``
33
+ model and stores the external id data in ClickHouse. This model stores the relationships
34
+ between users and their xAPI unique identifiers. Object name: ``external_id``
35
+ - ``UserProfile`` - Listens for the ``post_save`` Django signal on the ``UserProfile``
36
+ model and stores the user profile data in ClickHouse. Object name: ``user_profile``
37
+ - ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
38
+ the course enrollment data. Object name: ``course_enrollment``
39
+ - ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
40
+ remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
41
+
42
+ Commands
43
+ ========
44
+
45
+ In addition to being an event listener, this package provides the following commands:
46
+
47
+ - ``dump_data_to_clickhouse`` - This command allows bulk export of the data from the Sinks.
48
+ Allows bootstrapping a new data platform or backfilling lost or missing data. Each sink object
49
+ is dumped individually. For large dumps, you can use the ``--batch_size`` and ``--sleep_time`` to control
50
+ how much load is placed on your LMS / Studio servers. Examples:
51
+
52
+ Dump any courses that the system thinks are out of date (last publish time is newer than the
53
+ last dump time in ClickHouse):
54
+
55
+ .. code-block:: bash
56
+
57
+ python manage.py cms dump_data_to_clickhouse --object course_overviews
58
+
59
+ The ``force`` option will dump all objects, regardless of the data ClickHouse currently has
60
+ so this command will push all course data for all courses:
61
+
62
+ .. code-block:: bash
63
+
64
+ python manage.py cms dump_data_to_clickhouse --object course_overviews --force
65
+
66
+ These commands will dump the user data Aspects uses when PII is turned on:
67
+
68
+ .. code-block:: bash
69
+
70
+ python manage.py cms dump_data_to_clickhouse --object external_id
71
+ python manage.py cms dump_data_to_clickhouse --object user_profile
72
+
73
+ To reduce server load, this command will dump 1000 user profiles at a time, with a 5-second
74
+ sleep in between:
75
+
76
+ .. code-block:: bash
77
+
78
+ python manage.py cms dump_data_to_clickhouse --object user_profile --batch_size 1000 --sleep_time 5
79
+
80
+ There are many more options that can be used for different circumstances. Please refer to
81
+ the commands help for more information. There is also a Tutor command that wraps this, so
82
+ that you don't need to get shell on a container to execute this command. More information on
83
+ that can be found in the `Aspects backfill documentation`_.
84
+
85
+ - ``load_test_tracking_events`` - This command allows loading test tracking events into
86
+ ClickHouse. This is useful for testing the ClickHouse connection to measure the performance of the
87
+ different data pipelines, such as Vector, Event Bus (Redis and Kafka), and Celery.
88
+
89
+ **Do not use this command in production**, as it will generate a large amount of data
90
+ and will slow down the system.
91
+
92
+ .. code-block:: bash
93
+
94
+ python manage.py cms load_test_tracking_events
95
+
96
+ - ``monitor_load_test_tracking`` - Monitors the load test tracking script and saves
97
+ output for later analysis.
98
+
99
+ .. code-block:: bash
100
+
101
+ python manage.py cms monitor_load_test_tracking
102
+
103
+ Instructor Dashboard Integration
104
+ ================================
105
+
106
+ Dashboards from `Aspects`_ are integrated into the Instructor Dashboard via `Superset Embedded SDK`_.
107
+ See `Configuration`_ for more details.
108
+
109
+ Getting Started with Development
110
+ ********************************
111
+
112
+ Please see the Open edX documentation for `guidance on Python development <https://docs.openedx.org/en/latest/developers/how-tos/get-ready-for-python-dev.html>`_ in this repo.
113
+
114
+ Deploying
115
+ *********
116
+
117
+ The ``Platform Plugin Aspects`` component is a django plugin that doesn't
118
+ need independent deployment. Therefore, its setup is reasonably straightforward.
119
+ First, it needs to be added to your service requirements, and then it will be
120
+ installed alongside the requirements of the service.
121
+
122
+ Configuration
123
+ *************
124
+
125
+ Instructor Dashboard Configuration
126
+ ==================================
127
+
128
+ The Instructor Dashboard integration uses the `Open edX Filters`_. To learn more about
129
+ the filters, see the `Open edX Filters`_ documentation. Make sure to configure the
130
+ superset pipeline into the filter as follows:
131
+
132
+ .. code-block:: python
133
+
134
+ OPEN_EDX_FILTERS_CONFIG = {
135
+ "org.openedx.learning.instructor.dashboard.render.started.v1": {
136
+ "fail_silently": False,
137
+ "pipeline": [
138
+ "platform_plugin_superset.extensions.filters.AddSupersetTab",
139
+ ]
140
+ },
141
+ }
142
+
143
+ - ``SUPERSET_CONFIG`` - This setting is used to configure the Superset Embedded SDK.
144
+ The configuration is a dictionary that contains the following keys:
145
+
146
+ - ``internal_service_url`` - The URL of the Superset instance (useful in development, omit in production).
147
+ - ``service_url`` - The URL of the Superset instance.
148
+ - ``username`` - The username of the Superset user.
149
+ - ``password`` - The password of the Superset user.
150
+
151
+ - ``ASPECTS_INSTRUCTOR_DASHBOARDS`` - This setting is used to configure the dashboards
152
+ that will be displayed in the Instructor Dashboard. The configuration is a list of
153
+ dictionaries that contains the following keys:
154
+
155
+ - ``name`` - The name of the dashboard.
156
+ - ``slug`` - The slug of the dashboard.
157
+ - ``uuid`` - The UUID of the dashboard.
158
+ - ``allow_translations`` - A boolean value that determines if the dashboard is translated in `Aspects`_.
159
+
160
+ - ``SUPERSET_EXTRA_FILTERS_FORMAT`` - This setting is used to configure the extra filters
161
+ that will be applied to the dashboards. The configuration is a list of strings that
162
+ can be formatted with the following variables:
163
+
164
+ - ``user`` - The user object.
165
+ - ``course`` - The course object.
166
+
167
+ - ``SUPERSET_DASHBOARD_LOCALES`` - This setting is used to configure the available locales
168
+ for the dashboards. The configuration is a list of supported locales by `Aspects`_.
169
+
170
+ - ``ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS`` - This setting turns on and off the in-context
171
+ metrics feature. It must be turned off in Open edX releases before Sumac, and when using
172
+ ``tutor-contrib-aspects`` before v2.2.0, as those dashboards will not exist, causing errors in
173
+ the embedded Instructor Dashboards.
174
+
175
+ - ``ASPECTS_IN_CONTEXT_DASHBOARDS`` - This setting mirrors the ``ASPECTS_INSTRUCTOR_DASHBOARDS``, but
176
+ with additional keys used for filtering the boards to specific courses and blocks.
177
+
178
+ Event Sink Configuration
179
+ ========================
180
+
181
+ - ``EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`` - This setting is used to configure the ClickHouse
182
+ connection. The configuration is a dictionary that contains the following keys:
183
+
184
+ - ``url`` - The host of the ClickHouse instance.
185
+ - ``database`` - The database name.
186
+ - ``username`` - The username of the ClickHouse user.
187
+ - ``password`` - The password of the ClickHouse user.
188
+ - ``timeout_secs`` - The timeout in seconds for the ClickHouse connection.
189
+
190
+ - ``EVENT_SINK_CLICKHOUSE_PII_MODELS`` - This setting is used to configure the models that
191
+ contain PII information. The configuration is a list of strings that contain the
192
+ table names where the PII information is stored.
193
+
194
+ - ``EVENT_SINK_CLICKHOUSE_MODEL_CONFIG`` - This setting is used to provide compatibility
195
+ with multiple Open edX models. The configuration is a dictionary that contains the
196
+ following: a key per model that contains a dictionary with the following keys:
197
+
198
+ - ``module`` - The module path of the model.
199
+ - ``model`` - The model class name.
200
+
201
+ Event Sinks are disabled by default. To enable them, you need to enable the following
202
+ waffle flag: ``event_sink_clickhouse.{{model_name}}.enabled``, where model_name is the name
203
+ of the model that you want to enable. Or, you can enable them via settings by setting
204
+ ``EVENT_SINK_CLICKHOUSE_{{model_name}}_ENABLED`` to ``True``.
205
+
206
+
207
+ Getting Help
208
+ ************
209
+
210
+ Documentation
211
+ =============
212
+
213
+ Start by going through `the documentation`_. If you need more help see below.
214
+
215
+ .. _the documentation: https://docs.openedx.org/projects/platform-plugin-aspects
216
+
217
+ More Help
218
+ =========
219
+
220
+ If you're having trouble, we have discussion forums at
221
+ https://discuss.openedx.org, where you can connect with others in the
222
+ community.
223
+
224
+ Our real-time conversations are on Slack. You can request a `Slack
225
+ invitation`_, then join our `community Slack workspace`_.
226
+
227
+ For anything non-trivial, the best path is to open an issue in this
228
+ repository with as many details about the issue you are facing as you
229
+ can provide.
230
+
231
+ https://github.com/openedx/platform-plugin-aspects/issues
232
+
233
+ For more information about these options, see the `Getting Help <https://openedx.org/getting-help>`__ page.
234
+
235
+ .. _Slack invitation: https://openedx.org/slack
236
+ .. _community Slack workspace: https://openedx.slack.com/
237
+
238
+ License
239
+ *******
240
+
241
+ Please see `LICENSE.txt <LICENSE.txt>`_ for details.
242
+
243
+ Contributing
244
+ ************
245
+
246
+ Contributions are very welcome.
247
+ Please read `How To Contribute <https://openedx.org/r/how-to-contribute>`_ for details.
248
+
249
+ This project is currently accepting all types of contributions, bug fixes,
250
+ security fixes, maintenance work, or new features. However, please make sure
251
+ to have a discussion about your new feature idea with the maintainers prior to
252
+ beginning development to maximize the chances of your change being accepted.
253
+ You can start a conversation by creating a new issue on this repo summarizing
254
+ your idea.
255
+
256
+ The Open edX Code of Conduct
257
+ ****************************
258
+
259
+ All community members are expected to follow the `Open edX Code of Conduct`_.
260
+
261
+ .. _Open edX Code of Conduct: https://openedx.org/code-of-conduct/
262
+
263
+ People
264
+ ******
265
+
266
+ The assigned maintainers for this component and other project details may be
267
+ found in `Backstage`_. Backstage pulls this data from the ``catalog-info.yaml``
268
+ file in this repo.
269
+
270
+ .. _Backstage: https://backstage.openedx.org/catalog/default/component/platform-plugin-aspects
271
+
272
+ Reporting Security Issues
273
+ *************************
274
+
275
+ Please do not report security issues in public. Please email security@openedx.org.
276
+
277
+ .. _Open edX events: https://github.com/openedx/openedx-events
278
+ .. _Edx Platform: https://github.com/openedx/edx-platform
279
+ .. _ClickHouse: https://clickhouse.com
280
+ .. _Aspects: https://docs.openedx.org/projects/openedx-aspects/en/latest/index.html
281
+ .. _Superset Embedded SDK: https://www.npmjs.com/package/@superset-ui/embedded-sdk
282
+ .. _Open edX Filters: https://docs.openedx.org/projects/openedx-filters/en/latest/
283
+ .. _Aspects backfill documentation: https://docs.openedx.org/projects/openedx-aspects/en/latest/technical_documentation/how-tos/backfill.html
@@ -5,6 +5,6 @@ Aspects plugins for edx-platform.
5
5
  import os
6
6
  from pathlib import Path
7
7
 
8
- __version__ = "1.1.0"
8
+ __version__ = "1.1.2"
9
9
 
10
10
  ROOT_DIRECTORY = Path(os.path.dirname(os.path.abspath(__file__)))
@@ -99,15 +99,13 @@ class LoadTest:
99
99
  "display_name": f"Course {self.course_shortname}",
100
100
  }
101
101
 
102
- log.info(
103
- f"""Creating course:
102
+ log.info(f"""Creating course:
104
103
  Instructor: {self.instructor.id}
105
104
  Org: "OEX"
106
105
  Number: "{self.course_shortname}"
107
106
  Run: "2024-1"
108
107
  Fields: {fields}
109
- """
110
- )
108
+ """)
111
109
 
112
110
  self.course = create_new_course_in_store(
113
111
  ModuleStoreEnum.Type.split,
@@ -105,6 +105,10 @@ def plugin_settings(settings):
105
105
  },
106
106
  }
107
107
 
108
+ # This is on by default here, off by default in tutor-contrib-aspects until we only
109
+ # support versions Sumac and beyond
110
+ settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = True
111
+
108
112
  settings.ASPECTS_IN_CONTEXT_DASHBOARDS = {
109
113
  "course": {
110
114
  "name": _("Course"),
@@ -28,6 +28,10 @@ def plugin_settings(settings):
28
28
  "EVENT_SINK_CLICKHOUSE_PII_MODELS",
29
29
  settings.EVENT_SINK_CLICKHOUSE_PII_MODELS,
30
30
  )
31
+ settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS = settings.ENV_TOKENS.get(
32
+ "ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS",
33
+ settings.ASPECTS_ENABLE_STUDIO_IN_CONTEXT_METRICS,
34
+ )
31
35
  settings.ASPECTS_IN_CONTEXT_DASHBOARDS = settings.ENV_TOKENS.get(
32
36
  "ASPECTS_IN_CONTEXT_DASHBOARDS",
33
37
  settings.ASPECTS_IN_CONTEXT_DASHBOARDS,
@@ -155,26 +155,23 @@ def generate_guest_token(user, course, dashboards, filters) -> str:
155
155
  token = response.json().get("token")
156
156
  return token
157
157
 
158
- except HTTPError as err:
159
- # Superset server errors sometimes come with messages, so log the response.
160
- logger.error(
161
- f"{err.response.status_code} {err.response.json()} for url: {err.response.url}, data: {data}"
162
- )
163
- raise ImproperlyConfigured(
164
- _(
165
- "Unable to fetch Superset guest token, "
166
- "Superset server error {server_response}"
167
- ).format(server_response=err.response.json())
168
- ) from err
158
+ # We manually log the error message here because ImproperlyConfigured is eaten up the stack
159
+ except HTTPError as exc:
160
+ err = f"""Unable to fetch Superset guest token, mostly likely due to invalid
161
+ settings.SUPERSET_CONFIG or because one of the dashboard UUIDs requested does not
162
+ exist in Superset.\n
163
+ {exc.response.status_code} {exc.response.json()} for url:
164
+ {exc.response.url}, data: {data}\n\n dashboards: {dashboards}"""
165
+
166
+ logger.error(err)
167
+ raise ImproperlyConfigured(err) from exc
169
168
 
170
169
  except Exception as exc:
171
- logger.error(exc)
172
- raise ImproperlyConfigured(
173
- _(
174
- "Unable to fetch Superset guest token, "
175
- "mostly likely due to invalid settings.SUPERSET_CONFIG"
176
- )
177
- ) from exc
170
+ err = f"""Error trying to fetch Superset guest token, mostly likely due to invalid
171
+ settings.SUPERSET_CONFIG or because one of the dashboard UUIDs requested does not
172
+ exist in Superset. data: {data} \n\n exception: {exc}"""
173
+ logger.error(err)
174
+ raise ImproperlyConfigured(err) from exc
178
175
 
179
176
 
180
177
  def _fix_service_url(url: str) -> str: