platform-plugin-aspects 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- platform_plugin_aspects/__init__.py +11 -0
- platform_plugin_aspects/apps.py +77 -0
- platform_plugin_aspects/conf/locale/babel_mako.cfg +10 -0
- platform_plugin_aspects/conf/locale/config.yaml +85 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +99 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +70 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/extensions/__init__.py +0 -0
- platform_plugin_aspects/extensions/filters.py +108 -0
- platform_plugin_aspects/management/__init__.py +0 -0
- platform_plugin_aspects/management/commands/__init__.py +0 -0
- platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +214 -0
- platform_plugin_aspects/management/commands/load_test_tracking_events.py +310 -0
- platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +440 -0
- platform_plugin_aspects/settings/__init__.py +0 -0
- platform_plugin_aspects/settings/common.py +147 -0
- platform_plugin_aspects/settings/production.py +38 -0
- platform_plugin_aspects/signals.py +305 -0
- platform_plugin_aspects/sinks/__init__.py +11 -0
- platform_plugin_aspects/sinks/base_sink.py +381 -0
- platform_plugin_aspects/sinks/course_enrollment_sink.py +20 -0
- platform_plugin_aspects/sinks/course_overview_sink.py +286 -0
- platform_plugin_aspects/sinks/external_id_sink.py +20 -0
- platform_plugin_aspects/sinks/serializers.py +274 -0
- platform_plugin_aspects/sinks/tag_sink.py +47 -0
- platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
- platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
- platform_plugin_aspects/static/css/superset.css +53 -0
- platform_plugin_aspects/static/html/superset.html +39 -0
- platform_plugin_aspects/static/html/superset_student.html +5 -0
- platform_plugin_aspects/static/js/embed_dashboard.js +94 -0
- platform_plugin_aspects/static/js/install_required.js +11 -0
- platform_plugin_aspects/static/js/superset.js +38 -0
- platform_plugin_aspects/tasks.py +64 -0
- platform_plugin_aspects/templates/instructor_dashboard/aspects.html +22 -0
- platform_plugin_aspects/urls.py +39 -0
- platform_plugin_aspects/utils.py +359 -0
- platform_plugin_aspects/views.py +296 -0
- platform_plugin_aspects/waffle.py +5 -0
- platform_plugin_aspects/xblock.py +239 -0
- platform_plugin_aspects-2.0.1.dist-info/METADATA +335 -0
- platform_plugin_aspects-2.0.1.dist-info/RECORD +51 -0
- platform_plugin_aspects-2.0.1.dist-info/WHEEL +5 -0
- platform_plugin_aspects-2.0.1.dist-info/entry_points.txt +8 -0
- platform_plugin_aspects-2.0.1.dist-info/licenses/LICENSE +201 -0
- platform_plugin_aspects-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""User profile sink"""
|
|
2
|
+
|
|
3
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
4
|
+
from platform_plugin_aspects.sinks.serializers import CourseEnrollmentSerializer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CourseEnrollmentSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
8
|
+
"""
|
|
9
|
+
Sink for user CourseEnrollment model
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
model = "course_enrollment"
|
|
13
|
+
unique_key = "id"
|
|
14
|
+
clickhouse_table_name = "course_enrollment"
|
|
15
|
+
timestamp_field = "time_last_dumped"
|
|
16
|
+
name = "Course Enrollment"
|
|
17
|
+
serializer_class = CourseEnrollmentSerializer
|
|
18
|
+
|
|
19
|
+
def get_queryset(self, start_pk=None):
|
|
20
|
+
return super().get_queryset(start_pk).select_related("user")
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Handler for the CMS COURSE_PUBLISHED event
|
|
3
|
+
|
|
4
|
+
Does the following:
|
|
5
|
+
- Pulls the course structure from modulestore
|
|
6
|
+
- Serialize the xblocks
|
|
7
|
+
- Sends them to ClickHouse in CSV format
|
|
8
|
+
|
|
9
|
+
Note that the serialization format does not include all fields as there may be things like
|
|
10
|
+
LTI passwords and other secrets. We just take the fields necessary for reporting at this time.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import datetime
|
|
14
|
+
import json
|
|
15
|
+
|
|
16
|
+
from opaque_keys.edx.keys import CourseKey
|
|
17
|
+
|
|
18
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
19
|
+
from platform_plugin_aspects.sinks.serializers import CourseOverviewSerializer
|
|
20
|
+
from platform_plugin_aspects.utils import (
|
|
21
|
+
get_detached_xblock_types,
|
|
22
|
+
get_modulestore,
|
|
23
|
+
get_tags_for_block,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Defaults we want to ensure we fail early on bulk inserts
|
|
27
|
+
CLICKHOUSE_BULK_INSERT_PARAMS = {
|
|
28
|
+
"input_format_allow_errors_num": 1,
|
|
29
|
+
"input_format_allow_errors_ratio": 0.1,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
MODULESTORE_PUBLISHED_ONLY_FLAG = "rev-opt-published-only"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class XBlockSink(ModelBaseSink):
|
|
36
|
+
"""
|
|
37
|
+
Sink for XBlock model
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
unique_key = "location"
|
|
41
|
+
clickhouse_table_name = "course_blocks"
|
|
42
|
+
timestamp_field = "time_last_dumped"
|
|
43
|
+
name = "XBlock"
|
|
44
|
+
nested_sinks = []
|
|
45
|
+
|
|
46
|
+
def dump_related(self, serialized_item, dump_id, time_last_dumped):
|
|
47
|
+
"""Dump all XBlocks for a course"""
|
|
48
|
+
self.dump(
|
|
49
|
+
serialized_item,
|
|
50
|
+
many=True,
|
|
51
|
+
initial={"dump_id": dump_id, "time_last_dumped": time_last_dumped},
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def get_xblocks_recursive(self, parent_block, detached_xblock_types, initial):
|
|
55
|
+
"""
|
|
56
|
+
Serialize the course tree recursively, return a flattened list of XBlocks.
|
|
57
|
+
|
|
58
|
+
Note that this list will not include detached blocks, those are handled
|
|
59
|
+
in get_detached_xblocks. This method preserves the course ordering for
|
|
60
|
+
non-detached blocks.
|
|
61
|
+
"""
|
|
62
|
+
items = [
|
|
63
|
+
self.serialize_xblock(
|
|
64
|
+
parent_block,
|
|
65
|
+
detached_xblock_types,
|
|
66
|
+
initial["dump_id"],
|
|
67
|
+
initial["time_last_dumped"],
|
|
68
|
+
)
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
for child in parent_block.get_children():
|
|
72
|
+
items.extend(
|
|
73
|
+
self.get_xblocks_recursive(child, detached_xblock_types, initial)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
return items
|
|
77
|
+
|
|
78
|
+
def get_detached_xblocks(self, course_blocks, detached_xblock_types, initial):
|
|
79
|
+
"""
|
|
80
|
+
Spin through the flat list of all blocks in a course and return only
|
|
81
|
+
the detached blocks. Ordering of non-detached blocks is already
|
|
82
|
+
guaranteed in get_xblocks_recursive. Order of detached blocks
|
|
83
|
+
is not guaranteed.
|
|
84
|
+
"""
|
|
85
|
+
return [
|
|
86
|
+
self.serialize_xblock(
|
|
87
|
+
block,
|
|
88
|
+
detached_xblock_types,
|
|
89
|
+
initial["dump_id"],
|
|
90
|
+
initial["time_last_dumped"],
|
|
91
|
+
)
|
|
92
|
+
for block in course_blocks
|
|
93
|
+
if block.scope_ids.block_type in detached_xblock_types
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
def serialize_item(self, item, many=False, initial=None):
|
|
97
|
+
"""
|
|
98
|
+
Serialize an XBlock into a dict
|
|
99
|
+
"""
|
|
100
|
+
course_key = CourseKey.from_string(item["course_key"])
|
|
101
|
+
modulestore = get_modulestore()
|
|
102
|
+
detached_xblock_types = get_detached_xblock_types()
|
|
103
|
+
|
|
104
|
+
location_to_node = {}
|
|
105
|
+
|
|
106
|
+
# This call gets the entire course tree in order, because the
|
|
107
|
+
# get_items call does not guarantee ordering. It does not return
|
|
108
|
+
# detached blocks, so we gather them separately below.
|
|
109
|
+
course_block = modulestore.get_course(
|
|
110
|
+
course_key, revision=MODULESTORE_PUBLISHED_ONLY_FLAG
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
items = self.get_xblocks_recursive(course_block, detached_xblock_types, initial)
|
|
114
|
+
|
|
115
|
+
# Here we fetch the detached blocks and add them to the list.
|
|
116
|
+
detached = self.get_detached_xblocks(
|
|
117
|
+
modulestore.get_items(course_key, revision=MODULESTORE_PUBLISHED_ONLY_FLAG),
|
|
118
|
+
detached_xblock_types,
|
|
119
|
+
initial,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
items.extend(detached)
|
|
123
|
+
|
|
124
|
+
# Add location and tag data to the dict mappings of the blocks
|
|
125
|
+
index = 0
|
|
126
|
+
section_idx = 0
|
|
127
|
+
subsection_idx = 0
|
|
128
|
+
unit_idx = 0
|
|
129
|
+
|
|
130
|
+
for block in items:
|
|
131
|
+
index += 1
|
|
132
|
+
|
|
133
|
+
block["order"] = index
|
|
134
|
+
|
|
135
|
+
# Ensure that detached types aren't part of the tree
|
|
136
|
+
if block["xblock_data_json"]["detached"]:
|
|
137
|
+
block["xblock_data_json"]["section"] = 0
|
|
138
|
+
block["xblock_data_json"]["subsection"] = 0
|
|
139
|
+
block["xblock_data_json"]["unit"] = 0
|
|
140
|
+
else:
|
|
141
|
+
if block["xblock_data_json"]["block_type"] == "chapter":
|
|
142
|
+
section_idx += 1
|
|
143
|
+
subsection_idx = 0
|
|
144
|
+
unit_idx = 0
|
|
145
|
+
elif block["xblock_data_json"]["block_type"] == "sequential":
|
|
146
|
+
subsection_idx += 1
|
|
147
|
+
unit_idx = 0
|
|
148
|
+
elif block["xblock_data_json"]["block_type"] == "vertical":
|
|
149
|
+
unit_idx += 1
|
|
150
|
+
|
|
151
|
+
block["xblock_data_json"]["section"] = section_idx
|
|
152
|
+
block["xblock_data_json"]["subsection"] = subsection_idx
|
|
153
|
+
block["xblock_data_json"]["unit"] = unit_idx
|
|
154
|
+
|
|
155
|
+
block["xblock_data_json"]["tags"] = get_tags_for_block(
|
|
156
|
+
block["location"],
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
block["xblock_data_json"] = json.dumps(block["xblock_data_json"])
|
|
160
|
+
location_to_node[block["location"]] = block
|
|
161
|
+
|
|
162
|
+
return list(location_to_node.values())
|
|
163
|
+
|
|
164
|
+
def serialize_xblock(self, item, detached_xblock_types, dump_id, time_last_dumped):
|
|
165
|
+
"""Serialize an XBlock instance into a dict"""
|
|
166
|
+
course_key = item.scope_ids.usage_id.course_key
|
|
167
|
+
block_type = item.scope_ids.block_type
|
|
168
|
+
|
|
169
|
+
# Extra data not needed for the table to function, things can be
|
|
170
|
+
# added here without needing to rebuild the whole table.
|
|
171
|
+
json_data = {
|
|
172
|
+
"course": course_key.course,
|
|
173
|
+
"run": course_key.run,
|
|
174
|
+
"block_type": block_type,
|
|
175
|
+
"detached": 1 if block_type in detached_xblock_types else 0,
|
|
176
|
+
"graded": 1 if getattr(item, "graded", False) else 0,
|
|
177
|
+
"completion_mode": getattr(item, "completion_mode", ""),
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
# Core table data, if things change here it's a big deal.
|
|
181
|
+
serialized_block = {
|
|
182
|
+
"org": course_key.org,
|
|
183
|
+
"course_key": str(course_key),
|
|
184
|
+
"location": str(XBlockSink.strip_branch_and_version(item.location)),
|
|
185
|
+
"display_name": item.display_name_with_default.replace("'", "'"),
|
|
186
|
+
"xblock_data_json": json_data,
|
|
187
|
+
# We need to add this here so the key will be in the right place
|
|
188
|
+
# in the generated csv
|
|
189
|
+
"order": -1,
|
|
190
|
+
"edited_on": str(getattr(item, "edited_on", "")),
|
|
191
|
+
"dump_id": dump_id,
|
|
192
|
+
"time_last_dumped": time_last_dumped,
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return serialized_block
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def strip_branch_and_version(location):
|
|
199
|
+
"""
|
|
200
|
+
Removes the branch and version information from a location.
|
|
201
|
+
Args:
|
|
202
|
+
location: an xblock's location.
|
|
203
|
+
Returns: that xblock's location without branch and version information.
|
|
204
|
+
"""
|
|
205
|
+
return location.for_branch(None)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class CourseOverviewSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
209
|
+
"""
|
|
210
|
+
Sink for CourseOverview model
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
model = "course_overviews"
|
|
214
|
+
unique_key = "course_key"
|
|
215
|
+
clickhouse_table_name = "course_overviews"
|
|
216
|
+
timestamp_field = "time_last_dumped"
|
|
217
|
+
name = "Course Overview"
|
|
218
|
+
serializer_class = CourseOverviewSerializer
|
|
219
|
+
nested_sinks = [XBlockSink]
|
|
220
|
+
pk_format = str
|
|
221
|
+
|
|
222
|
+
def should_dump_item(self, item):
|
|
223
|
+
"""
|
|
224
|
+
Only dump the course if it's been changed since the last time it's been
|
|
225
|
+
dumped.
|
|
226
|
+
Args:
|
|
227
|
+
course_key: a CourseKey object.
|
|
228
|
+
Returns:
|
|
229
|
+
- whether this course should be dumped (bool)
|
|
230
|
+
- reason why course needs, or does not need, to be dumped (string)
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
course_last_dump_time = self.get_last_dumped_timestamp(item)
|
|
234
|
+
|
|
235
|
+
# If we don't have a record of the last time this command was run,
|
|
236
|
+
# we should serialize the course and dump it
|
|
237
|
+
if course_last_dump_time is None:
|
|
238
|
+
return True, "Course is not present in ClickHouse"
|
|
239
|
+
|
|
240
|
+
course_last_published_date = self.get_course_last_published(item)
|
|
241
|
+
|
|
242
|
+
# If we've somehow dumped this course but there is no publish date
|
|
243
|
+
# skip it
|
|
244
|
+
if course_last_dump_time and course_last_published_date is None:
|
|
245
|
+
return False, "No last modified date in CourseOverview"
|
|
246
|
+
|
|
247
|
+
# Otherwise, dump it if it is newer
|
|
248
|
+
course_last_dump_time = datetime.datetime.strptime(
|
|
249
|
+
course_last_dump_time, "%Y-%m-%d %H:%M:%S.%f+00:00"
|
|
250
|
+
)
|
|
251
|
+
course_last_published_date = datetime.datetime.strptime(
|
|
252
|
+
course_last_published_date, "%Y-%m-%d %H:%M:%S.%f+00:00"
|
|
253
|
+
)
|
|
254
|
+
needs_dump = course_last_dump_time < course_last_published_date
|
|
255
|
+
|
|
256
|
+
if needs_dump:
|
|
257
|
+
reason = (
|
|
258
|
+
"Course has been published since last dump time - "
|
|
259
|
+
f"last dumped {course_last_dump_time} < last published {str(course_last_published_date)}"
|
|
260
|
+
)
|
|
261
|
+
else:
|
|
262
|
+
reason = (
|
|
263
|
+
f"Course has NOT been published since last dump time - "
|
|
264
|
+
f"last dumped {course_last_dump_time} >= last published {str(course_last_published_date)}"
|
|
265
|
+
)
|
|
266
|
+
return needs_dump, reason
|
|
267
|
+
|
|
268
|
+
def get_course_last_published(self, course_overview):
|
|
269
|
+
"""
|
|
270
|
+
Get approximate last publish date for the given course.
|
|
271
|
+
We use the 'modified' column in the CourseOverview table as a quick and easy
|
|
272
|
+
(although perhaps inexact) way of determining when a course was last
|
|
273
|
+
published. This works because CourseOverview rows are re-written upon
|
|
274
|
+
course publish.
|
|
275
|
+
Args:
|
|
276
|
+
course_key: a CourseKey
|
|
277
|
+
Returns: The datetime the course was last published at, stringified.
|
|
278
|
+
Uses Python's default str(...) implementation for datetimes, which
|
|
279
|
+
is sortable and similar to ISO 8601:
|
|
280
|
+
https://docs.python.org/3/library/datetime.html#datetime.date.__str__
|
|
281
|
+
"""
|
|
282
|
+
approx_last_published = course_overview.modified
|
|
283
|
+
if approx_last_published:
|
|
284
|
+
return str(approx_last_published)
|
|
285
|
+
|
|
286
|
+
return None
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""User profile sink"""
|
|
2
|
+
|
|
3
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
4
|
+
from platform_plugin_aspects.sinks.serializers import UserExternalIDSerializer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ExternalIdSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
8
|
+
"""
|
|
9
|
+
Sink for user external ID serializer
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
model = "external_id"
|
|
13
|
+
unique_key = "id"
|
|
14
|
+
clickhouse_table_name = "external_id"
|
|
15
|
+
timestamp_field = "time_last_dumped"
|
|
16
|
+
name = "External ID"
|
|
17
|
+
serializer_class = UserExternalIDSerializer
|
|
18
|
+
|
|
19
|
+
def get_queryset(self, start_pk=None):
|
|
20
|
+
return super().get_queryset(start_pk).select_related("user", "external_id_type")
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Django serializers for the event_sink_clickhouse app."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import uuid
|
|
5
|
+
from datetime import date, datetime
|
|
6
|
+
|
|
7
|
+
from django.utils import timezone
|
|
8
|
+
from pytz import UTC
|
|
9
|
+
from rest_framework import serializers
|
|
10
|
+
|
|
11
|
+
from platform_plugin_aspects.utils import get_model, get_tags_for_block
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DateTimeJSONEncoder(json.JSONEncoder):
|
|
15
|
+
"""JSON encoder aware of datetime.datetime and datetime.date objects"""
|
|
16
|
+
|
|
17
|
+
def default(self, obj): # pylint: disable=arguments-renamed
|
|
18
|
+
"""
|
|
19
|
+
Serialize datetime and date objects of iso format.
|
|
20
|
+
|
|
21
|
+
datetime objects are converted to UTC.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
if isinstance(obj, datetime):
|
|
25
|
+
if obj.tzinfo is None:
|
|
26
|
+
# Localize to UTC naive datetime objects
|
|
27
|
+
obj = UTC.localize(obj) # pylint: disable=no-value-for-parameter
|
|
28
|
+
else:
|
|
29
|
+
# Convert to UTC datetime objects from other timezones
|
|
30
|
+
obj = obj.astimezone(UTC)
|
|
31
|
+
return obj.isoformat()
|
|
32
|
+
elif isinstance(obj, date):
|
|
33
|
+
return obj.isoformat()
|
|
34
|
+
|
|
35
|
+
return super().default(obj)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class BaseSinkSerializer(serializers.Serializer): # pylint: disable=abstract-method
|
|
39
|
+
"""Base sink serializer for ClickHouse."""
|
|
40
|
+
|
|
41
|
+
dump_id = serializers.SerializerMethodField()
|
|
42
|
+
time_last_dumped = serializers.SerializerMethodField()
|
|
43
|
+
|
|
44
|
+
class Meta:
|
|
45
|
+
"""Meta class for base sink serializer."""
|
|
46
|
+
|
|
47
|
+
fields = [
|
|
48
|
+
"dump_id",
|
|
49
|
+
"time_last_dumped",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
def get_dump_id(self, instance): # pylint: disable=unused-argument
|
|
53
|
+
"""Return a unique ID for the dump."""
|
|
54
|
+
return uuid.uuid4()
|
|
55
|
+
|
|
56
|
+
def get_time_last_dumped(self, instance): # pylint: disable=unused-argument
|
|
57
|
+
"""Return the timestamp for the dump."""
|
|
58
|
+
return timezone.now()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class UserProfileSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
62
|
+
"""Serializer for user profile events."""
|
|
63
|
+
|
|
64
|
+
email = serializers.CharField(source="user.email")
|
|
65
|
+
username = serializers.CharField(source="user.username")
|
|
66
|
+
|
|
67
|
+
class Meta:
|
|
68
|
+
"""Meta class for user profile serializer."""
|
|
69
|
+
|
|
70
|
+
model = get_model("user_profile")
|
|
71
|
+
|
|
72
|
+
fields = [
|
|
73
|
+
"id",
|
|
74
|
+
"user_id",
|
|
75
|
+
"name",
|
|
76
|
+
"username",
|
|
77
|
+
"email",
|
|
78
|
+
"meta",
|
|
79
|
+
"courseware",
|
|
80
|
+
"language",
|
|
81
|
+
"location",
|
|
82
|
+
"year_of_birth",
|
|
83
|
+
"gender",
|
|
84
|
+
"level_of_education",
|
|
85
|
+
"mailing_address",
|
|
86
|
+
"city",
|
|
87
|
+
"country",
|
|
88
|
+
"state",
|
|
89
|
+
"goals",
|
|
90
|
+
"bio",
|
|
91
|
+
"profile_image_uploaded_at",
|
|
92
|
+
"phone_number",
|
|
93
|
+
"dump_id",
|
|
94
|
+
"time_last_dumped",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class UserExternalIDSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
99
|
+
"""Serializer for user external ID events."""
|
|
100
|
+
|
|
101
|
+
external_id_type = serializers.CharField(source="external_id_type.name")
|
|
102
|
+
username = serializers.CharField(source="user.username")
|
|
103
|
+
|
|
104
|
+
class Meta:
|
|
105
|
+
"""Meta class for user external ID serializer."""
|
|
106
|
+
|
|
107
|
+
model = get_model("external_id")
|
|
108
|
+
fields = [
|
|
109
|
+
"external_user_id",
|
|
110
|
+
"external_id_type",
|
|
111
|
+
"username",
|
|
112
|
+
"user_id",
|
|
113
|
+
"dump_id",
|
|
114
|
+
"time_last_dumped",
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class UserRetirementSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
119
|
+
"""Serializer for user retirement events."""
|
|
120
|
+
|
|
121
|
+
user_id = serializers.CharField(source="id")
|
|
122
|
+
|
|
123
|
+
class Meta:
|
|
124
|
+
"""Meta class for user retirement serializer."""
|
|
125
|
+
|
|
126
|
+
model = get_model("auth_user")
|
|
127
|
+
fields = [
|
|
128
|
+
"user_id",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class CourseOverviewSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
133
|
+
"""Serializer for course overview events."""
|
|
134
|
+
|
|
135
|
+
course_data_json = serializers.SerializerMethodField()
|
|
136
|
+
course_key = serializers.SerializerMethodField()
|
|
137
|
+
course_start = serializers.CharField(source="start")
|
|
138
|
+
course_end = serializers.CharField(source="end")
|
|
139
|
+
|
|
140
|
+
class Meta:
|
|
141
|
+
"""Meta classes for course overview serializer."""
|
|
142
|
+
|
|
143
|
+
model = get_model("course_overviews")
|
|
144
|
+
fields = [
|
|
145
|
+
"org",
|
|
146
|
+
"course_key",
|
|
147
|
+
"display_name",
|
|
148
|
+
"course_start",
|
|
149
|
+
"course_end",
|
|
150
|
+
"enrollment_start",
|
|
151
|
+
"enrollment_end",
|
|
152
|
+
"self_paced",
|
|
153
|
+
"course_data_json",
|
|
154
|
+
"created",
|
|
155
|
+
"modified",
|
|
156
|
+
"dump_id",
|
|
157
|
+
"time_last_dumped",
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
def get_course_data_json(self, overview):
|
|
161
|
+
"""Return the course data as a JSON string."""
|
|
162
|
+
json_fields = {
|
|
163
|
+
"advertised_start": getattr(overview, "advertised_start", ""),
|
|
164
|
+
"announcement": getattr(overview, "announcement", ""),
|
|
165
|
+
"lowest_passing_grade": float(
|
|
166
|
+
getattr(overview, "lowest_passing_grade", 0.0)
|
|
167
|
+
),
|
|
168
|
+
"invitation_only": getattr(overview, "invitation_only", ""),
|
|
169
|
+
"max_student_enrollments_allowed": getattr(
|
|
170
|
+
overview, "max_student_enrollments_allowed", None
|
|
171
|
+
),
|
|
172
|
+
"effort": getattr(overview, "effort", ""),
|
|
173
|
+
"enable_proctored_exams": getattr(overview, "enable_proctored_exams", ""),
|
|
174
|
+
"entrance_exam_enabled": getattr(overview, "entrance_exam_enabled", ""),
|
|
175
|
+
"external_id": getattr(overview, "external_id", ""),
|
|
176
|
+
"language": getattr(overview, "language", ""),
|
|
177
|
+
"tags": get_tags_for_block(overview.id),
|
|
178
|
+
}
|
|
179
|
+
return json.dumps(json_fields, cls=DateTimeJSONEncoder)
|
|
180
|
+
|
|
181
|
+
def get_course_key(self, overview):
|
|
182
|
+
"""Return the course key as a string."""
|
|
183
|
+
return str(overview.id)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class CourseEnrollmentSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
187
|
+
"""Serializer for the Course Enrollment model."""
|
|
188
|
+
|
|
189
|
+
course_key = serializers.SerializerMethodField()
|
|
190
|
+
username = serializers.CharField(source="user.username")
|
|
191
|
+
|
|
192
|
+
class Meta:
|
|
193
|
+
"""Meta class for the CourseEnrollmentSerializer"""
|
|
194
|
+
|
|
195
|
+
model = get_model("course_enrollment")
|
|
196
|
+
fields = [
|
|
197
|
+
"id",
|
|
198
|
+
"course_key",
|
|
199
|
+
"created",
|
|
200
|
+
"is_active",
|
|
201
|
+
"mode",
|
|
202
|
+
"username",
|
|
203
|
+
"user_id",
|
|
204
|
+
"dump_id",
|
|
205
|
+
"time_last_dumped",
|
|
206
|
+
]
|
|
207
|
+
|
|
208
|
+
def get_course_key(self, obj):
|
|
209
|
+
"""Return the course key as a string."""
|
|
210
|
+
return str(obj.course_id)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class TagSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
214
|
+
"""Serializer for the Tag model."""
|
|
215
|
+
|
|
216
|
+
lineage = serializers.SerializerMethodField()
|
|
217
|
+
|
|
218
|
+
class Meta:
|
|
219
|
+
"""Meta class for the TagSerializer."""
|
|
220
|
+
|
|
221
|
+
model = get_model("tag")
|
|
222
|
+
fields = [
|
|
223
|
+
"id",
|
|
224
|
+
"taxonomy",
|
|
225
|
+
"parent",
|
|
226
|
+
"value",
|
|
227
|
+
"external_id",
|
|
228
|
+
"lineage",
|
|
229
|
+
"dump_id",
|
|
230
|
+
"time_last_dumped",
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
def get_lineage(self, instance):
|
|
234
|
+
return json.dumps(instance.get_lineage())
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class ObjectTagSerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
238
|
+
"""Serializer for the ObjectTag model."""
|
|
239
|
+
|
|
240
|
+
lineage = serializers.SerializerMethodField()
|
|
241
|
+
|
|
242
|
+
class Meta:
|
|
243
|
+
"""Meta class for the ObjectTagSerializer"""
|
|
244
|
+
|
|
245
|
+
model = get_model("object_tag")
|
|
246
|
+
fields = [
|
|
247
|
+
"id",
|
|
248
|
+
"object_id",
|
|
249
|
+
"taxonomy",
|
|
250
|
+
"tag",
|
|
251
|
+
"_value",
|
|
252
|
+
"_export_id",
|
|
253
|
+
"lineage",
|
|
254
|
+
"dump_id",
|
|
255
|
+
"time_last_dumped",
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
def get_lineage(self, instance):
|
|
259
|
+
return json.dumps(instance.get_lineage())
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class TaxonomySerializer(BaseSinkSerializer, serializers.ModelSerializer):
|
|
263
|
+
"""Serializer for the Taxonomy model."""
|
|
264
|
+
|
|
265
|
+
class Meta:
|
|
266
|
+
"""Meta class for the TaxonomySerializer."""
|
|
267
|
+
|
|
268
|
+
model = get_model("taxonomy")
|
|
269
|
+
fields = [
|
|
270
|
+
"id",
|
|
271
|
+
"name",
|
|
272
|
+
"dump_id",
|
|
273
|
+
"time_last_dumped",
|
|
274
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Tag sink"""
|
|
2
|
+
|
|
3
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
4
|
+
from platform_plugin_aspects.sinks.serializers import (
|
|
5
|
+
ObjectTagSerializer,
|
|
6
|
+
TagSerializer,
|
|
7
|
+
TaxonomySerializer,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TagSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
12
|
+
"""
|
|
13
|
+
Sink for content tags
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
model = "tag"
|
|
17
|
+
unique_key = "id"
|
|
18
|
+
clickhouse_table_name = "tag"
|
|
19
|
+
timestamp_field = "time_last_dumped"
|
|
20
|
+
name = "Tag"
|
|
21
|
+
serializer_class = TagSerializer
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TaxonomySink(ModelBaseSink): # pylint: disable=abstract-method
|
|
25
|
+
"""
|
|
26
|
+
Sink for content taxonomy
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
model = "taxonomy"
|
|
30
|
+
unique_key = "id"
|
|
31
|
+
clickhouse_table_name = "taxonomy"
|
|
32
|
+
timestamp_field = "time_last_dumped"
|
|
33
|
+
name = "Taxonomy"
|
|
34
|
+
serializer_class = TaxonomySerializer
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ObjectTagSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
38
|
+
"""
|
|
39
|
+
Sink for tagged objects
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
model = "object_tag"
|
|
43
|
+
unique_key = "id"
|
|
44
|
+
clickhouse_table_name = "object_tag"
|
|
45
|
+
timestamp_field = "time_last_dumped"
|
|
46
|
+
name = "ObjectTag"
|
|
47
|
+
serializer_class = ObjectTagSerializer
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""User profile sink"""
|
|
2
|
+
|
|
3
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
4
|
+
from platform_plugin_aspects.sinks.serializers import UserProfileSerializer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class UserProfileSink(ModelBaseSink): # pylint: disable=abstract-method
|
|
8
|
+
"""
|
|
9
|
+
Sink for user profile events
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
model = "user_profile"
|
|
13
|
+
unique_key = "id"
|
|
14
|
+
clickhouse_table_name = "user_profile"
|
|
15
|
+
timestamp_field = "time_last_dumped"
|
|
16
|
+
name = "User Profile"
|
|
17
|
+
serializer_class = UserProfileSerializer
|
|
18
|
+
|
|
19
|
+
def get_queryset(self, start_pk=None):
|
|
20
|
+
return super().get_queryset(start_pk).select_related("user")
|