platform-plugin-aspects 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- platform_plugin_aspects/__init__.py +11 -0
- platform_plugin_aspects/apps.py +77 -0
- platform_plugin_aspects/conf/locale/babel_mako.cfg +10 -0
- platform_plugin_aspects/conf/locale/config.yaml +85 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +99 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +70 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
- platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +23 -0
- platform_plugin_aspects/extensions/__init__.py +0 -0
- platform_plugin_aspects/extensions/filters.py +108 -0
- platform_plugin_aspects/management/__init__.py +0 -0
- platform_plugin_aspects/management/commands/__init__.py +0 -0
- platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +214 -0
- platform_plugin_aspects/management/commands/load_test_tracking_events.py +310 -0
- platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +440 -0
- platform_plugin_aspects/settings/__init__.py +0 -0
- platform_plugin_aspects/settings/common.py +147 -0
- platform_plugin_aspects/settings/production.py +38 -0
- platform_plugin_aspects/signals.py +305 -0
- platform_plugin_aspects/sinks/__init__.py +11 -0
- platform_plugin_aspects/sinks/base_sink.py +381 -0
- platform_plugin_aspects/sinks/course_enrollment_sink.py +20 -0
- platform_plugin_aspects/sinks/course_overview_sink.py +286 -0
- platform_plugin_aspects/sinks/external_id_sink.py +20 -0
- platform_plugin_aspects/sinks/serializers.py +274 -0
- platform_plugin_aspects/sinks/tag_sink.py +47 -0
- platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
- platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
- platform_plugin_aspects/static/css/superset.css +53 -0
- platform_plugin_aspects/static/html/superset.html +39 -0
- platform_plugin_aspects/static/html/superset_student.html +5 -0
- platform_plugin_aspects/static/js/embed_dashboard.js +94 -0
- platform_plugin_aspects/static/js/install_required.js +11 -0
- platform_plugin_aspects/static/js/superset.js +38 -0
- platform_plugin_aspects/tasks.py +64 -0
- platform_plugin_aspects/templates/instructor_dashboard/aspects.html +22 -0
- platform_plugin_aspects/urls.py +39 -0
- platform_plugin_aspects/utils.py +359 -0
- platform_plugin_aspects/views.py +296 -0
- platform_plugin_aspects/waffle.py +5 -0
- platform_plugin_aspects/xblock.py +239 -0
- platform_plugin_aspects-2.0.1.dist-info/METADATA +335 -0
- platform_plugin_aspects-2.0.1.dist-info/RECORD +51 -0
- platform_plugin_aspects-2.0.1.dist-info/WHEEL +5 -0
- platform_plugin_aspects-2.0.1.dist-info/entry_points.txt +8 -0
- platform_plugin_aspects-2.0.1.dist-info/licenses/LICENSE +201 -0
- platform_plugin_aspects-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Signal handler functions, mapped to specific signals in apps.py.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.db import transaction
|
|
6
|
+
from django.db.models.signals import post_delete, post_save
|
|
7
|
+
from django.dispatch import Signal, receiver
|
|
8
|
+
from opaque_keys import InvalidKeyError
|
|
9
|
+
|
|
10
|
+
from platform_plugin_aspects.sinks import (
|
|
11
|
+
CourseEnrollmentSink,
|
|
12
|
+
ExternalIdSink,
|
|
13
|
+
ObjectTagSink,
|
|
14
|
+
TagSink,
|
|
15
|
+
TaxonomySink,
|
|
16
|
+
UserProfileSink,
|
|
17
|
+
UserRetirementSink,
|
|
18
|
+
)
|
|
19
|
+
from platform_plugin_aspects.utils import get_model
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
from openedx.core.djangoapps.user_api.accounts.signals import USER_RETIRE_LMS_MISC
|
|
23
|
+
except ImportError:
|
|
24
|
+
# Tests don't have the platform installed
|
|
25
|
+
USER_RETIRE_LMS_MISC = Signal()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def receive_course_publish( # pylint: disable=unused-argument # pragma: no cover
|
|
29
|
+
sender, course_key, **kwargs
|
|
30
|
+
):
|
|
31
|
+
"""
|
|
32
|
+
Receives COURSE_PUBLISHED signal and queues the dump job.
|
|
33
|
+
"""
|
|
34
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
35
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
36
|
+
dump_course_to_clickhouse,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
dump_course_to_clickhouse.delay(str(course_key))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def receive_course_enrollment_changed( # pylint: disable=unused-argument # pragma: no cover
|
|
43
|
+
sender, **kwargs
|
|
44
|
+
):
|
|
45
|
+
"""
|
|
46
|
+
Receives ENROLL_STATUS_CHANGE signal and queues the dump job.
|
|
47
|
+
|
|
48
|
+
Handles the course enrollment dumping in the middle of a transaction.
|
|
49
|
+
If this gets fired before the transaction commits, the task may try to query
|
|
50
|
+
an id that doesn't exist yet and throw an error. This should postpone
|
|
51
|
+
queuing the Celery task until after the transaction is committed.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def on_course_enrollment_changed(**kwargs):
|
|
55
|
+
"""
|
|
56
|
+
Queues the CourseEnrollment dump job when the parent transaction is committed.
|
|
57
|
+
"""
|
|
58
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
59
|
+
dump_data_to_clickhouse,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
user = kwargs.get("user")
|
|
63
|
+
course_id = kwargs.get("course_id")
|
|
64
|
+
CourseEnrollment = get_model("course_enrollment")
|
|
65
|
+
instance = CourseEnrollment.objects.get(user=user, course_id=course_id)
|
|
66
|
+
|
|
67
|
+
sink = CourseEnrollmentSink(None, None)
|
|
68
|
+
dump_data_to_clickhouse.delay(
|
|
69
|
+
sink_module=sink.__module__,
|
|
70
|
+
sink_name=sink.__class__.__name__,
|
|
71
|
+
object_id=str(instance.id),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
transaction.on_commit(lambda: on_course_enrollment_changed(**kwargs))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def on_user_profile_updated_txn(*args, **kwargs):
|
|
78
|
+
"""
|
|
79
|
+
Handle user_profile saves in the middle of a transaction.
|
|
80
|
+
|
|
81
|
+
Handle saves in the middle of a transaction.
|
|
82
|
+
If this gets fired before the transaction commits, the task may try to
|
|
83
|
+
query an id that doesn't exist yet and throw an error. This should postpone
|
|
84
|
+
queuing the Celery task until after the transaction is committed.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def on_user_profile_updated(instance, **kwargs):
|
|
88
|
+
"""
|
|
89
|
+
Queues the UserProfile dump job when the parent transaction is committed.
|
|
90
|
+
"""
|
|
91
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
92
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
93
|
+
dump_data_to_clickhouse,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
sink = UserProfileSink(None, None)
|
|
97
|
+
dump_data_to_clickhouse.delay(
|
|
98
|
+
sink_module=sink.__module__,
|
|
99
|
+
sink_name=sink.__class__.__name__,
|
|
100
|
+
object_id=str(instance.id),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
transaction.on_commit(lambda: on_user_profile_updated(*args, **kwargs))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# Connect the UserProfile.post_save signal handler only if we have a model to attach to.
|
|
107
|
+
# (prevents celery errors during tests)
|
|
108
|
+
_user_profile = get_model("user_profile")
|
|
109
|
+
if _user_profile:
|
|
110
|
+
post_save.connect(
|
|
111
|
+
on_user_profile_updated_txn, sender=_user_profile
|
|
112
|
+
) # pragma: no cover
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def on_externalid_saved_txn(*args, **kwargs):
|
|
116
|
+
"""
|
|
117
|
+
Handle external_id saves in the middle of a transaction.
|
|
118
|
+
|
|
119
|
+
Handle saves in the middle of a transaction.
|
|
120
|
+
If this gets fired before the transaction commits, the task may try to
|
|
121
|
+
query an id that doesn't exist yet and throw an error. This should postpone
|
|
122
|
+
queuing the Celery task until after the transaction is committed.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
def on_externalid_saved( # pylint: disable=unused-argument # pragma: no cover
|
|
126
|
+
sender, instance, **kwargs
|
|
127
|
+
):
|
|
128
|
+
"""
|
|
129
|
+
Receives post save signal and queues the dump job.
|
|
130
|
+
"""
|
|
131
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
132
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
133
|
+
dump_data_to_clickhouse,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
sink = ExternalIdSink(None, None)
|
|
137
|
+
dump_data_to_clickhouse.delay(
|
|
138
|
+
sink_module=sink.__module__,
|
|
139
|
+
sink_name=sink.__class__.__name__,
|
|
140
|
+
object_id=str(instance.id),
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
transaction.on_commit(lambda: on_externalid_saved(*args, **kwargs))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# Connect the ExternalId.post_save signal handler only if we have a model to attach to.
|
|
147
|
+
# (prevents celery errors during tests)
|
|
148
|
+
_external_id = get_model("external_id")
|
|
149
|
+
if _external_id:
|
|
150
|
+
post_save.connect(on_externalid_saved_txn, sender=_external_id) # pragma: no cover
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@receiver(USER_RETIRE_LMS_MISC)
|
|
154
|
+
def on_user_retirement( # pylint: disable=unused-argument # pragma: no cover
|
|
155
|
+
sender, user, **kwargs
|
|
156
|
+
):
|
|
157
|
+
"""
|
|
158
|
+
Receives a user retirement signal and queues the retire_user job.
|
|
159
|
+
"""
|
|
160
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
161
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
162
|
+
dump_data_to_clickhouse,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
sink = UserRetirementSink(None, None)
|
|
166
|
+
dump_data_to_clickhouse.delay(
|
|
167
|
+
sink_module=sink.__module__,
|
|
168
|
+
sink_name=sink.__class__.__name__,
|
|
169
|
+
object_id=str(user.id),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def on_tag_saved_txn(*args, **kwargs):
|
|
174
|
+
"""
|
|
175
|
+
Handle external_id saves in the middle of a transaction.
|
|
176
|
+
|
|
177
|
+
Handle saves in the middle of a transaction.
|
|
178
|
+
If this gets fired before the transaction commits, the task may try to
|
|
179
|
+
query an id that doesn't exist yet and throw an error. This should postpone
|
|
180
|
+
queuing the Celery task until after the transaction is committed.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
def on_tag_saved( # pylint: disable=unused-argument # pragma: no cover
|
|
184
|
+
sender, instance, **kwargs
|
|
185
|
+
):
|
|
186
|
+
"""
|
|
187
|
+
Receives post save signal and queues the dump job.
|
|
188
|
+
"""
|
|
189
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
190
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
191
|
+
dump_data_to_clickhouse,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
sink = TagSink(None, None)
|
|
195
|
+
dump_data_to_clickhouse.delay(
|
|
196
|
+
sink_module=sink.__module__,
|
|
197
|
+
sink_name=sink.__class__.__name__,
|
|
198
|
+
object_id=str(instance.id),
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
transaction.on_commit(lambda: on_tag_saved(*args, **kwargs))
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# Connect the ExternalId.post_save signal handler only if we have a model to attach to.
|
|
205
|
+
# (prevents celery errors during tests)
|
|
206
|
+
_tag = get_model("tag")
|
|
207
|
+
if _tag:
|
|
208
|
+
post_save.connect(on_tag_saved_txn, sender=_tag) # pragma: no cover
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def on_taxonomy_saved_txn(*args, **kwargs):
|
|
212
|
+
"""
|
|
213
|
+
Handle external_id saves in the middle of a transaction.
|
|
214
|
+
|
|
215
|
+
Handle saves in the middle of a transaction.
|
|
216
|
+
If this gets fired before the transaction commits, the task may try to
|
|
217
|
+
query an id that doesn't exist yet and throw an error. This should postpone
|
|
218
|
+
queuing the Celery task until after the transaction is committed.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
def on_taxonomy_saved( # pylint: disable=unused-argument # pragma: no cover
|
|
222
|
+
sender, instance, **kwargs
|
|
223
|
+
):
|
|
224
|
+
"""
|
|
225
|
+
Receives post save signal and queues the dump job.
|
|
226
|
+
"""
|
|
227
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
228
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
229
|
+
dump_data_to_clickhouse,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
sink = TaxonomySink(None, None)
|
|
233
|
+
dump_data_to_clickhouse.delay(
|
|
234
|
+
sink_module=sink.__module__,
|
|
235
|
+
sink_name=sink.__class__.__name__,
|
|
236
|
+
object_id=str(instance.id),
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
transaction.on_commit(lambda: on_taxonomy_saved(*args, **kwargs))
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# Connect the ExternalId.post_save signal handler only if we have a model to attach to.
|
|
243
|
+
# (prevents celery errors during tests)
|
|
244
|
+
_taxonomy = get_model("taxonomy")
|
|
245
|
+
if _taxonomy:
|
|
246
|
+
post_save.connect(on_taxonomy_saved_txn, sender=_taxonomy) # pragma: no cover
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def on_object_tag_saved_txn(*args, **kwargs):
|
|
250
|
+
"""
|
|
251
|
+
Handle external_id saves in the middle of a transaction.
|
|
252
|
+
|
|
253
|
+
Handle saves in the middle of a transaction.
|
|
254
|
+
If this gets fired before the transaction commits, the task may try to
|
|
255
|
+
query an id that doesn't exist yet and throw an error. This should postpone
|
|
256
|
+
queuing the Celery task until after the transaction is committed.
|
|
257
|
+
"""
|
|
258
|
+
|
|
259
|
+
def on_object_tag_saved(sender, instance, **kwargs): # pragma: no cover
|
|
260
|
+
"""
|
|
261
|
+
Receives post save signal and queues the dump job.
|
|
262
|
+
"""
|
|
263
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
264
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
265
|
+
dump_data_to_clickhouse,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
sink = ObjectTagSink(None, None)
|
|
269
|
+
dump_data_to_clickhouse.delay(
|
|
270
|
+
sink_module=sink.__module__,
|
|
271
|
+
sink_name=sink.__class__.__name__,
|
|
272
|
+
object_id=str(instance.id),
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
on_object_tag_deleted(sender, instance, **kwargs)
|
|
276
|
+
|
|
277
|
+
transaction.on_commit(lambda: on_object_tag_saved(*args, **kwargs))
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def on_object_tag_deleted( # pylint: disable=unused-argument # pragma: no cover
|
|
281
|
+
sender, instance, **kwargs
|
|
282
|
+
):
|
|
283
|
+
"""
|
|
284
|
+
Receives post save signal and queues the dump job.
|
|
285
|
+
"""
|
|
286
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
287
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
288
|
+
dump_course_to_clickhouse,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
CourseOverview = get_model("course_overviews")
|
|
292
|
+
if CourseOverview:
|
|
293
|
+
try:
|
|
294
|
+
CourseOverview.objects.get(id=instance.object_id)
|
|
295
|
+
dump_course_to_clickhouse.delay(instance.object_id)
|
|
296
|
+
except (CourseOverview.DoesNotExist, InvalidKeyError):
|
|
297
|
+
pass
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
# Connect the ExternalId.post_save signal handler only if we have a model to attach to.
|
|
301
|
+
# (prevents celery errors during tests)
|
|
302
|
+
_object_tag = get_model("object_tag")
|
|
303
|
+
if _object_tag: # pragma: no cover
|
|
304
|
+
post_save.connect(on_object_tag_saved_txn, sender=_object_tag)
|
|
305
|
+
post_delete.connect(on_object_tag_deleted, sender=_object_tag)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module contains the sinks for the platform plugin aspects.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .base_sink import BaseSink, ModelBaseSink
|
|
6
|
+
from .course_enrollment_sink import CourseEnrollmentSink
|
|
7
|
+
from .course_overview_sink import CourseOverviewSink, XBlockSink
|
|
8
|
+
from .external_id_sink import ExternalIdSink
|
|
9
|
+
from .tag_sink import ObjectTagSink, TagSink, TaxonomySink
|
|
10
|
+
from .user_profile_sink import UserProfileSink
|
|
11
|
+
from .user_retire_sink import UserRetirementSink
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base classes for event sinks
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import datetime
|
|
7
|
+
import io
|
|
8
|
+
import logging
|
|
9
|
+
from collections import namedtuple
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
from django.conf import settings
|
|
13
|
+
from django.core.paginator import Paginator
|
|
14
|
+
from edx_toggles.toggles import WaffleFlag
|
|
15
|
+
|
|
16
|
+
from platform_plugin_aspects.utils import get_model
|
|
17
|
+
from platform_plugin_aspects.waffle import WAFFLE_FLAG_NAMESPACE
|
|
18
|
+
|
|
19
|
+
ClickHouseAuth = namedtuple("ClickHouseAuth", ["username", "password"])
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BaseSink:
|
|
23
|
+
"""
|
|
24
|
+
Base class for ClickHouse event sink, allows overwriting of default settings
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
CLICKHOUSE_BULK_INSERT_PARAMS = {
|
|
28
|
+
"input_format_allow_errors_num": 1,
|
|
29
|
+
"input_format_allow_errors_ratio": 0.1,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
def __init__(self, connection_overrides, log):
|
|
33
|
+
self.connection_overrides = connection_overrides
|
|
34
|
+
self.log = log
|
|
35
|
+
self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
|
|
36
|
+
self.ch_auth = ClickHouseAuth(
|
|
37
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
|
|
38
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
|
|
39
|
+
)
|
|
40
|
+
self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
|
|
41
|
+
self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
|
|
42
|
+
"timeout_secs"
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
# If any overrides to the ClickHouse connection
|
|
46
|
+
if connection_overrides:
|
|
47
|
+
self.ch_url = connection_overrides.get("url", self.ch_url)
|
|
48
|
+
self.ch_auth = ClickHouseAuth(
|
|
49
|
+
connection_overrides.get("username", self.ch_auth.username),
|
|
50
|
+
connection_overrides.get("password", self.ch_auth.password),
|
|
51
|
+
)
|
|
52
|
+
self.ch_database = connection_overrides.get("database", self.ch_database)
|
|
53
|
+
self.ch_timeout_secs = connection_overrides.get(
|
|
54
|
+
"timeout_secs", self.ch_timeout_secs
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def _send_clickhouse_request(self, request):
|
|
58
|
+
"""
|
|
59
|
+
Perform the actual HTTP requests to ClickHouse.
|
|
60
|
+
"""
|
|
61
|
+
session = requests.Session()
|
|
62
|
+
prepared_request = request.prepare()
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
response = session.send(prepared_request, timeout=self.ch_timeout_secs)
|
|
66
|
+
response.raise_for_status()
|
|
67
|
+
return response
|
|
68
|
+
except requests.exceptions.HTTPError as e:
|
|
69
|
+
self.log.error(str(e))
|
|
70
|
+
self.log.error(e.response.headers)
|
|
71
|
+
self.log.error(e.response)
|
|
72
|
+
self.log.error(e.response.text)
|
|
73
|
+
raise
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ModelBaseSink(BaseSink):
|
|
77
|
+
"""
|
|
78
|
+
Base class for ClickHouse event sink, allows overwriting of default settings
|
|
79
|
+
|
|
80
|
+
This class is used for the model based event sink, which uses the Django ORM to write
|
|
81
|
+
events to ClickHouse.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
unique_key = None
|
|
85
|
+
"""
|
|
86
|
+
str: A unique identifier key used to distinguish between different instances of the sink.
|
|
87
|
+
It can be used to specify the uniqueness constraint when writing events to ClickHouse.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
clickhouse_table_name = None
|
|
91
|
+
"""
|
|
92
|
+
str: The name of the ClickHouse table where the events will be written.
|
|
93
|
+
This should be set to the desired table name for the specific event type.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
queryset = None
|
|
97
|
+
"""
|
|
98
|
+
QuerySet: A Django QuerySet that represents the initial set of data to be processed by the sink.
|
|
99
|
+
It can be used to filter and select specific data for writing to ClickHouse.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
name = None
|
|
103
|
+
"""
|
|
104
|
+
str: A human-readable name for the sink instance. This can be used for logging and identification purposes.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
timestamp_field = None
|
|
108
|
+
"""
|
|
109
|
+
str: The name of the field in the model representing the timestamp of the event.
|
|
110
|
+
It is used to extract the timestamp from the event data for writing to ClickHouse.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
serializer_class = None
|
|
114
|
+
"""
|
|
115
|
+
Serializer: The serializer class responsible for converting event data into a format suitable for storage.
|
|
116
|
+
This serializer should be compatible with Django's serialization framework.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
model = None
|
|
120
|
+
"""
|
|
121
|
+
Model: The Django model class representing the structure of the event data.
|
|
122
|
+
This is used to validate and organize the data before writing it to ClickHouse.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
nested_sinks = []
|
|
126
|
+
"""
|
|
127
|
+
list: A list of nested sink instances that can be used to further process or route the event data.
|
|
128
|
+
Nested sinks allow chaining multiple sinks together for more complex event processing pipelines.
|
|
129
|
+
"""
|
|
130
|
+
pk_format = int
|
|
131
|
+
"""
|
|
132
|
+
function: A function to format the primary key of the model
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
def __init__(self, connection_overrides, log):
|
|
136
|
+
super().__init__(connection_overrides, log)
|
|
137
|
+
|
|
138
|
+
required_fields = [
|
|
139
|
+
self.clickhouse_table_name,
|
|
140
|
+
self.timestamp_field,
|
|
141
|
+
self.unique_key,
|
|
142
|
+
self.name,
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
if not all(required_fields):
|
|
146
|
+
raise NotImplementedError(
|
|
147
|
+
"ModelBaseSink needs to be subclassed with clickhouse_table_name,"
|
|
148
|
+
"timestamp_field, unique_key, and name"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
self._nested_sinks = [
|
|
152
|
+
sink(connection_overrides, log) for sink in self.nested_sinks
|
|
153
|
+
]
|
|
154
|
+
|
|
155
|
+
def get_model(self):
|
|
156
|
+
"""
|
|
157
|
+
Return the model to be used for the insert
|
|
158
|
+
"""
|
|
159
|
+
return get_model(self.model)
|
|
160
|
+
|
|
161
|
+
def get_queryset(self, start_pk=None):
|
|
162
|
+
"""
|
|
163
|
+
Return the queryset to be used for the insert
|
|
164
|
+
"""
|
|
165
|
+
if start_pk:
|
|
166
|
+
start_pk = self.pk_format(start_pk)
|
|
167
|
+
return self.get_model().objects.filter(pk__gt=start_pk).order_by("pk")
|
|
168
|
+
else:
|
|
169
|
+
return self.get_model().objects.all().order_by("pk")
|
|
170
|
+
|
|
171
|
+
def dump(self, item_id, many=False, initial=None):
|
|
172
|
+
"""
|
|
173
|
+
Do the serialization and send to ClickHouse
|
|
174
|
+
"""
|
|
175
|
+
if many:
|
|
176
|
+
# If we're dumping many items, we expect to get a list of items
|
|
177
|
+
serialized_item = self.serialize_item(item_id, many=many, initial=initial)
|
|
178
|
+
self.log.info(
|
|
179
|
+
f"Now dumping {len(serialized_item)} {self.name} to ClickHouse",
|
|
180
|
+
)
|
|
181
|
+
self.send_item_and_log(item_id, serialized_item, many)
|
|
182
|
+
self.log.info(
|
|
183
|
+
f"Completed dumping {len(serialized_item)} {self.name} to ClickHouse"
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
for item in serialized_item:
|
|
187
|
+
for nested_sink in self._nested_sinks:
|
|
188
|
+
nested_sink.dump_related(
|
|
189
|
+
item, item["dump_id"], item["time_last_dumped"]
|
|
190
|
+
)
|
|
191
|
+
else:
|
|
192
|
+
item = self.get_object(item_id)
|
|
193
|
+
serialized_item = self.serialize_item(item, many=many, initial=initial)
|
|
194
|
+
self.log.info(
|
|
195
|
+
f"Now dumping {self.name} {item_id} to ClickHouse",
|
|
196
|
+
)
|
|
197
|
+
self.send_item_and_log(item_id, serialized_item, many)
|
|
198
|
+
self.log.info(f"Completed dumping {self.name} {item_id} to ClickHouse")
|
|
199
|
+
|
|
200
|
+
for nested_sink in self._nested_sinks:
|
|
201
|
+
nested_sink.dump_related(
|
|
202
|
+
serialized_item,
|
|
203
|
+
serialized_item["dump_id"],
|
|
204
|
+
serialized_item["time_last_dumped"],
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
def send_item_and_log(
|
|
208
|
+
self,
|
|
209
|
+
item_id,
|
|
210
|
+
serialized_item,
|
|
211
|
+
many,
|
|
212
|
+
):
|
|
213
|
+
"""Send the item to clickhouse and log any errors"""
|
|
214
|
+
try:
|
|
215
|
+
self.send_item(serialized_item, many=many)
|
|
216
|
+
except Exception:
|
|
217
|
+
self.log.exception(
|
|
218
|
+
f"Error trying to dump {self.name} {str(item_id)} to ClickHouse!",
|
|
219
|
+
)
|
|
220
|
+
raise
|
|
221
|
+
|
|
222
|
+
def get_object(self, item_id):
|
|
223
|
+
"""
|
|
224
|
+
Return the object to be dumped to ClickHouse
|
|
225
|
+
"""
|
|
226
|
+
return self.get_model().objects.get(id=item_id)
|
|
227
|
+
|
|
228
|
+
def dump_related(self, serialized_item, dump_id, time_last_dumped):
|
|
229
|
+
"""
|
|
230
|
+
Dump related items to ClickHouse
|
|
231
|
+
"""
|
|
232
|
+
raise NotImplementedError(
|
|
233
|
+
"dump_related needs to be implemented in the subclass"
|
|
234
|
+
f"{self.__class__.__name__}!"
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
def serialize_item(self, item, many=False, initial=None):
|
|
238
|
+
"""
|
|
239
|
+
Serialize the data to be sent to ClickHouse
|
|
240
|
+
"""
|
|
241
|
+
Serializer = self.get_serializer()
|
|
242
|
+
serializer = Serializer( # pylint: disable=not-callable
|
|
243
|
+
item, many=many, initial=initial
|
|
244
|
+
)
|
|
245
|
+
return serializer.data
|
|
246
|
+
|
|
247
|
+
def get_serializer(self):
|
|
248
|
+
"""
|
|
249
|
+
Return the serializer to be used for the insert
|
|
250
|
+
"""
|
|
251
|
+
return self.serializer_class
|
|
252
|
+
|
|
253
|
+
def send_item(self, serialized_item, many=False):
|
|
254
|
+
"""
|
|
255
|
+
Create the insert query and CSV to send the serialized CourseOverview to ClickHouse.
|
|
256
|
+
|
|
257
|
+
We still use a CSV here even though there's only 1 row because it affords handles
|
|
258
|
+
type serialization for us and keeps the pattern consistent.
|
|
259
|
+
"""
|
|
260
|
+
params = self.CLICKHOUSE_BULK_INSERT_PARAMS.copy()
|
|
261
|
+
|
|
262
|
+
# "query" is a special param for the query, it's the best way to get the FORMAT CSV in there.
|
|
263
|
+
params["query"] = (
|
|
264
|
+
f"INSERT INTO {self.ch_database}.{self.clickhouse_table_name} FORMAT CSV"
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
output = io.StringIO()
|
|
268
|
+
writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
|
|
269
|
+
|
|
270
|
+
if many:
|
|
271
|
+
for node in serialized_item:
|
|
272
|
+
writer.writerow(node.values())
|
|
273
|
+
else:
|
|
274
|
+
writer.writerow(serialized_item.values())
|
|
275
|
+
|
|
276
|
+
request = requests.Request(
|
|
277
|
+
"POST",
|
|
278
|
+
self.ch_url,
|
|
279
|
+
data=output.getvalue().encode("utf-8"),
|
|
280
|
+
params=params,
|
|
281
|
+
auth=self.ch_auth,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
self._send_clickhouse_request(request)
|
|
285
|
+
|
|
286
|
+
def fetch_target_items(
|
|
287
|
+
self, start_pk=None, ids=None, skip_ids=None, force_dump=False, batch_size=None
|
|
288
|
+
):
|
|
289
|
+
"""
|
|
290
|
+
Fetch the items that should be dumped to ClickHouse
|
|
291
|
+
"""
|
|
292
|
+
queryset = self.get_queryset(start_pk)
|
|
293
|
+
if ids:
|
|
294
|
+
ids = [self.pk_format(id) for id in ids]
|
|
295
|
+
queryset = queryset.filter(pk__in=ids)
|
|
296
|
+
|
|
297
|
+
if skip_ids:
|
|
298
|
+
skip_ids = [self.pk_format(id) for id in skip_ids]
|
|
299
|
+
queryset = queryset.exclude(pk__in=skip_ids)
|
|
300
|
+
|
|
301
|
+
paginator = Paginator(queryset, batch_size)
|
|
302
|
+
for i in range(1, paginator.num_pages + 1):
|
|
303
|
+
page = paginator.page(i)
|
|
304
|
+
items = page.object_list
|
|
305
|
+
for item in items:
|
|
306
|
+
if force_dump:
|
|
307
|
+
yield item, True, "Force is set"
|
|
308
|
+
else:
|
|
309
|
+
should_be_dumped, reason = self.should_dump_item(item)
|
|
310
|
+
yield item, should_be_dumped, reason
|
|
311
|
+
|
|
312
|
+
def should_dump_item(self, item): # pylint: disable=unused-argument
|
|
313
|
+
"""
|
|
314
|
+
Return True if the item should be dumped to ClickHouse, False otherwise
|
|
315
|
+
"""
|
|
316
|
+
return True, "No reason"
|
|
317
|
+
|
|
318
|
+
def get_last_dumped_timestamp(self, item_id):
|
|
319
|
+
"""
|
|
320
|
+
Return the last timestamp that was dumped to ClickHouse
|
|
321
|
+
"""
|
|
322
|
+
params = {
|
|
323
|
+
"query": f"SELECT max({self.timestamp_field}) as time_last_dumped "
|
|
324
|
+
f"FROM {self.ch_database}.{self.clickhouse_table_name} "
|
|
325
|
+
f"WHERE {self.unique_key} = '{item_id}'"
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
request = requests.Request("GET", self.ch_url, params=params, auth=self.ch_auth)
|
|
329
|
+
|
|
330
|
+
response = self._send_clickhouse_request(request)
|
|
331
|
+
response.raise_for_status()
|
|
332
|
+
if response.text.strip():
|
|
333
|
+
# ClickHouse returns timestamps in the format: "2023-05-03 15:47:39.331024+00:00"
|
|
334
|
+
# Our internal comparisons use the str() of a datetime object, this handles that
|
|
335
|
+
# transformation so that downstream comparisons will work.
|
|
336
|
+
return str(datetime.datetime.fromisoformat(response.text.strip()))
|
|
337
|
+
|
|
338
|
+
# Item has never been dumped, return None
|
|
339
|
+
return None
|
|
340
|
+
|
|
341
|
+
@classmethod
|
|
342
|
+
def is_enabled(cls):
|
|
343
|
+
"""
|
|
344
|
+
Return True if the sink is enabled, False otherwise
|
|
345
|
+
"""
|
|
346
|
+
enabled_settings = getattr(
|
|
347
|
+
settings,
|
|
348
|
+
f"{WAFFLE_FLAG_NAMESPACE.upper()}_{cls.model.upper()}_ENABLED",
|
|
349
|
+
False,
|
|
350
|
+
)
|
|
351
|
+
# .. toggle_name: event_sink_clickhouse.model.enabled
|
|
352
|
+
# .. toggle_implementation: WaffleFlag
|
|
353
|
+
# .. toggle_default: False
|
|
354
|
+
# .. toggle_description: Waffle flag to enable sink
|
|
355
|
+
# .. toggle_use_cases: open_edx
|
|
356
|
+
# .. toggle_creation_date: 2022-08-17
|
|
357
|
+
waffle_flag = WaffleFlag(
|
|
358
|
+
f"{WAFFLE_FLAG_NAMESPACE}.{cls.model}.enabled",
|
|
359
|
+
__name__,
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
enabled = enabled_settings or waffle_flag.is_enabled()
|
|
363
|
+
|
|
364
|
+
if enabled and not get_model(cls.model):
|
|
365
|
+
logging.warning(
|
|
366
|
+
f"Event Sink Model {cls.model} is not installed, but is enabled in settings or waffle flag."
|
|
367
|
+
)
|
|
368
|
+
enabled = False
|
|
369
|
+
|
|
370
|
+
return enabled
|
|
371
|
+
|
|
372
|
+
@classmethod
|
|
373
|
+
def get_sink_by_model_name(cls, model):
|
|
374
|
+
"""
|
|
375
|
+
Return the sink instance for the given model
|
|
376
|
+
"""
|
|
377
|
+
for sink in cls.__subclasses__():
|
|
378
|
+
if sink.model == model:
|
|
379
|
+
return sink
|
|
380
|
+
|
|
381
|
+
return None
|