platform-plugin-aspects 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. platform_plugin_aspects/__init__.py +11 -0
  2. platform_plugin_aspects/apps.py +77 -0
  3. platform_plugin_aspects/conf/locale/babel_mako.cfg +10 -0
  4. platform_plugin_aspects/conf/locale/config.yaml +85 -0
  5. platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
  6. platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +99 -0
  7. platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
  8. platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +23 -0
  9. platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
  10. platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +70 -0
  11. platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
  12. platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +23 -0
  13. platform_plugin_aspects/extensions/__init__.py +0 -0
  14. platform_plugin_aspects/extensions/filters.py +108 -0
  15. platform_plugin_aspects/management/__init__.py +0 -0
  16. platform_plugin_aspects/management/commands/__init__.py +0 -0
  17. platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +214 -0
  18. platform_plugin_aspects/management/commands/load_test_tracking_events.py +310 -0
  19. platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +440 -0
  20. platform_plugin_aspects/settings/__init__.py +0 -0
  21. platform_plugin_aspects/settings/common.py +147 -0
  22. platform_plugin_aspects/settings/production.py +38 -0
  23. platform_plugin_aspects/signals.py +305 -0
  24. platform_plugin_aspects/sinks/__init__.py +11 -0
  25. platform_plugin_aspects/sinks/base_sink.py +381 -0
  26. platform_plugin_aspects/sinks/course_enrollment_sink.py +20 -0
  27. platform_plugin_aspects/sinks/course_overview_sink.py +286 -0
  28. platform_plugin_aspects/sinks/external_id_sink.py +20 -0
  29. platform_plugin_aspects/sinks/serializers.py +274 -0
  30. platform_plugin_aspects/sinks/tag_sink.py +47 -0
  31. platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
  32. platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
  33. platform_plugin_aspects/static/css/superset.css +53 -0
  34. platform_plugin_aspects/static/html/superset.html +39 -0
  35. platform_plugin_aspects/static/html/superset_student.html +5 -0
  36. platform_plugin_aspects/static/js/embed_dashboard.js +94 -0
  37. platform_plugin_aspects/static/js/install_required.js +11 -0
  38. platform_plugin_aspects/static/js/superset.js +38 -0
  39. platform_plugin_aspects/tasks.py +64 -0
  40. platform_plugin_aspects/templates/instructor_dashboard/aspects.html +22 -0
  41. platform_plugin_aspects/urls.py +39 -0
  42. platform_plugin_aspects/utils.py +359 -0
  43. platform_plugin_aspects/views.py +296 -0
  44. platform_plugin_aspects/waffle.py +5 -0
  45. platform_plugin_aspects/xblock.py +239 -0
  46. platform_plugin_aspects-2.0.1.dist-info/METADATA +335 -0
  47. platform_plugin_aspects-2.0.1.dist-info/RECORD +51 -0
  48. platform_plugin_aspects-2.0.1.dist-info/WHEEL +5 -0
  49. platform_plugin_aspects-2.0.1.dist-info/entry_points.txt +8 -0
  50. platform_plugin_aspects-2.0.1.dist-info/licenses/LICENSE +201 -0
  51. platform_plugin_aspects-2.0.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,305 @@
1
+ """
2
+ Signal handler functions, mapped to specific signals in apps.py.
3
+ """
4
+
5
+ from django.db import transaction
6
+ from django.db.models.signals import post_delete, post_save
7
+ from django.dispatch import Signal, receiver
8
+ from opaque_keys import InvalidKeyError
9
+
10
+ from platform_plugin_aspects.sinks import (
11
+ CourseEnrollmentSink,
12
+ ExternalIdSink,
13
+ ObjectTagSink,
14
+ TagSink,
15
+ TaxonomySink,
16
+ UserProfileSink,
17
+ UserRetirementSink,
18
+ )
19
+ from platform_plugin_aspects.utils import get_model
20
+
21
+ try:
22
+ from openedx.core.djangoapps.user_api.accounts.signals import USER_RETIRE_LMS_MISC
23
+ except ImportError:
24
+ # Tests don't have the platform installed
25
+ USER_RETIRE_LMS_MISC = Signal()
26
+
27
+
28
+ def receive_course_publish( # pylint: disable=unused-argument # pragma: no cover
29
+ sender, course_key, **kwargs
30
+ ):
31
+ """
32
+ Receives COURSE_PUBLISHED signal and queues the dump job.
33
+ """
34
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
35
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
36
+ dump_course_to_clickhouse,
37
+ )
38
+
39
+ dump_course_to_clickhouse.delay(str(course_key))
40
+
41
+
42
+ def receive_course_enrollment_changed( # pylint: disable=unused-argument # pragma: no cover
43
+ sender, **kwargs
44
+ ):
45
+ """
46
+ Receives ENROLL_STATUS_CHANGE signal and queues the dump job.
47
+
48
+ Handles the course enrollment dumping in the middle of a transaction.
49
+ If this gets fired before the transaction commits, the task may try to query
50
+ an id that doesn't exist yet and throw an error. This should postpone
51
+ queuing the Celery task until after the transaction is committed.
52
+ """
53
+
54
+ def on_course_enrollment_changed(**kwargs):
55
+ """
56
+ Queues the CourseEnrollment dump job when the parent transaction is committed.
57
+ """
58
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
59
+ dump_data_to_clickhouse,
60
+ )
61
+
62
+ user = kwargs.get("user")
63
+ course_id = kwargs.get("course_id")
64
+ CourseEnrollment = get_model("course_enrollment")
65
+ instance = CourseEnrollment.objects.get(user=user, course_id=course_id)
66
+
67
+ sink = CourseEnrollmentSink(None, None)
68
+ dump_data_to_clickhouse.delay(
69
+ sink_module=sink.__module__,
70
+ sink_name=sink.__class__.__name__,
71
+ object_id=str(instance.id),
72
+ )
73
+
74
+ transaction.on_commit(lambda: on_course_enrollment_changed(**kwargs))
75
+
76
+
77
+ def on_user_profile_updated_txn(*args, **kwargs):
78
+ """
79
+ Handle user_profile saves in the middle of a transaction.
80
+
81
+ Handle saves in the middle of a transaction.
82
+ If this gets fired before the transaction commits, the task may try to
83
+ query an id that doesn't exist yet and throw an error. This should postpone
84
+ queuing the Celery task until after the transaction is committed.
85
+ """
86
+
87
+ def on_user_profile_updated(instance, **kwargs):
88
+ """
89
+ Queues the UserProfile dump job when the parent transaction is committed.
90
+ """
91
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
92
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
93
+ dump_data_to_clickhouse,
94
+ )
95
+
96
+ sink = UserProfileSink(None, None)
97
+ dump_data_to_clickhouse.delay(
98
+ sink_module=sink.__module__,
99
+ sink_name=sink.__class__.__name__,
100
+ object_id=str(instance.id),
101
+ )
102
+
103
+ transaction.on_commit(lambda: on_user_profile_updated(*args, **kwargs))
104
+
105
+
106
+ # Connect the UserProfile.post_save signal handler only if we have a model to attach to.
107
+ # (prevents celery errors during tests)
108
+ _user_profile = get_model("user_profile")
109
+ if _user_profile:
110
+ post_save.connect(
111
+ on_user_profile_updated_txn, sender=_user_profile
112
+ ) # pragma: no cover
113
+
114
+
115
+ def on_externalid_saved_txn(*args, **kwargs):
116
+ """
117
+ Handle external_id saves in the middle of a transaction.
118
+
119
+ Handle saves in the middle of a transaction.
120
+ If this gets fired before the transaction commits, the task may try to
121
+ query an id that doesn't exist yet and throw an error. This should postpone
122
+ queuing the Celery task until after the transaction is committed.
123
+ """
124
+
125
+ def on_externalid_saved( # pylint: disable=unused-argument # pragma: no cover
126
+ sender, instance, **kwargs
127
+ ):
128
+ """
129
+ Receives post save signal and queues the dump job.
130
+ """
131
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
132
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
133
+ dump_data_to_clickhouse,
134
+ )
135
+
136
+ sink = ExternalIdSink(None, None)
137
+ dump_data_to_clickhouse.delay(
138
+ sink_module=sink.__module__,
139
+ sink_name=sink.__class__.__name__,
140
+ object_id=str(instance.id),
141
+ )
142
+
143
+ transaction.on_commit(lambda: on_externalid_saved(*args, **kwargs))
144
+
145
+
146
+ # Connect the ExternalId.post_save signal handler only if we have a model to attach to.
147
+ # (prevents celery errors during tests)
148
+ _external_id = get_model("external_id")
149
+ if _external_id:
150
+ post_save.connect(on_externalid_saved_txn, sender=_external_id) # pragma: no cover
151
+
152
+
153
+ @receiver(USER_RETIRE_LMS_MISC)
154
+ def on_user_retirement( # pylint: disable=unused-argument # pragma: no cover
155
+ sender, user, **kwargs
156
+ ):
157
+ """
158
+ Receives a user retirement signal and queues the retire_user job.
159
+ """
160
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
161
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
162
+ dump_data_to_clickhouse,
163
+ )
164
+
165
+ sink = UserRetirementSink(None, None)
166
+ dump_data_to_clickhouse.delay(
167
+ sink_module=sink.__module__,
168
+ sink_name=sink.__class__.__name__,
169
+ object_id=str(user.id),
170
+ )
171
+
172
+
173
+ def on_tag_saved_txn(*args, **kwargs):
174
+ """
175
+ Handle external_id saves in the middle of a transaction.
176
+
177
+ Handle saves in the middle of a transaction.
178
+ If this gets fired before the transaction commits, the task may try to
179
+ query an id that doesn't exist yet and throw an error. This should postpone
180
+ queuing the Celery task until after the transaction is committed.
181
+ """
182
+
183
+ def on_tag_saved( # pylint: disable=unused-argument # pragma: no cover
184
+ sender, instance, **kwargs
185
+ ):
186
+ """
187
+ Receives post save signal and queues the dump job.
188
+ """
189
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
190
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
191
+ dump_data_to_clickhouse,
192
+ )
193
+
194
+ sink = TagSink(None, None)
195
+ dump_data_to_clickhouse.delay(
196
+ sink_module=sink.__module__,
197
+ sink_name=sink.__class__.__name__,
198
+ object_id=str(instance.id),
199
+ )
200
+
201
+ transaction.on_commit(lambda: on_tag_saved(*args, **kwargs))
202
+
203
+
204
+ # Connect the ExternalId.post_save signal handler only if we have a model to attach to.
205
+ # (prevents celery errors during tests)
206
+ _tag = get_model("tag")
207
+ if _tag:
208
+ post_save.connect(on_tag_saved_txn, sender=_tag) # pragma: no cover
209
+
210
+
211
+ def on_taxonomy_saved_txn(*args, **kwargs):
212
+ """
213
+ Handle external_id saves in the middle of a transaction.
214
+
215
+ Handle saves in the middle of a transaction.
216
+ If this gets fired before the transaction commits, the task may try to
217
+ query an id that doesn't exist yet and throw an error. This should postpone
218
+ queuing the Celery task until after the transaction is committed.
219
+ """
220
+
221
+ def on_taxonomy_saved( # pylint: disable=unused-argument # pragma: no cover
222
+ sender, instance, **kwargs
223
+ ):
224
+ """
225
+ Receives post save signal and queues the dump job.
226
+ """
227
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
228
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
229
+ dump_data_to_clickhouse,
230
+ )
231
+
232
+ sink = TaxonomySink(None, None)
233
+ dump_data_to_clickhouse.delay(
234
+ sink_module=sink.__module__,
235
+ sink_name=sink.__class__.__name__,
236
+ object_id=str(instance.id),
237
+ )
238
+
239
+ transaction.on_commit(lambda: on_taxonomy_saved(*args, **kwargs))
240
+
241
+
242
+ # Connect the ExternalId.post_save signal handler only if we have a model to attach to.
243
+ # (prevents celery errors during tests)
244
+ _taxonomy = get_model("taxonomy")
245
+ if _taxonomy:
246
+ post_save.connect(on_taxonomy_saved_txn, sender=_taxonomy) # pragma: no cover
247
+
248
+
249
+ def on_object_tag_saved_txn(*args, **kwargs):
250
+ """
251
+ Handle external_id saves in the middle of a transaction.
252
+
253
+ Handle saves in the middle of a transaction.
254
+ If this gets fired before the transaction commits, the task may try to
255
+ query an id that doesn't exist yet and throw an error. This should postpone
256
+ queuing the Celery task until after the transaction is committed.
257
+ """
258
+
259
+ def on_object_tag_saved(sender, instance, **kwargs): # pragma: no cover
260
+ """
261
+ Receives post save signal and queues the dump job.
262
+ """
263
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
264
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
265
+ dump_data_to_clickhouse,
266
+ )
267
+
268
+ sink = ObjectTagSink(None, None)
269
+ dump_data_to_clickhouse.delay(
270
+ sink_module=sink.__module__,
271
+ sink_name=sink.__class__.__name__,
272
+ object_id=str(instance.id),
273
+ )
274
+
275
+ on_object_tag_deleted(sender, instance, **kwargs)
276
+
277
+ transaction.on_commit(lambda: on_object_tag_saved(*args, **kwargs))
278
+
279
+
280
+ def on_object_tag_deleted( # pylint: disable=unused-argument # pragma: no cover
281
+ sender, instance, **kwargs
282
+ ):
283
+ """
284
+ Receives post save signal and queues the dump job.
285
+ """
286
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
287
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
288
+ dump_course_to_clickhouse,
289
+ )
290
+
291
+ CourseOverview = get_model("course_overviews")
292
+ if CourseOverview:
293
+ try:
294
+ CourseOverview.objects.get(id=instance.object_id)
295
+ dump_course_to_clickhouse.delay(instance.object_id)
296
+ except (CourseOverview.DoesNotExist, InvalidKeyError):
297
+ pass
298
+
299
+
300
+ # Connect the ExternalId.post_save signal handler only if we have a model to attach to.
301
+ # (prevents celery errors during tests)
302
+ _object_tag = get_model("object_tag")
303
+ if _object_tag: # pragma: no cover
304
+ post_save.connect(on_object_tag_saved_txn, sender=_object_tag)
305
+ post_delete.connect(on_object_tag_deleted, sender=_object_tag)
@@ -0,0 +1,11 @@
1
+ """
2
+ This module contains the sinks for the platform plugin aspects.
3
+ """
4
+
5
+ from .base_sink import BaseSink, ModelBaseSink
6
+ from .course_enrollment_sink import CourseEnrollmentSink
7
+ from .course_overview_sink import CourseOverviewSink, XBlockSink
8
+ from .external_id_sink import ExternalIdSink
9
+ from .tag_sink import ObjectTagSink, TagSink, TaxonomySink
10
+ from .user_profile_sink import UserProfileSink
11
+ from .user_retire_sink import UserRetirementSink
@@ -0,0 +1,381 @@
1
+ """
2
+ Base classes for event sinks
3
+ """
4
+
5
+ import csv
6
+ import datetime
7
+ import io
8
+ import logging
9
+ from collections import namedtuple
10
+
11
+ import requests
12
+ from django.conf import settings
13
+ from django.core.paginator import Paginator
14
+ from edx_toggles.toggles import WaffleFlag
15
+
16
+ from platform_plugin_aspects.utils import get_model
17
+ from platform_plugin_aspects.waffle import WAFFLE_FLAG_NAMESPACE
18
+
19
+ ClickHouseAuth = namedtuple("ClickHouseAuth", ["username", "password"])
20
+
21
+
22
+ class BaseSink:
23
+ """
24
+ Base class for ClickHouse event sink, allows overwriting of default settings
25
+ """
26
+
27
+ CLICKHOUSE_BULK_INSERT_PARAMS = {
28
+ "input_format_allow_errors_num": 1,
29
+ "input_format_allow_errors_ratio": 0.1,
30
+ }
31
+
32
+ def __init__(self, connection_overrides, log):
33
+ self.connection_overrides = connection_overrides
34
+ self.log = log
35
+ self.ch_url = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["url"]
36
+ self.ch_auth = ClickHouseAuth(
37
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["username"],
38
+ settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["password"],
39
+ )
40
+ self.ch_database = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG["database"]
41
+ self.ch_timeout_secs = settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG[
42
+ "timeout_secs"
43
+ ]
44
+
45
+ # If any overrides to the ClickHouse connection
46
+ if connection_overrides:
47
+ self.ch_url = connection_overrides.get("url", self.ch_url)
48
+ self.ch_auth = ClickHouseAuth(
49
+ connection_overrides.get("username", self.ch_auth.username),
50
+ connection_overrides.get("password", self.ch_auth.password),
51
+ )
52
+ self.ch_database = connection_overrides.get("database", self.ch_database)
53
+ self.ch_timeout_secs = connection_overrides.get(
54
+ "timeout_secs", self.ch_timeout_secs
55
+ )
56
+
57
+ def _send_clickhouse_request(self, request):
58
+ """
59
+ Perform the actual HTTP requests to ClickHouse.
60
+ """
61
+ session = requests.Session()
62
+ prepared_request = request.prepare()
63
+
64
+ try:
65
+ response = session.send(prepared_request, timeout=self.ch_timeout_secs)
66
+ response.raise_for_status()
67
+ return response
68
+ except requests.exceptions.HTTPError as e:
69
+ self.log.error(str(e))
70
+ self.log.error(e.response.headers)
71
+ self.log.error(e.response)
72
+ self.log.error(e.response.text)
73
+ raise
74
+
75
+
76
+ class ModelBaseSink(BaseSink):
77
+ """
78
+ Base class for ClickHouse event sink, allows overwriting of default settings
79
+
80
+ This class is used for the model based event sink, which uses the Django ORM to write
81
+ events to ClickHouse.
82
+ """
83
+
84
+ unique_key = None
85
+ """
86
+ str: A unique identifier key used to distinguish between different instances of the sink.
87
+ It can be used to specify the uniqueness constraint when writing events to ClickHouse.
88
+ """
89
+
90
+ clickhouse_table_name = None
91
+ """
92
+ str: The name of the ClickHouse table where the events will be written.
93
+ This should be set to the desired table name for the specific event type.
94
+ """
95
+
96
+ queryset = None
97
+ """
98
+ QuerySet: A Django QuerySet that represents the initial set of data to be processed by the sink.
99
+ It can be used to filter and select specific data for writing to ClickHouse.
100
+ """
101
+
102
+ name = None
103
+ """
104
+ str: A human-readable name for the sink instance. This can be used for logging and identification purposes.
105
+ """
106
+
107
+ timestamp_field = None
108
+ """
109
+ str: The name of the field in the model representing the timestamp of the event.
110
+ It is used to extract the timestamp from the event data for writing to ClickHouse.
111
+ """
112
+
113
+ serializer_class = None
114
+ """
115
+ Serializer: The serializer class responsible for converting event data into a format suitable for storage.
116
+ This serializer should be compatible with Django's serialization framework.
117
+ """
118
+
119
+ model = None
120
+ """
121
+ Model: The Django model class representing the structure of the event data.
122
+ This is used to validate and organize the data before writing it to ClickHouse.
123
+ """
124
+
125
+ nested_sinks = []
126
+ """
127
+ list: A list of nested sink instances that can be used to further process or route the event data.
128
+ Nested sinks allow chaining multiple sinks together for more complex event processing pipelines.
129
+ """
130
+ pk_format = int
131
+ """
132
+ function: A function to format the primary key of the model
133
+ """
134
+
135
+ def __init__(self, connection_overrides, log):
136
+ super().__init__(connection_overrides, log)
137
+
138
+ required_fields = [
139
+ self.clickhouse_table_name,
140
+ self.timestamp_field,
141
+ self.unique_key,
142
+ self.name,
143
+ ]
144
+
145
+ if not all(required_fields):
146
+ raise NotImplementedError(
147
+ "ModelBaseSink needs to be subclassed with clickhouse_table_name,"
148
+ "timestamp_field, unique_key, and name"
149
+ )
150
+
151
+ self._nested_sinks = [
152
+ sink(connection_overrides, log) for sink in self.nested_sinks
153
+ ]
154
+
155
+ def get_model(self):
156
+ """
157
+ Return the model to be used for the insert
158
+ """
159
+ return get_model(self.model)
160
+
161
+ def get_queryset(self, start_pk=None):
162
+ """
163
+ Return the queryset to be used for the insert
164
+ """
165
+ if start_pk:
166
+ start_pk = self.pk_format(start_pk)
167
+ return self.get_model().objects.filter(pk__gt=start_pk).order_by("pk")
168
+ else:
169
+ return self.get_model().objects.all().order_by("pk")
170
+
171
+ def dump(self, item_id, many=False, initial=None):
172
+ """
173
+ Do the serialization and send to ClickHouse
174
+ """
175
+ if many:
176
+ # If we're dumping many items, we expect to get a list of items
177
+ serialized_item = self.serialize_item(item_id, many=many, initial=initial)
178
+ self.log.info(
179
+ f"Now dumping {len(serialized_item)} {self.name} to ClickHouse",
180
+ )
181
+ self.send_item_and_log(item_id, serialized_item, many)
182
+ self.log.info(
183
+ f"Completed dumping {len(serialized_item)} {self.name} to ClickHouse"
184
+ )
185
+
186
+ for item in serialized_item:
187
+ for nested_sink in self._nested_sinks:
188
+ nested_sink.dump_related(
189
+ item, item["dump_id"], item["time_last_dumped"]
190
+ )
191
+ else:
192
+ item = self.get_object(item_id)
193
+ serialized_item = self.serialize_item(item, many=many, initial=initial)
194
+ self.log.info(
195
+ f"Now dumping {self.name} {item_id} to ClickHouse",
196
+ )
197
+ self.send_item_and_log(item_id, serialized_item, many)
198
+ self.log.info(f"Completed dumping {self.name} {item_id} to ClickHouse")
199
+
200
+ for nested_sink in self._nested_sinks:
201
+ nested_sink.dump_related(
202
+ serialized_item,
203
+ serialized_item["dump_id"],
204
+ serialized_item["time_last_dumped"],
205
+ )
206
+
207
+ def send_item_and_log(
208
+ self,
209
+ item_id,
210
+ serialized_item,
211
+ many,
212
+ ):
213
+ """Send the item to clickhouse and log any errors"""
214
+ try:
215
+ self.send_item(serialized_item, many=many)
216
+ except Exception:
217
+ self.log.exception(
218
+ f"Error trying to dump {self.name} {str(item_id)} to ClickHouse!",
219
+ )
220
+ raise
221
+
222
+ def get_object(self, item_id):
223
+ """
224
+ Return the object to be dumped to ClickHouse
225
+ """
226
+ return self.get_model().objects.get(id=item_id)
227
+
228
+ def dump_related(self, serialized_item, dump_id, time_last_dumped):
229
+ """
230
+ Dump related items to ClickHouse
231
+ """
232
+ raise NotImplementedError(
233
+ "dump_related needs to be implemented in the subclass"
234
+ f"{self.__class__.__name__}!"
235
+ )
236
+
237
+ def serialize_item(self, item, many=False, initial=None):
238
+ """
239
+ Serialize the data to be sent to ClickHouse
240
+ """
241
+ Serializer = self.get_serializer()
242
+ serializer = Serializer( # pylint: disable=not-callable
243
+ item, many=many, initial=initial
244
+ )
245
+ return serializer.data
246
+
247
+ def get_serializer(self):
248
+ """
249
+ Return the serializer to be used for the insert
250
+ """
251
+ return self.serializer_class
252
+
253
+ def send_item(self, serialized_item, many=False):
254
+ """
255
+ Create the insert query and CSV to send the serialized CourseOverview to ClickHouse.
256
+
257
+ We still use a CSV here even though there's only 1 row because it affords handles
258
+ type serialization for us and keeps the pattern consistent.
259
+ """
260
+ params = self.CLICKHOUSE_BULK_INSERT_PARAMS.copy()
261
+
262
+ # "query" is a special param for the query, it's the best way to get the FORMAT CSV in there.
263
+ params["query"] = (
264
+ f"INSERT INTO {self.ch_database}.{self.clickhouse_table_name} FORMAT CSV"
265
+ )
266
+
267
+ output = io.StringIO()
268
+ writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
269
+
270
+ if many:
271
+ for node in serialized_item:
272
+ writer.writerow(node.values())
273
+ else:
274
+ writer.writerow(serialized_item.values())
275
+
276
+ request = requests.Request(
277
+ "POST",
278
+ self.ch_url,
279
+ data=output.getvalue().encode("utf-8"),
280
+ params=params,
281
+ auth=self.ch_auth,
282
+ )
283
+
284
+ self._send_clickhouse_request(request)
285
+
286
+ def fetch_target_items(
287
+ self, start_pk=None, ids=None, skip_ids=None, force_dump=False, batch_size=None
288
+ ):
289
+ """
290
+ Fetch the items that should be dumped to ClickHouse
291
+ """
292
+ queryset = self.get_queryset(start_pk)
293
+ if ids:
294
+ ids = [self.pk_format(id) for id in ids]
295
+ queryset = queryset.filter(pk__in=ids)
296
+
297
+ if skip_ids:
298
+ skip_ids = [self.pk_format(id) for id in skip_ids]
299
+ queryset = queryset.exclude(pk__in=skip_ids)
300
+
301
+ paginator = Paginator(queryset, batch_size)
302
+ for i in range(1, paginator.num_pages + 1):
303
+ page = paginator.page(i)
304
+ items = page.object_list
305
+ for item in items:
306
+ if force_dump:
307
+ yield item, True, "Force is set"
308
+ else:
309
+ should_be_dumped, reason = self.should_dump_item(item)
310
+ yield item, should_be_dumped, reason
311
+
312
+ def should_dump_item(self, item): # pylint: disable=unused-argument
313
+ """
314
+ Return True if the item should be dumped to ClickHouse, False otherwise
315
+ """
316
+ return True, "No reason"
317
+
318
+ def get_last_dumped_timestamp(self, item_id):
319
+ """
320
+ Return the last timestamp that was dumped to ClickHouse
321
+ """
322
+ params = {
323
+ "query": f"SELECT max({self.timestamp_field}) as time_last_dumped "
324
+ f"FROM {self.ch_database}.{self.clickhouse_table_name} "
325
+ f"WHERE {self.unique_key} = '{item_id}'"
326
+ }
327
+
328
+ request = requests.Request("GET", self.ch_url, params=params, auth=self.ch_auth)
329
+
330
+ response = self._send_clickhouse_request(request)
331
+ response.raise_for_status()
332
+ if response.text.strip():
333
+ # ClickHouse returns timestamps in the format: "2023-05-03 15:47:39.331024+00:00"
334
+ # Our internal comparisons use the str() of a datetime object, this handles that
335
+ # transformation so that downstream comparisons will work.
336
+ return str(datetime.datetime.fromisoformat(response.text.strip()))
337
+
338
+ # Item has never been dumped, return None
339
+ return None
340
+
341
+ @classmethod
342
+ def is_enabled(cls):
343
+ """
344
+ Return True if the sink is enabled, False otherwise
345
+ """
346
+ enabled_settings = getattr(
347
+ settings,
348
+ f"{WAFFLE_FLAG_NAMESPACE.upper()}_{cls.model.upper()}_ENABLED",
349
+ False,
350
+ )
351
+ # .. toggle_name: event_sink_clickhouse.model.enabled
352
+ # .. toggle_implementation: WaffleFlag
353
+ # .. toggle_default: False
354
+ # .. toggle_description: Waffle flag to enable sink
355
+ # .. toggle_use_cases: open_edx
356
+ # .. toggle_creation_date: 2022-08-17
357
+ waffle_flag = WaffleFlag(
358
+ f"{WAFFLE_FLAG_NAMESPACE}.{cls.model}.enabled",
359
+ __name__,
360
+ )
361
+
362
+ enabled = enabled_settings or waffle_flag.is_enabled()
363
+
364
+ if enabled and not get_model(cls.model):
365
+ logging.warning(
366
+ f"Event Sink Model {cls.model} is not installed, but is enabled in settings or waffle flag."
367
+ )
368
+ enabled = False
369
+
370
+ return enabled
371
+
372
+ @classmethod
373
+ def get_sink_by_model_name(cls, model):
374
+ """
375
+ Return the sink instance for the given model
376
+ """
377
+ for sink in cls.__subclasses__():
378
+ if sink.model == model:
379
+ return sink
380
+
381
+ return None