platform-plugin-aspects 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/CHANGELOG.rst +7 -0
- {platform-plugin-aspects-0.1.0/platform_plugin_aspects.egg-info → platform-plugin-aspects-0.2.0}/PKG-INFO +14 -1
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/__init__.py +1 -1
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/apps.py +54 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/extensions/filters.py +1 -3
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/management/commands/__init__.py +0 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +207 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/settings/__init__.py +0 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/settings/common.py +62 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/settings/production.py +11 -1
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/signals.py +93 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/__init__.py +9 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/base_sink.py +372 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/course_overview_sink.py +219 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/external_id_sink.py +20 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/serializers.py +154 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/user_profile_sink.py +20 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/sinks/user_retire_sink.py +50 -0
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/tasks.py +61 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/utils.py +63 -3
- platform-plugin-aspects-0.2.0/platform_plugin_aspects/waffle.py +5 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0/platform_plugin_aspects.egg-info}/PKG-INFO +14 -1
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/SOURCES.txt +13 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/requires.txt +6 -0
- platform-plugin-aspects-0.2.0/requirements/base.in +16 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/setup.cfg +4 -3
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/setup.py +57 -40
- platform-plugin-aspects-0.1.0/platform_plugin_aspects/apps.py +0 -30
- platform-plugin-aspects-0.1.0/platform_plugin_aspects/settings/common.py +0 -23
- platform-plugin-aspects-0.1.0/requirements/base.in +0 -10
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/LICENSE +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/MANIFEST.in +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/README.rst +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/extensions/__init__.py +0 -0
- {platform-plugin-aspects-0.1.0/platform_plugin_aspects/settings → platform-plugin-aspects-0.2.0/platform_plugin_aspects/management}/__init__.py +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/static/css/superset.css +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/static/html/superset.html +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/static/js/embed_dashboard.js +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/static/js/superset.js +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects/templates/instructor_dashboard/aspects.html +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/dependency_links.txt +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/entry_points.txt +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/not-zip-safe +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/platform_plugin_aspects.egg-info/top_level.txt +0 -0
- {platform-plugin-aspects-0.1.0 → platform-plugin-aspects-0.2.0}/requirements/constraints.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: platform-plugin-aspects
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Aspects plugins for edx-platform
|
|
5
5
|
Home-page: https://github.com/openedx/platform-plugin-aspects
|
|
6
6
|
Author: Open edX Project
|
|
@@ -18,9 +18,15 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
18
18
|
Requires-Python: >=3.8
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: Django
|
|
21
|
+
Requires-Dist: celery
|
|
22
|
+
Requires-Dist: django-rest-framework
|
|
21
23
|
Requires-Dist: django_crum
|
|
24
|
+
Requires-Dist: edx-django-utils
|
|
25
|
+
Requires-Dist: edx-opaque-keys
|
|
26
|
+
Requires-Dist: edx-toggles
|
|
22
27
|
Requires-Dist: openedx-atlas
|
|
23
28
|
Requires-Dist: openedx-filters
|
|
29
|
+
Requires-Dist: requests
|
|
24
30
|
Requires-Dist: superset-api-client
|
|
25
31
|
Requires-Dist: web_fragments
|
|
26
32
|
|
|
@@ -142,6 +148,13 @@ Unreleased
|
|
|
142
148
|
|
|
143
149
|
*
|
|
144
150
|
|
|
151
|
+
0.2.0 – 2024-03-05
|
|
152
|
+
******************
|
|
153
|
+
Added
|
|
154
|
+
=====
|
|
155
|
+
|
|
156
|
+
* Imported code from event-sink-clickhouse.
|
|
157
|
+
|
|
145
158
|
0.1.0 – 2024-02-29
|
|
146
159
|
**********************************************
|
|
147
160
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
platform_plugin_aspects Django application initialization.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.apps import AppConfig
|
|
6
|
+
from edx_django_utils.plugins import PluginSettings, PluginSignals
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PlatformPluginAspectsConfig(AppConfig):
|
|
10
|
+
"""
|
|
11
|
+
Configuration for the aspects Django application.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
name = "platform_plugin_aspects"
|
|
15
|
+
|
|
16
|
+
plugin_app = {
|
|
17
|
+
PluginSettings.CONFIG: {
|
|
18
|
+
"lms.djangoapp": {
|
|
19
|
+
"production": {PluginSettings.RELATIVE_PATH: "settings.production"},
|
|
20
|
+
"common": {PluginSettings.RELATIVE_PATH: "settings.common"},
|
|
21
|
+
},
|
|
22
|
+
"cms.djangoapp": {
|
|
23
|
+
"production": {PluginSettings.RELATIVE_PATH: "settings.production"},
|
|
24
|
+
"common": {PluginSettings.RELATIVE_PATH: "settings.common"},
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
# Configuration setting for Plugin Signals for this app.
|
|
28
|
+
PluginSignals.CONFIG: {
|
|
29
|
+
# Configure the Plugin Signals for each Project Type, as needed.
|
|
30
|
+
"cms.djangoapp": {
|
|
31
|
+
# List of all plugin Signal receivers for this app and project type.
|
|
32
|
+
PluginSignals.RECEIVERS: [
|
|
33
|
+
{
|
|
34
|
+
# The name of the app's signal receiver function.
|
|
35
|
+
PluginSignals.RECEIVER_FUNC_NAME: "receive_course_publish",
|
|
36
|
+
# The full path to the module where the signal is defined.
|
|
37
|
+
PluginSignals.SIGNAL_PATH: "xmodule.modulestore.django.COURSE_PUBLISHED",
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
def ready(self):
|
|
45
|
+
"""Load modules of Aspects."""
|
|
46
|
+
super().ready()
|
|
47
|
+
from platform_plugin_aspects import ( # pylint: disable=import-outside-toplevel, unused-import
|
|
48
|
+
signals,
|
|
49
|
+
sinks,
|
|
50
|
+
tasks,
|
|
51
|
+
)
|
|
52
|
+
from platform_plugin_aspects.extensions import ( # pylint: disable=unused-import, import-outside-toplevel
|
|
53
|
+
filters,
|
|
54
|
+
)
|
|
@@ -37,9 +37,7 @@ class AddSupersetTab(PipelineStep):
|
|
|
37
37
|
|
|
38
38
|
filters = ASPECTS_SECURITY_FILTERS_FORMAT + extra_filters_format
|
|
39
39
|
|
|
40
|
-
context = generate_superset_context(
|
|
41
|
-
context, dashboard_uuid, filters
|
|
42
|
-
)
|
|
40
|
+
context = generate_superset_context(context, dashboard_uuid, filters)
|
|
43
41
|
|
|
44
42
|
template = Template(self.resource_string("static/html/superset.html"))
|
|
45
43
|
html = template.render(Context(context))
|
|
File without changes
|
platform-plugin-aspects-0.2.0/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Management command for exporting the modulestore ClickHouse.
|
|
3
|
+
|
|
4
|
+
Example usages (see usage for more options):
|
|
5
|
+
|
|
6
|
+
# Dump all objects published since last dump.
|
|
7
|
+
# Use connection parameters from `settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`:
|
|
8
|
+
python manage.py cms dump_objects_to_clickhouse --object user_profile
|
|
9
|
+
|
|
10
|
+
# Specify certain objects instead of dumping all of them.
|
|
11
|
+
# Use connection parameters from `settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG`.
|
|
12
|
+
python manage.py cms dump_objects_to_clickhouse --object user_profile --objects 123 124 125
|
|
13
|
+
|
|
14
|
+
# Dump a limited number of objects to prevent stress on production systems
|
|
15
|
+
python manage.py cms dump_objects_to_clickhouse --limit 1000
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
import time
|
|
20
|
+
from textwrap import dedent
|
|
21
|
+
|
|
22
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
23
|
+
|
|
24
|
+
from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def dump_target_objects_to_clickhouse(
|
|
30
|
+
sink=None,
|
|
31
|
+
start_pk=None,
|
|
32
|
+
object_ids=None,
|
|
33
|
+
objects_to_skip=None,
|
|
34
|
+
force=False,
|
|
35
|
+
limit=None,
|
|
36
|
+
batch_size=1000,
|
|
37
|
+
sleep_time=10,
|
|
38
|
+
):
|
|
39
|
+
"""
|
|
40
|
+
Iterates through a list of objects in the ORN, serializes them to csv,
|
|
41
|
+
then submits tasks to post them to ClickHouse.
|
|
42
|
+
|
|
43
|
+
Arguments:
|
|
44
|
+
force: serialize the objects even if they've been recently
|
|
45
|
+
serialized
|
|
46
|
+
|
|
47
|
+
Returns: two lists--one of the objects that had dump jobs queued for them
|
|
48
|
+
and one of objects that did not.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
count = 0
|
|
52
|
+
skipped_objects = []
|
|
53
|
+
objects_to_submit = []
|
|
54
|
+
|
|
55
|
+
for obj, should_be_dumped, reason in sink.fetch_target_items(
|
|
56
|
+
start_pk, object_ids, objects_to_skip, force, batch_size
|
|
57
|
+
):
|
|
58
|
+
if not should_be_dumped:
|
|
59
|
+
skipped_objects.append(obj.pk)
|
|
60
|
+
log.info(f"{sink.model}: Skipping object {obj.pk}, reason: '{reason}'")
|
|
61
|
+
else:
|
|
62
|
+
objects_to_submit.append(obj)
|
|
63
|
+
if len(objects_to_submit) % batch_size == 0:
|
|
64
|
+
count += len(objects_to_submit)
|
|
65
|
+
sink.dump(objects_to_submit, many=True)
|
|
66
|
+
objects_to_submit = []
|
|
67
|
+
log.info(f"Last ID: {obj.pk}")
|
|
68
|
+
time.sleep(sleep_time)
|
|
69
|
+
|
|
70
|
+
if limit and count == limit:
|
|
71
|
+
log.info(
|
|
72
|
+
f"Limit of {limit} eligible objects has been reached, quitting!"
|
|
73
|
+
)
|
|
74
|
+
break
|
|
75
|
+
|
|
76
|
+
if objects_to_submit:
|
|
77
|
+
sink.dump(objects_to_submit, many=True)
|
|
78
|
+
count += len(objects_to_submit)
|
|
79
|
+
log.info(f"Last ID: {objects_to_submit[-1].pk}")
|
|
80
|
+
|
|
81
|
+
log.info(f"Dumped {count} objects to ClickHouse")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Command(BaseCommand):
|
|
85
|
+
"""
|
|
86
|
+
Dump objects to a ClickHouse instance.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
help = dedent(__doc__).strip()
|
|
90
|
+
|
|
91
|
+
def add_arguments(self, parser):
|
|
92
|
+
parser.add_argument(
|
|
93
|
+
"--url",
|
|
94
|
+
type=str,
|
|
95
|
+
help="the URL of the ClickHouse server",
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"--username",
|
|
99
|
+
type=str,
|
|
100
|
+
help="the username of the ClickHouse user",
|
|
101
|
+
)
|
|
102
|
+
parser.add_argument(
|
|
103
|
+
"--password",
|
|
104
|
+
type=str,
|
|
105
|
+
help="the password of the ClickHouse user",
|
|
106
|
+
)
|
|
107
|
+
parser.add_argument(
|
|
108
|
+
"--database",
|
|
109
|
+
type=str,
|
|
110
|
+
help="the database in ClickHouse to connect to",
|
|
111
|
+
)
|
|
112
|
+
parser.add_argument(
|
|
113
|
+
"--timeout_secs",
|
|
114
|
+
type=int,
|
|
115
|
+
help="timeout for ClickHouse requests, in seconds",
|
|
116
|
+
)
|
|
117
|
+
parser.add_argument(
|
|
118
|
+
"--object",
|
|
119
|
+
type=str,
|
|
120
|
+
help="the type of object to dump",
|
|
121
|
+
)
|
|
122
|
+
parser.add_argument(
|
|
123
|
+
"--start_pk",
|
|
124
|
+
type=int,
|
|
125
|
+
help="the primary key to start at",
|
|
126
|
+
default=None,
|
|
127
|
+
)
|
|
128
|
+
parser.add_argument(
|
|
129
|
+
"--ids",
|
|
130
|
+
metavar="KEY",
|
|
131
|
+
type=str,
|
|
132
|
+
nargs="*",
|
|
133
|
+
help="keys of objects to serialize; if omitted all objects in system are serialized",
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument(
|
|
136
|
+
"--ids_to_skip",
|
|
137
|
+
metavar="KEY",
|
|
138
|
+
type=str,
|
|
139
|
+
nargs="*",
|
|
140
|
+
help="keys of objects to NOT to serialize",
|
|
141
|
+
)
|
|
142
|
+
parser.add_argument(
|
|
143
|
+
"--force",
|
|
144
|
+
action="store_true",
|
|
145
|
+
help="dump all objects regardless of when they were last published",
|
|
146
|
+
)
|
|
147
|
+
parser.add_argument(
|
|
148
|
+
"--limit",
|
|
149
|
+
type=int,
|
|
150
|
+
help="maximum number of objects to dump, cannot be used with '--ids' or '--force'",
|
|
151
|
+
)
|
|
152
|
+
parser.add_argument(
|
|
153
|
+
"--batch_size",
|
|
154
|
+
type=int,
|
|
155
|
+
default=10000,
|
|
156
|
+
help="number of objects to dump in a single batch",
|
|
157
|
+
)
|
|
158
|
+
parser.add_argument(
|
|
159
|
+
"--sleep_time",
|
|
160
|
+
type=int,
|
|
161
|
+
default=1,
|
|
162
|
+
help="number of seconds to sleep between batches",
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
def handle(self, *args, **options):
|
|
166
|
+
"""
|
|
167
|
+
Iterates through each objects, serializes and saves them to clickhouse.
|
|
168
|
+
"""
|
|
169
|
+
connection_overrides = {
|
|
170
|
+
key: options[key]
|
|
171
|
+
for key in ["url", "username", "password", "database", "timeout_secs"]
|
|
172
|
+
if options[key]
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
ids = options["ids"] if options["ids"] else []
|
|
176
|
+
ids_to_skip = options["ids_to_skip"] if options["ids_to_skip"] else []
|
|
177
|
+
|
|
178
|
+
if options["limit"] is not None and int(options["limit"]) < 1:
|
|
179
|
+
message = "'limit' must be greater than 0!"
|
|
180
|
+
log.error(message)
|
|
181
|
+
raise CommandError(message)
|
|
182
|
+
|
|
183
|
+
if options["limit"] and options["force"]:
|
|
184
|
+
message = (
|
|
185
|
+
"The 'limit' option cannot be used with 'force' as running the "
|
|
186
|
+
"command repeatedly will result in the same objects being dumped every time."
|
|
187
|
+
)
|
|
188
|
+
log.error(message)
|
|
189
|
+
raise CommandError(message)
|
|
190
|
+
|
|
191
|
+
if options["object"] is None:
|
|
192
|
+
message = "You must specify an object type to dump!"
|
|
193
|
+
log.error(message)
|
|
194
|
+
raise CommandError(message)
|
|
195
|
+
|
|
196
|
+
Sink = ModelBaseSink.get_sink_by_model_name(options["object"])
|
|
197
|
+
sink = Sink(connection_overrides, log)
|
|
198
|
+
dump_target_objects_to_clickhouse(
|
|
199
|
+
sink,
|
|
200
|
+
options["start_pk"],
|
|
201
|
+
[object_id.strip() for object_id in ids],
|
|
202
|
+
[object_id.strip() for object_id in ids_to_skip],
|
|
203
|
+
options["force"],
|
|
204
|
+
options["limit"],
|
|
205
|
+
options["batch_size"],
|
|
206
|
+
options["sleep_time"],
|
|
207
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Common Django settings for eox_hooks project.
|
|
3
|
+
For more information on this file, see
|
|
4
|
+
https://docs.djangoproject.com/en/2.22/topics/settings/
|
|
5
|
+
For the full list of settings and their values, see
|
|
6
|
+
https://docs.djangoproject.com/en/2.22/ref/settings/
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from platform_plugin_aspects import ROOT_DIRECTORY
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def plugin_settings(settings):
|
|
13
|
+
"""
|
|
14
|
+
Set of plugin settings used by the Open Edx platform.
|
|
15
|
+
More info: https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/plugins/README.rst
|
|
16
|
+
"""
|
|
17
|
+
settings.MAKO_TEMPLATE_DIRS_BASE.append(ROOT_DIRECTORY / "templates")
|
|
18
|
+
settings.SUPERSET_CONFIG = {
|
|
19
|
+
"url": "http://superset.local.overhang.io:8088",
|
|
20
|
+
"username": "superset",
|
|
21
|
+
"password": "superset",
|
|
22
|
+
}
|
|
23
|
+
settings.ASPECTS_INSTRUCTOR_DASHBOARD_UUID = "1d6bf904-f53f-47fd-b1c9-6cd7e284d286"
|
|
24
|
+
settings.SUPERSET_EXTRA_FILTERS_FORMAT = []
|
|
25
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG = {
|
|
26
|
+
# URL to a running ClickHouse server's HTTP interface. ex: https://foo.openedx.org:8443/ or
|
|
27
|
+
# http://foo.openedx.org:8123/ . Note that we only support the ClickHouse HTTP interface
|
|
28
|
+
# to avoid pulling in more dependencies to the platform than necessary.
|
|
29
|
+
"url": "http://clickhouse:8123",
|
|
30
|
+
"username": "ch_cms",
|
|
31
|
+
"password": "password",
|
|
32
|
+
"database": "event_sink",
|
|
33
|
+
"timeout_secs": 5,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS = [
|
|
37
|
+
"user_profile",
|
|
38
|
+
"external_id",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
settings.EVENT_SINK_CLICKHOUSE_MODEL_CONFIG = {
|
|
42
|
+
"auth_user": {
|
|
43
|
+
"module": "django.contrib.auth.models",
|
|
44
|
+
"model": "User",
|
|
45
|
+
},
|
|
46
|
+
"user_profile": {
|
|
47
|
+
"module": "common.djangoapps.student.models",
|
|
48
|
+
"model": "UserProfile",
|
|
49
|
+
},
|
|
50
|
+
"course_overviews": {
|
|
51
|
+
"module": "openedx.core.djangoapps.content.course_overviews.models",
|
|
52
|
+
"model": "CourseOverview",
|
|
53
|
+
},
|
|
54
|
+
"external_id": {
|
|
55
|
+
"module": "openedx.core.djangoapps.external_user_ids.models",
|
|
56
|
+
"model": "ExternalId",
|
|
57
|
+
},
|
|
58
|
+
"custom_course_edx": {
|
|
59
|
+
"module": "lms.djangoapps.ccx.models",
|
|
60
|
+
"model": "CustomCourseForEdX",
|
|
61
|
+
},
|
|
62
|
+
}
|
|
@@ -11,9 +11,19 @@ def plugin_settings(settings):
|
|
|
11
11
|
settings.SUPERSET_CONFIG = getattr(settings, "ENV_TOKENS", {}).get(
|
|
12
12
|
"SUPERSET_CONFIG", settings.SUPERSET_CONFIG
|
|
13
13
|
)
|
|
14
|
-
settings.ASPECTS_INSTRUCTOR_DASHBOARD_UUID = getattr(
|
|
14
|
+
settings.ASPECTS_INSTRUCTOR_DASHBOARD_UUID = getattr(
|
|
15
|
+
settings, "ENV_TOKENS", {}
|
|
16
|
+
).get(
|
|
15
17
|
"ASPECTS_INSTRUCTOR_DASHBOARD_UUID", settings.ASPECTS_INSTRUCTOR_DASHBOARD_UUID
|
|
16
18
|
)
|
|
17
19
|
settings.SUPERSET_EXTRA_FILTERS_FORMAT = getattr(settings, "ENV_TOKENS", {}).get(
|
|
18
20
|
"SUPERSET_EXTRA_FILTERS_FORMAT", settings.SUPERSET_EXTRA_FILTERS_FORMAT
|
|
19
21
|
)
|
|
22
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG = settings.ENV_TOKENS.get(
|
|
23
|
+
"EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG",
|
|
24
|
+
settings.EVENT_SINK_CLICKHOUSE_BACKEND_CONFIG,
|
|
25
|
+
)
|
|
26
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS = settings.ENV_TOKENS.get(
|
|
27
|
+
"EVENT_SINK_CLICKHOUSE_PII_MODELS",
|
|
28
|
+
settings.EVENT_SINK_CLICKHOUSE_PII_MODELS,
|
|
29
|
+
)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Signal handler functions, mapped to specific signals in apps.py.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.db.models.signals import post_save
|
|
6
|
+
from django.dispatch import Signal, receiver
|
|
7
|
+
|
|
8
|
+
from platform_plugin_aspects.sinks import (
|
|
9
|
+
ExternalIdSink,
|
|
10
|
+
UserProfileSink,
|
|
11
|
+
UserRetirementSink,
|
|
12
|
+
)
|
|
13
|
+
from platform_plugin_aspects.utils import get_model
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
from openedx.core.djangoapps.user_api.accounts.signals import USER_RETIRE_LMS_MISC
|
|
17
|
+
except ImportError:
|
|
18
|
+
# Tests don't have the platform installed
|
|
19
|
+
USER_RETIRE_LMS_MISC = Signal()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def receive_course_publish( # pylint: disable=unused-argument # pragma: no cover
|
|
23
|
+
sender, course_key, **kwargs
|
|
24
|
+
):
|
|
25
|
+
"""
|
|
26
|
+
Receives COURSE_PUBLISHED signal and queues the dump job.
|
|
27
|
+
"""
|
|
28
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
29
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
30
|
+
dump_course_to_clickhouse,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
dump_course_to_clickhouse.delay(str(course_key))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@receiver(post_save, sender=get_model("user_profile"))
|
|
37
|
+
def on_user_profile_updated( # pylint: disable=unused-argument # pragma: no cover
|
|
38
|
+
sender, instance, **kwargs
|
|
39
|
+
):
|
|
40
|
+
"""
|
|
41
|
+
Receives post save signal and queues the dump job.
|
|
42
|
+
"""
|
|
43
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
44
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
45
|
+
dump_data_to_clickhouse,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
sink = UserProfileSink(None, None)
|
|
49
|
+
dump_data_to_clickhouse.delay(
|
|
50
|
+
sink_module=sink.__module__,
|
|
51
|
+
sink_name=sink.__class__.__name__,
|
|
52
|
+
object_id=str(instance.id),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@receiver(post_save, sender=get_model("external_id"))
|
|
57
|
+
def on_externalid_saved( # pylint: disable=unused-argument # pragma: no cover
|
|
58
|
+
sender, instance, **kwargs
|
|
59
|
+
):
|
|
60
|
+
"""
|
|
61
|
+
Receives post save signal and queues the dump job.
|
|
62
|
+
"""
|
|
63
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
64
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
65
|
+
dump_data_to_clickhouse,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
sink = ExternalIdSink(None, None)
|
|
69
|
+
dump_data_to_clickhouse.delay(
|
|
70
|
+
sink_module=sink.__module__,
|
|
71
|
+
sink_name=sink.__class__.__name__,
|
|
72
|
+
object_id=str(instance.id),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@receiver(USER_RETIRE_LMS_MISC)
|
|
77
|
+
def on_user_retirement( # pylint: disable=unused-argument # pragma: no cover
|
|
78
|
+
sender, user, **kwargs
|
|
79
|
+
):
|
|
80
|
+
"""
|
|
81
|
+
Receives a user retirement signal and queues the retire_user job.
|
|
82
|
+
"""
|
|
83
|
+
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
|
|
84
|
+
from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
|
|
85
|
+
dump_data_to_clickhouse,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
sink = UserRetirementSink(None, None)
|
|
89
|
+
dump_data_to_clickhouse.delay(
|
|
90
|
+
sink_module=sink.__module__,
|
|
91
|
+
sink_name=sink.__class__.__name__,
|
|
92
|
+
object_id=str(user.id),
|
|
93
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module contains the sinks for the platform plugin aspects.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .base_sink import BaseSink, ModelBaseSink
|
|
6
|
+
from .course_overview_sink import CourseOverviewSink, XBlockSink
|
|
7
|
+
from .external_id_sink import ExternalIdSink
|
|
8
|
+
from .user_profile_sink import UserProfileSink
|
|
9
|
+
from .user_retire_sink import UserRetirementSink
|