platform-plugin-aspects 2.1.0__tar.gz → 2.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/PKG-INFO +7 -1
  2. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/README.rst +6 -0
  3. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/settings/common.py +8 -0
  4. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/signals.py +105 -0
  5. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/__init__.py +2 -0
  6. platform_plugin_aspects-2.2.1/src/platform_plugin_aspects/sinks/catalog_pathway_sink.py +80 -0
  7. platform_plugin_aspects-2.2.1/src/platform_plugin_aspects/sinks/pathway_enrollment_sink.py +20 -0
  8. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/serializers.py +70 -1
  9. platform_plugin_aspects-2.2.1/src/platform_plugin_aspects/sinks/tests/test_catalog_pathway_sink.py +106 -0
  10. platform_plugin_aspects-2.2.1/src/platform_plugin_aspects/sinks/tests/test_pathway_enrollment_sink.py +20 -0
  11. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_serializers.py +38 -0
  12. platform_plugin_aspects-2.2.1/src/platform_plugin_aspects/tests/test_signals.py +106 -0
  13. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/test_utils.py +33 -1
  14. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/utils.py +26 -1
  15. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/PKG-INFO +7 -1
  16. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/SOURCES.txt +4 -0
  17. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/scm_file_list.json +4 -0
  18. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/test_settings.py +1 -3
  19. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/uv.lock +561 -487
  20. platform_plugin_aspects-2.1.0/src/platform_plugin_aspects/tests/test_signals.py +0 -59
  21. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/.annotation_safe_list.yml +0 -0
  22. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/.editorconfig +0 -0
  23. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/.pii_annotations.yml +0 -0
  24. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/.readthedocs.yaml +0 -0
  25. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/LICENSE +0 -0
  26. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/MANIFEST.in +0 -0
  27. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/catalog-info.yaml +0 -0
  28. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/codecov.yml +0 -0
  29. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/manage.py +0 -0
  30. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/pylintrc +0 -0
  31. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/pylintrc_tweaks +0 -0
  32. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/pyproject.toml +0 -0
  33. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/setup.cfg +0 -0
  34. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/__init__.py +0 -0
  35. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/apps.py +0 -0
  36. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/babel_mako.cfg +0 -0
  37. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/config.yaml +0 -0
  38. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.mo +0 -0
  39. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/en/LC_MESSAGES/django.po +0 -0
  40. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.mo +0 -0
  41. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/en/LC_MESSAGES/mako.po +0 -0
  42. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.mo +0 -0
  43. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/django.po +0 -0
  44. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.mo +0 -0
  45. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/conf/locale/eo/LC_MESSAGES/mako.po +0 -0
  46. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/extensions/__init__.py +0 -0
  47. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/extensions/filters.py +0 -0
  48. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/extensions/tests/__init__.py +0 -0
  49. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/extensions/tests/test_filters.py +0 -0
  50. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/management/__init__.py +0 -0
  51. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/management/commands/__init__.py +0 -0
  52. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/management/commands/dump_data_to_clickhouse.py +0 -0
  53. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/management/commands/load_test_tracking_events.py +0 -0
  54. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/management/commands/monitor_load_test_tracking.py +0 -0
  55. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/settings/__init__.py +0 -0
  56. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/settings/production.py +0 -0
  57. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/settings/tests/__init__.py +0 -0
  58. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/base_sink.py +0 -0
  59. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/course_enrollment_sink.py +0 -0
  60. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/course_overview_sink.py +0 -0
  61. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/external_id_sink.py +0 -0
  62. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tag_sink.py +0 -0
  63. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_base_sink.py +0 -0
  64. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_course_enrollment_sink.py +0 -0
  65. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_course_overview_sink.py +0 -0
  66. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_external_id_sink.py +0 -0
  67. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_user_profile_sink.py +0 -0
  68. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/tests/test_user_retire_sink.py +0 -0
  69. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/user_profile_sink.py +0 -0
  70. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/sinks/user_retire_sink.py +0 -0
  71. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/css/superset.css +0 -0
  72. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/html/superset.html +0 -0
  73. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/html/superset_student.html +0 -0
  74. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/js/embed_dashboard.js +0 -0
  75. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/js/install_required.js +0 -0
  76. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/static/js/superset.js +0 -0
  77. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tasks.py +0 -0
  78. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/templates/instructor_dashboard/aspects.html +0 -0
  79. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/__init__.py +0 -0
  80. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/commands/test_dump_data_to_clickhouse.py +0 -0
  81. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/commands/test_load_test_monitor.py +0 -0
  82. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/commands/test_load_test_tracking_events.py +0 -0
  83. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/test_tasks.py +0 -0
  84. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/test_views.py +0 -0
  85. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/tests/test_xblock.py +0 -0
  86. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/urls.py +0 -0
  87. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/views.py +0 -0
  88. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/waffle.py +0 -0
  89. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects/xblock.py +0 -0
  90. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/dependency_links.txt +0 -0
  91. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/entry_points.txt +0 -0
  92. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/requires.txt +0 -0
  93. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/src/platform_plugin_aspects.egg-info/top_level.txt +0 -0
  94. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/test_utils/__init__.py +0 -0
  95. {platform_plugin_aspects-2.1.0 → platform_plugin_aspects-2.2.1}/test_utils/helpers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: platform-plugin-aspects
3
- Version: 2.1.0
3
+ Version: 2.2.1
4
4
  Summary: Aspects plugins for edx-platform
5
5
  Author-email: Open edX Project <oscm@openedx.org>
6
6
  License-Expression: Apache-2.0
@@ -72,6 +72,12 @@ Below are the existing sink names and their corresponding object names (as neede
72
72
  model and stores the user profile data in ClickHouse. Object name: ``user_profile``
73
73
  - ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
74
74
  the course enrollment data. Object name: ``course_enrollment``
75
+ - ``CatalogPathwaySink`` - Listens for the ``post_save`` Django signal on the openedx-core
76
+ ``CatalogPathway`` model. Object name: ``catalog_pathway``
77
+ - ``CatalogPathwayTombstoneSink`` - Listens for the ``post_delete`` Django signal on the
78
+ ``CatalogPathway`` model and marks the pathway deleted in ``catalog_pathway``.
79
+ - ``PathwayEnrollmentSink`` - Listens for the ``post_save`` Django signal on the openedx-core
80
+ ``PathwayEnrollment`` model. Object name: ``pathway_enrollment``
75
81
  - ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
76
82
  remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
77
83
 
@@ -36,6 +36,12 @@ Below are the existing sink names and their corresponding object names (as neede
36
36
  model and stores the user profile data in ClickHouse. Object name: ``user_profile``
37
37
  - ``CourseEnrollmentSink`` - Listen for the ``ENROLL_STATUS_CHANGE`` event and stores
38
38
  the course enrollment data. Object name: ``course_enrollment``
39
+ - ``CatalogPathwaySink`` - Listens for the ``post_save`` Django signal on the openedx-core
40
+ ``CatalogPathway`` model. Object name: ``catalog_pathway``
41
+ - ``CatalogPathwayTombstoneSink`` - Listens for the ``post_delete`` Django signal on the
42
+ ``CatalogPathway`` model and marks the pathway deleted in ``catalog_pathway``.
43
+ - ``PathwayEnrollmentSink`` - Listens for the ``post_save`` Django signal on the openedx-core
44
+ ``PathwayEnrollment`` model. Object name: ``pathway_enrollment``
39
45
  - ``UserRetirementSink`` - Listen for the ``USER_RETIRE_LMS_MISC`` Django signal and
40
46
  remove the user PII information from ClickHouse. This is a special sink with no assigned object name.
41
47
 
@@ -103,6 +103,14 @@ def plugin_settings(settings):
103
103
  "module": "openedx_tagging.models.base",
104
104
  "model": "ObjectTag",
105
105
  },
106
+ "catalog_pathway": {
107
+ "module": "openedx_catalog.models",
108
+ "model": "CatalogPathway",
109
+ },
110
+ "pathway_enrollment": {
111
+ "module": "openedx_catalog.models",
112
+ "model": "PathwayEnrollment",
113
+ },
106
114
  }
107
115
 
108
116
  # This is on by default here, off by default in tutor-contrib-aspects until we only
@@ -8,9 +8,12 @@ from django.dispatch import Signal, receiver
8
8
  from opaque_keys import InvalidKeyError
9
9
 
10
10
  from platform_plugin_aspects.sinks import (
11
+ CatalogPathwaySink,
12
+ CatalogPathwayTombstoneSink,
11
13
  CourseEnrollmentSink,
12
14
  ExternalIdSink,
13
15
  ObjectTagSink,
16
+ PathwayEnrollmentSink,
14
17
  TagSink,
15
18
  TaxonomySink,
16
19
  UserProfileSink,
@@ -303,3 +306,105 @@ _object_tag = get_model("object_tag")
303
306
  if _object_tag: # pragma: no cover
304
307
  post_save.connect(on_object_tag_saved_txn, sender=_object_tag)
305
308
  post_delete.connect(on_object_tag_deleted, sender=_object_tag)
309
+
310
+
311
+ def on_catalog_pathway_deleted( # pylint: disable=unused-argument
312
+ sender, instance, **kwargs
313
+ ):
314
+ """
315
+ Receives post delete signal for CatalogPathway and queues a tombstone dump on commit.
316
+ """
317
+ # Django clears instance.pk after post_delete, so the id is read now, not in on_commit.
318
+ object_id = str(instance.id)
319
+
320
+ def queue_tombstone():
321
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
322
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
323
+ dump_data_to_clickhouse,
324
+ )
325
+
326
+ sink = CatalogPathwayTombstoneSink(None, None)
327
+ dump_data_to_clickhouse.delay(
328
+ sink_module=sink.__module__,
329
+ sink_name=sink.__class__.__name__,
330
+ object_id=object_id,
331
+ )
332
+
333
+ transaction.on_commit(queue_tombstone, robust=True)
334
+
335
+
336
+ def on_catalog_pathway_saved_txn(*args, **kwargs):
337
+ """
338
+ Handle catalog_pathway saves in the middle of a transaction.
339
+
340
+ Handle saves in the middle of a transaction.
341
+ If this gets fired before the transaction commits, the task may try to
342
+ query an id that doesn't exist yet and throw an error. This should postpone
343
+ queuing the Celery task until after the transaction is committed.
344
+ """
345
+
346
+ def on_catalog_pathway_saved( # pylint: disable=unused-argument # pragma: no cover
347
+ sender, instance, **kwargs
348
+ ):
349
+ """
350
+ Receives post save signal and queues the dump job.
351
+ """
352
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
353
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
354
+ dump_data_to_clickhouse,
355
+ )
356
+
357
+ sink = CatalogPathwaySink(None, None)
358
+ dump_data_to_clickhouse.delay(
359
+ sink_module=sink.__module__,
360
+ sink_name=sink.__class__.__name__,
361
+ object_id=str(instance.id),
362
+ )
363
+
364
+ transaction.on_commit(lambda: on_catalog_pathway_saved(*args, **kwargs))
365
+
366
+
367
+ # Connect the CatalogPathway.post_save signal handler only if we have a model to attach to.
368
+ # (prevents celery errors during tests)
369
+ _catalog_pathway = get_model("catalog_pathway")
370
+ if _catalog_pathway: # pragma: no cover
371
+ post_save.connect(on_catalog_pathway_saved_txn, sender=_catalog_pathway)
372
+ post_delete.connect(on_catalog_pathway_deleted, sender=_catalog_pathway)
373
+
374
+
375
+ def on_pathway_enrollment_saved_txn(*args, **kwargs):
376
+ """
377
+ Handle pathway_enrollment saves in the middle of a transaction.
378
+
379
+ Handle saves in the middle of a transaction.
380
+ If this gets fired before the transaction commits, the task may try to
381
+ query an id that doesn't exist yet and throw an error. This should postpone
382
+ queuing the Celery task until after the transaction is committed.
383
+ """
384
+
385
+ def on_pathway_enrollment_saved( # pylint: disable=unused-argument # pragma: no cover
386
+ sender, instance, **kwargs
387
+ ):
388
+ """
389
+ Receives post save signal and queues the dump job.
390
+ """
391
+ # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
392
+ from platform_plugin_aspects.tasks import ( # pylint: disable=import-outside-toplevel
393
+ dump_data_to_clickhouse,
394
+ )
395
+
396
+ sink = PathwayEnrollmentSink(None, None)
397
+ dump_data_to_clickhouse.delay(
398
+ sink_module=sink.__module__,
399
+ sink_name=sink.__class__.__name__,
400
+ object_id=str(instance.id),
401
+ )
402
+
403
+ transaction.on_commit(lambda: on_pathway_enrollment_saved(*args, **kwargs))
404
+
405
+
406
+ # Connect the PathwayEnrollment.post_save signal handler only if we have a model to attach to.
407
+ # (prevents celery errors during tests)
408
+ _pathway_enrollment = get_model("pathway_enrollment")
409
+ if _pathway_enrollment: # pragma: no cover
410
+ post_save.connect(on_pathway_enrollment_saved_txn, sender=_pathway_enrollment)
@@ -3,9 +3,11 @@ This module contains the sinks for the platform plugin aspects.
3
3
  """
4
4
 
5
5
  from .base_sink import BaseSink, ModelBaseSink
6
+ from .catalog_pathway_sink import CatalogPathwaySink, CatalogPathwayTombstoneSink
6
7
  from .course_enrollment_sink import CourseEnrollmentSink
7
8
  from .course_overview_sink import CourseOverviewSink, XBlockSink
8
9
  from .external_id_sink import ExternalIdSink
10
+ from .pathway_enrollment_sink import PathwayEnrollmentSink
9
11
  from .tag_sink import ObjectTagSink, TagSink, TaxonomySink
10
12
  from .user_profile_sink import UserProfileSink
11
13
  from .user_retire_sink import UserRetirementSink
@@ -0,0 +1,80 @@
1
+ """Catalog Pathway sink"""
2
+
3
+ import json
4
+
5
+ import requests
6
+
7
+ from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
8
+ from platform_plugin_aspects.sinks.serializers import CatalogPathwaySerializer
9
+
10
+
11
+ class CatalogPathwaySink(ModelBaseSink): # pylint: disable=abstract-method
12
+ """
13
+ Sink for the openedx-core CatalogPathway model
14
+ """
15
+
16
+ model = "catalog_pathway"
17
+ unique_key = "id"
18
+ clickhouse_table_name = "catalog_pathway"
19
+ timestamp_field = "time_last_dumped"
20
+ name = "Catalog Pathway"
21
+ serializer_class = CatalogPathwaySerializer
22
+
23
+ # Unlike the other sink tables, time_last_dumped is DateTime64 here because it is the
24
+ # ReplacingMergeTree version column; the serializer writes it with a UTC offset, which
25
+ # only "best_effort" parses.
26
+ CLICKHOUSE_BULK_INSERT_PARAMS = {
27
+ **ModelBaseSink.CLICKHOUSE_BULK_INSERT_PARAMS,
28
+ "date_time_input_format": "best_effort",
29
+ }
30
+
31
+ def get_queryset(self, start_pk=None):
32
+ return super().get_queryset(start_pk).select_related("org", "category")
33
+
34
+
35
+ # pylint: disable-next=abstract-method
36
+ class CatalogPathwayTombstoneSink(CatalogPathwaySink):
37
+ """
38
+ Sink for deleted openedx-core CatalogPathways
39
+
40
+ The pathway is gone from MySQL by the time this runs, so instead of serializing it, the
41
+ pathway's latest row in ClickHouse is copied with is_deleted set. A pathway that was never
42
+ dumped gets no row.
43
+ """
44
+
45
+ name = "Catalog Pathway Tombstone"
46
+
47
+ def dump(self, item_id, many=False, initial=None):
48
+ """
49
+ Write a tombstone row for the deleted pathway with id item_id.
50
+ """
51
+ table = f"{self.ch_database}.{self.clickhouse_table_name}"
52
+ params = {
53
+ "query": (
54
+ f"INSERT INTO {table} "
55
+ "SELECT * REPLACE (1 AS is_deleted, generateUUIDv4() AS dump_id, "
56
+ "now64(6) AS time_last_dumped) "
57
+ # Picked in a subquery: next to the REPLACE, ORDER BY time_last_dumped would
58
+ # resolve to the now64(6) alias and copy an arbitrary version.
59
+ f"FROM (SELECT * FROM {table} WHERE id = {int(item_id)} "
60
+ "ORDER BY time_last_dumped DESC LIMIT 1)"
61
+ ),
62
+ }
63
+ request = requests.Request(
64
+ "POST",
65
+ self.ch_url,
66
+ params=params,
67
+ auth=self.ch_auth,
68
+ )
69
+ self.log.info(f"Now dumping {self.name} {item_id} to ClickHouse")
70
+ # Retried once: the pathway is gone from MySQL, so a lost tombstone can't be backfilled.
71
+ try:
72
+ response = self._send_clickhouse_request(request)
73
+ except requests.RequestException:
74
+ response = self._send_clickhouse_request(request)
75
+ summary = json.loads(response.headers.get("X-ClickHouse-Summary", "{}"))
76
+ if summary.get("written_rows") == "0":
77
+ self.log.warning(
78
+ f"No {self.name} written for {item_id}, it was never dumped"
79
+ )
80
+ self.log.info(f"Completed dumping {self.name} {item_id} to ClickHouse")
@@ -0,0 +1,20 @@
1
+ """Pathway Enrollment sink"""
2
+
3
+ from platform_plugin_aspects.sinks.base_sink import ModelBaseSink
4
+ from platform_plugin_aspects.sinks.serializers import PathwayEnrollmentSerializer
5
+
6
+
7
+ class PathwayEnrollmentSink(ModelBaseSink): # pylint: disable=abstract-method
8
+ """
9
+ Sink for the openedx-core PathwayEnrollment model
10
+ """
11
+
12
+ model = "pathway_enrollment"
13
+ unique_key = "id"
14
+ clickhouse_table_name = "pathway_enrollment"
15
+ timestamp_field = "time_last_dumped"
16
+ name = "Pathway Enrollment"
17
+ serializer_class = PathwayEnrollmentSerializer
18
+
19
+ def get_queryset(self, start_pk=None):
20
+ return super().get_queryset(start_pk).select_related("user")
@@ -8,7 +8,11 @@ from django.utils import timezone
8
8
  from pytz import UTC
9
9
  from rest_framework import serializers
10
10
 
11
- from platform_plugin_aspects.utils import get_model, get_tags_for_block
11
+ from platform_plugin_aspects.utils import (
12
+ get_model,
13
+ get_or_create_xapi_actor_id,
14
+ get_tags_for_block,
15
+ )
12
16
 
13
17
 
14
18
  class DateTimeJSONEncoder(json.JSONEncoder):
@@ -210,6 +214,71 @@ class CourseEnrollmentSerializer(BaseSinkSerializer, serializers.ModelSerializer
210
214
  return str(obj.course_id)
211
215
 
212
216
 
217
+ class CatalogPathwaySerializer(BaseSinkSerializer, serializers.ModelSerializer):
218
+ """Serializer for the CatalogPathway model."""
219
+
220
+ org = serializers.CharField(source="org.short_name")
221
+ category_code = serializers.CharField(source="category.category_code")
222
+ category_title = serializers.CharField(source="category.title")
223
+ is_deleted = serializers.SerializerMethodField()
224
+
225
+ class Meta:
226
+ """Meta class for the CatalogPathwaySerializer"""
227
+
228
+ model = get_model("catalog_pathway")
229
+ fields = [
230
+ "id",
231
+ "org",
232
+ "pathway_code",
233
+ "title",
234
+ "category_code",
235
+ "category_title",
236
+ "created",
237
+ "modified",
238
+ "dump_id",
239
+ "time_last_dumped",
240
+ "is_deleted",
241
+ ]
242
+
243
+ def get_is_deleted(self, instance): # pylint: disable=unused-argument
244
+ """
245
+ Saved pathways are never deleted; CatalogPathwayTombstoneSink writes deletions.
246
+
247
+ 0 rather than False: the column is the UInt8 is_deleted of a ReplacingMergeTree, and
248
+ ClickHouse won't parse a CSV "False" as a UInt8.
249
+ """
250
+ return 0
251
+
252
+
253
+ class PathwayEnrollmentSerializer(BaseSinkSerializer, serializers.ModelSerializer):
254
+ """
255
+ Serializer for the PathwayEnrollment model.
256
+
257
+ The learner is stored as their xAPI actor id instead of user_id to keep the table PII free.
258
+ """
259
+
260
+ actor_id = serializers.SerializerMethodField()
261
+
262
+ class Meta:
263
+ """Meta class for the PathwayEnrollmentSerializer"""
264
+
265
+ model = get_model("pathway_enrollment")
266
+ fields = [
267
+ "id",
268
+ "catalog_pathway_id",
269
+ "actor_id",
270
+ "created",
271
+ "modified",
272
+ "is_active",
273
+ "dump_id",
274
+ "time_last_dumped",
275
+ ]
276
+
277
+ def get_actor_id(self, instance):
278
+ """Return the learner's xAPI actor id instead of any user identifier."""
279
+ return get_or_create_xapi_actor_id(instance.user)
280
+
281
+
213
282
  class TagSerializer(BaseSinkSerializer, serializers.ModelSerializer):
214
283
  """Serializer for the Tag model."""
215
284
 
@@ -0,0 +1,106 @@
1
+ """
2
+ Test the catalog pathway sink module.
3
+ """
4
+
5
+ import logging
6
+ from unittest.mock import Mock, patch
7
+
8
+ import pytest
9
+ import requests
10
+ import responses
11
+ from responses.registries import OrderedRegistry
12
+
13
+ from platform_plugin_aspects.sinks import (
14
+ CatalogPathwaySink,
15
+ CatalogPathwayTombstoneSink,
16
+ )
17
+
18
+ log = logging.getLogger(__name__)
19
+
20
+
21
+ @patch("platform_plugin_aspects.sinks.ModelBaseSink.get_queryset")
22
+ def test_get_queryset(mock_get_queryset):
23
+ """
24
+ Test the get_queryset method.
25
+ """
26
+ sink = CatalogPathwaySink(None, None)
27
+
28
+ sink.get_queryset()
29
+
30
+ mock_get_queryset.assert_called_once_with(None)
31
+ mock_get_queryset.return_value.select_related.assert_called_once_with(
32
+ "org", "category"
33
+ )
34
+
35
+
36
+ @responses.activate( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
37
+ registry=OrderedRegistry
38
+ )
39
+ def test_tombstone_dump():
40
+ """
41
+ Test that the tombstone sink copies the pathway's latest row with is_deleted set.
42
+ """
43
+ tombstone = responses.post(
44
+ "https://foo.bar/",
45
+ match=[
46
+ responses.matchers.query_param_matcher(
47
+ {
48
+ "query": "INSERT INTO cool_data.catalog_pathway "
49
+ "SELECT * REPLACE (1 AS is_deleted, generateUUIDv4() AS dump_id, "
50
+ "now64(6) AS time_last_dumped) "
51
+ "FROM (SELECT * FROM cool_data.catalog_pathway WHERE id = 7 "
52
+ "ORDER BY time_last_dumped DESC LIMIT 1)",
53
+ }
54
+ )
55
+ ],
56
+ headers={"X-ClickHouse-Summary": '{"written_rows":"1"}'},
57
+ )
58
+ mock_log = Mock()
59
+
60
+ CatalogPathwayTombstoneSink(None, mock_log).dump("7")
61
+
62
+ assert tombstone.call_count == 1
63
+ mock_log.warning.assert_not_called()
64
+
65
+
66
+ @responses.activate( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
67
+ registry=OrderedRegistry
68
+ )
69
+ def test_tombstone_dump_retries_once():
70
+ """
71
+ Test that a failed tombstone is retried once, and a second failure raises.
72
+ """
73
+ responses.post("https://foo.bar/", body=requests.ConnectionError())
74
+ responses.post("https://foo.bar/")
75
+
76
+ CatalogPathwayTombstoneSink(None, log).dump("7")
77
+
78
+ responses.post("https://foo.bar/", body=requests.ConnectionError())
79
+ responses.post("https://foo.bar/", body=requests.ConnectionError())
80
+
81
+ with pytest.raises(requests.ConnectionError):
82
+ CatalogPathwayTombstoneSink(None, log).dump("7")
83
+
84
+
85
+ @responses.activate # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
86
+ def test_tombstone_dump_warns_when_nothing_written():
87
+ """
88
+ Test that a pathway that was never dumped logs a warning.
89
+ """
90
+ responses.post(
91
+ "https://foo.bar/",
92
+ headers={"X-ClickHouse-Summary": '{"written_rows":"0"}'},
93
+ )
94
+ mock_log = Mock()
95
+
96
+ CatalogPathwayTombstoneSink(None, mock_log).dump("7")
97
+
98
+ mock_log.warning.assert_called_once()
99
+
100
+
101
+ def test_tombstone_dump_rejects_non_integer_id():
102
+ """
103
+ Test that an id that isn't an integer never reaches the query.
104
+ """
105
+ with pytest.raises(ValueError):
106
+ CatalogPathwayTombstoneSink(None, log).dump("1 OR 1=1")
@@ -0,0 +1,20 @@
1
+ """
2
+ Test the pathway enrollment sink module.
3
+ """
4
+
5
+ from unittest.mock import patch
6
+
7
+ from platform_plugin_aspects.sinks import PathwayEnrollmentSink
8
+
9
+
10
+ @patch("platform_plugin_aspects.sinks.ModelBaseSink.get_queryset")
11
+ def test_get_queryset(mock_get_queryset):
12
+ """
13
+ Test the get_queryset method.
14
+ """
15
+ sink = PathwayEnrollmentSink(None, None)
16
+
17
+ sink.get_queryset()
18
+
19
+ mock_get_queryset.assert_called_once_with(None)
20
+ mock_get_queryset.return_value.select_related.assert_called_once_with("user")
@@ -6,8 +6,10 @@ from django.test import TestCase
6
6
 
7
7
  from platform_plugin_aspects.sinks.serializers import (
8
8
  BaseSinkSerializer,
9
+ CatalogPathwaySerializer,
9
10
  CourseOverviewSerializer,
10
11
  DateTimeJSONEncoder,
12
+ PathwayEnrollmentSerializer,
11
13
  )
12
14
  from test_utils.helpers import course_key_factory
13
15
 
@@ -94,3 +96,39 @@ class TestCourseOverviewSerializer(TestCase):
94
96
  mock_id = Mock()
95
97
  mock_overview = Mock(id=mock_id)
96
98
  self.assertEqual(self.serializer.get_course_key(mock_overview), str(mock_id))
99
+
100
+
101
+ class TestCatalogPathwaySerializer(TestCase):
102
+ """
103
+ Test CatalogPathwaySerializer
104
+ """
105
+
106
+ def test_is_deleted(self):
107
+ """
108
+ Test that saved pathways are written with is_deleted 0, in the last column.
109
+ """
110
+ serializer = CatalogPathwaySerializer()
111
+
112
+ self.assertEqual(serializer.get_is_deleted(Mock()), 0)
113
+ self.assertEqual(CatalogPathwaySerializer.Meta.fields[-1], "is_deleted")
114
+
115
+
116
+ class TestPathwayEnrollmentSerializer(TestCase):
117
+ """
118
+ Test PathwayEnrollmentSerializer
119
+ """
120
+
121
+ @patch("platform_plugin_aspects.sinks.serializers.get_or_create_xapi_actor_id")
122
+ def test_get_actor_id(self, mock_get_or_create_xapi_actor_id):
123
+ """
124
+ Test that the learner is serialized as their xAPI actor id.
125
+ """
126
+ mock_get_or_create_xapi_actor_id.return_value = (
127
+ "29e5b22b-b457-49fa-9633-6c0123305d18"
128
+ )
129
+ enrollment = Mock()
130
+
131
+ actor_id = PathwayEnrollmentSerializer().get_actor_id(enrollment)
132
+
133
+ assert actor_id == "29e5b22b-b457-49fa-9633-6c0123305d18"
134
+ mock_get_or_create_xapi_actor_id.assert_called_once_with(enrollment.user)
@@ -0,0 +1,106 @@
1
+ """
2
+ Tests for signal handlers.
3
+ """
4
+
5
+ from unittest.mock import ANY, Mock, patch
6
+
7
+ from django.test import TestCase
8
+
9
+ from platform_plugin_aspects.signals import (
10
+ on_catalog_pathway_deleted,
11
+ on_catalog_pathway_saved_txn,
12
+ on_externalid_saved_txn,
13
+ on_pathway_enrollment_saved_txn,
14
+ on_user_retirement,
15
+ receive_course_publish,
16
+ )
17
+ from platform_plugin_aspects.sinks import CatalogPathwayTombstoneSink
18
+ from platform_plugin_aspects.sinks.user_retire_sink import UserRetirementSink
19
+
20
+
21
+ class SignalHandlersTestCase(TestCase):
22
+ """
23
+ Test cases for signal handlers.
24
+ """
25
+
26
+ @patch("platform_plugin_aspects.tasks.dump_course_to_clickhouse")
27
+ def test_receive_course_publish(self, mock_dump_task):
28
+ """
29
+ Test that receive_course_publish calls dump_course_to_clickhouse.
30
+ """
31
+ sender = Mock()
32
+ course_key = "sample_key"
33
+ receive_course_publish(sender, course_key)
34
+
35
+ mock_dump_task.delay.assert_called_once_with(course_key)
36
+
37
+ @patch("platform_plugin_aspects.signals.transaction")
38
+ def test_on_externalid_saved(self, mock_transaction):
39
+ """
40
+ Test that on_externalid_saved calls dump_data_to_clickhouse.
41
+ """
42
+ instance = Mock()
43
+ sender = Mock()
44
+ on_externalid_saved_txn(sender, instance)
45
+
46
+ mock_transaction.on_commit.assert_called_once()
47
+
48
+ @patch("platform_plugin_aspects.signals.transaction")
49
+ def test_on_catalog_pathway_saved(self, mock_transaction):
50
+ """
51
+ Test that on_catalog_pathway_saved_txn defers the dump to on_commit.
52
+ """
53
+ instance = Mock()
54
+ sender = Mock()
55
+ on_catalog_pathway_saved_txn(sender, instance)
56
+
57
+ mock_transaction.on_commit.assert_called_once()
58
+
59
+ @patch("platform_plugin_aspects.signals.transaction")
60
+ def test_on_pathway_enrollment_saved(self, mock_transaction):
61
+ """
62
+ Test that on_pathway_enrollment_saved_txn defers the dump to on_commit.
63
+ """
64
+ instance = Mock()
65
+ sender = Mock()
66
+ on_pathway_enrollment_saved_txn(sender, instance)
67
+
68
+ mock_transaction.on_commit.assert_called_once()
69
+
70
+ @patch("platform_plugin_aspects.tasks.dump_data_to_clickhouse")
71
+ @patch("platform_plugin_aspects.signals.transaction")
72
+ def test_on_catalog_pathway_deleted(self, mock_transaction, mock_dump_task):
73
+ """
74
+ Test that on_catalog_pathway_deleted queues the tombstone sink on commit, with the id
75
+ read before Django clears it.
76
+ """
77
+ instance = Mock(id=7)
78
+ on_catalog_pathway_deleted(Mock(), instance)
79
+ instance.id = None
80
+
81
+ mock_dump_task.delay.assert_not_called()
82
+ mock_transaction.on_commit.assert_called_once_with(ANY, robust=True)
83
+ mock_transaction.on_commit.call_args.args[0]()
84
+
85
+ mock_dump_task.delay.assert_called_once_with(
86
+ sink_module=CatalogPathwayTombstoneSink.__module__,
87
+ sink_name="CatalogPathwayTombstoneSink",
88
+ object_id="7",
89
+ )
90
+
91
+ @patch("platform_plugin_aspects.tasks.dump_data_to_clickhouse")
92
+ def test_on_user_retirement(self, mock_dump_task):
93
+ """
94
+ Test that on_user_retirement calls dump_data_to_clickhouse
95
+ """
96
+ instance = Mock()
97
+ sender = Mock()
98
+ on_user_retirement(sender, instance)
99
+
100
+ sink = UserRetirementSink(None, None)
101
+
102
+ mock_dump_task.delay.assert_called_once_with(
103
+ sink_module=sink.__module__,
104
+ sink_name=sink.__class__.__name__,
105
+ object_id=str(instance.id),
106
+ )