edx-enterprise-data 8.11.1__tar.gz → 8.12.0__tar.gz

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0}/CHANGELOG.rst

@@ -15,6 +15,10 @@ Unreleased
 ----------
 
 =========================
+[8.12.0] - 2024-09-06
+---------------------
+  * refactor: Performance optimizations for enrollments related API endpoints.
+
 [8.11.1] - 2024-08-29
 ---------------------
   * fix: Fixed a datetime conversion error appearing on production.

{edx_enterprise_data-8.11.1/edx_enterprise_data.egg-info → edx_enterprise_data-8.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.11.1
+Version: 8.12.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.11.1
+Version: 8.12.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "8.11.1"
+__version__ = "8.12.0"

edx_enterprise_data-8.12.0/enterprise_data/admin_analytics/database/queries/fact_enrollment_admin_dash.py

@@ -0,0 +1,139 @@
+"""
+Module containing queries for the fact_enrollment_admin_dash table.
+"""
+
+
+class FactEnrollmentAdminDashQueries:
+    """
+    Queries related to the fact_enrollment_admin_dash table.
+    """
+    @staticmethod
+    def get_enrollment_count_query():
+        """
+        Get the query to fetch the total number of enrollments for an enterprise customer.
+        """
+        return """
+            SELECT count(*)
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s;
+        """
+
+    @staticmethod
+    def get_all_enrollments_query():
+        """
+        Get the query to fetch all enrollments.
+        """
+        return """
+            SELECT email, course_title, course_subject, enroll_type, enterprise_enrollment_date
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s
+            ORDER BY ENTERPRISE_ENROLLMENT_DATE DESC LIMIT %(limit)s OFFSET %(offset)s
+        """
+
+    @staticmethod
+    def get_enrollment_date_range_query():
+        """
+        Get the query to fetch the enrollment date range.
+        """
+        return """
+            SELECT
+                MIN(enterprise_enrollment_date) AS min_enrollment_date,
+                MAX(enterprise_enrollment_date) AS max_enrollment_date
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s;
+        """
+
+    @staticmethod
+    def get_enrollment_and_course_count_query():
+        """
+        Get the query to fetch the enrollment and course count.
+        """
+        return """
+            SELECT
+                count(*) as enrolls, count(DISTINCT course_key) as courses
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s;
+        """
+
+    @staticmethod
+    def get_completion_count_query():
+        """
+        Get the query to fetch the completion count.
+        """
+        return """
+            SELECT
+                SUM(has_passed) as completions
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                passed_date BETWEEN %(start_date)s AND %(end_date)s;
+        """
+
+    @staticmethod
+    def get_learning_hours_and_daily_sessions_query():
+        """
+        Get the query to fetch the learning hours and daily sessions.
+        """
+        return """
+            SELECT
+                ROUND(SUM(learning_time_seconds) / 60 / 60, 1) as hours, SUM(is_engaged) as sessions
+            FROM fact_engagement_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                activity_date BETWEEN %(start_date)s AND %(end_date)s;
+        """
+
+    @staticmethod
+    def get_top_courses_enrollments_query(record_count=20):
+        """
+        Get the query to fetch the enrollment count by courses.
+
+        Query will fetch the top N courses by enrollment count. Where N is the value of record_count.
+
+        Arguments:
+            record_count (int): Number of records to fetch.
+        """
+        # Some local environments raise error when course_title is added in SELECT without GROUP BY.
+        # If you face this issue, you can remove course_title from SELECT.
+
+        # TODO: Re-add course_title after testing.
+        return f"""
+            SELECT course_key, enroll_type, count(course_key) as enrollment_count
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s
+            GROUP BY course_key, enroll_type ORDER BY enrollment_count DESC LIMIT {record_count};
+        """
+
+    @staticmethod
+    def get_top_subjects_by_enrollments_query(record_count=20):
+        """
+        Get the query to fetch the enrollment count by subjects.
+
+        Query will fetch the top N subjects by enrollment count. Where N is the value of record_count.
+
+        Arguments:
+            record_count (int): Number of records to fetch.
+        """
+        return f"""
+            SELECT course_subject, enroll_type, count(course_subject) enrollment_count
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s
+            GROUP BY course_subject, enroll_type ORDER BY enrollment_count DESC LIMIT {record_count};
+        """
+
+    @staticmethod
+    def get_enrolment_time_series_data_query():
+        """
+        Get the query to fetch the enrollment time series data with daily aggregation.
+        """
+        return """
+            SELECT enterprise_enrollment_date, enroll_type, COUNT(*) as enrollment_count
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                enterprise_enrollment_date BETWEEN %(start_date)s AND %(end_date)s
+            GROUP BY enterprise_enrollment_date, enroll_type
+            ORDER BY enterprise_enrollment_date;
+        """

edx_enterprise_data-8.12.0/enterprise_data/admin_analytics/database/tables/fact_enrollment_admin_dash.py

@@ -0,0 +1,201 @@
+"""
+Module for interacting with the fact_enrollment_admin_dash table.
+"""
+from datetime import date, datetime
+from uuid import UUID
+
+from ..queries import FactEnrollmentAdminDashQueries
+from ..utils import run_query
+from .base import BaseTable
+
+
+class FactEnrollmentAdminDashTable(BaseTable):
+    """
+    Class for communicating with the fact_enrollment_admin_dash table.
+    """
+    queries = FactEnrollmentAdminDashQueries()
+
+    def get_enrollment_count(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the total number of enrollments for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            (int): The total number of enrollments.
+        """
+        results = run_query(
+            query=self.queries.get_enrollment_count_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            }
+        )
+        return results[0][0]
+
+    def get_all_enrollments(
+            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+    ):
+        """
+        Get all enrollments for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+            limit (int): The maximum number of records to return.
+            offset (int): The number of records to skip.
+
+        Returns:
+            list<dict>: A list of dictionaries containing the enrollment data.
+        """
+        return run_query(
+            query=self.queries.get_all_enrollments_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+                'limit': limit,
+                'offset': offset,
+            },
+            as_dict=True,
+        )
+
+    def get_enrollment_date_range(self, enterprise_customer_uuid: UUID):
+        """
+        Get the enrollment date range for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+
+        Returns:
+            (tuple<date, date>): The minimum and maximum enrollment dates.
+        """
+        results = run_query(
+            query=self.queries.get_enrollment_date_range_query(),
+            params={'enterprise_customer_uuid': enterprise_customer_uuid}
+        )
+        min_date, max_date = results[0]
+
+        # We should return date objects, not datetime objects
+        # This is done to counter cases where database values are datetime objects.
+        if min_date and isinstance(min_date, datetime):
+            min_date = min_date.date()
+        if max_date and isinstance(max_date, datetime):
+            max_date = max_date.date()
+
+        return min_date, max_date
+
+    def get_enrollment_and_course_count(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the enrollment and course count for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            (tuple<int, int>): The enrollment and course count.
+        """
+        results = run_query(
+            query=self.queries.get_enrollment_and_course_count_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            }
+        )
+        return tuple(results[0])
+
+    def get_completion_count(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the completion count for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            int: The completion count.
+        """
+        results = run_query(
+            query=self.queries.get_completion_count_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            }
+        )
+        return results[0][0]
+
+    def get_top_courses_by_enrollments(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the top courses enrollments for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            list<dict>: A list of dictionaries containing the course key, course_title and enrollment count.
+        """
+        return run_query(
+            query=self.queries.get_top_courses_enrollments_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            },
+            as_dict=True,
+        )
+
+    def get_top_subjects_by_enrollments(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the top subjects by enrollments for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            list<dict>: A list of dictionaries containing the subject and enrollment count.
+        """
+        return run_query(
+            query=self.queries.get_top_subjects_by_enrollments_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            },
+            as_dict=True,
+        )
+
+    def get_enrolment_time_series_data(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the enrollment time series data for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            list<dict>: A list of dictionaries containing the date and enrollment count.
+        """
+        return run_query(
+            query=self.queries.get_enrolment_time_series_data_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            },
+            as_dict=True,
+        )

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0}/enterprise_data/admin_analytics/database/utils.py

@@ -30,22 +30,27 @@ def get_db_connection(database=settings.ENTERPRISE_REPORTING_DB_ALIAS):
 
 
 @timeit
-def run_query(query, params: dict = None):
+def run_query(query, params: dict = None, as_dict=False):
     """
     Run a query on the database and return the results.
 
     Arguments:
         query (str): The query to run.
         params (dict): The parameters to pass to the query.
+        as_dict (bool): When True, return the results as a dictionary.
 
     Returns:
-        (list): The results of the query.
+        (list | dict): The results of the query.
     """
     try:
         with closing(get_db_connection()) as connection:
             with closing(connection.cursor()) as cursor:
                 cursor.execute(query, params=params)
-                return cursor.fetchall()
+                if as_dict:
+                    columns = [column[0] for column in cursor.description]
+                    return [dict(zip(columns, row)) for row in cursor.fetchall()]
+                else:
+                    return cursor.fetchall()
     except Exception:
         LOGGER.exception(f'[run_query]: run_query failed for query "{query}".')
         raise

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0}/enterprise_data/api/v1/serializers.py

@@ -265,6 +265,8 @@ class AdvanceAnalyticsQueryParamSerializer(serializers.Serializer):  # pylint: d
     granularity = serializers.CharField(required=False)
     calculation = serializers.CharField(required=False)
     response_type = serializers.CharField(required=False)
+    page = serializers.IntegerField(required=False, min_value=1)
+    page_size = serializers.IntegerField(required=False, min_value=2)
 
     def validate(self, attrs):
         """

{edx_enterprise_data-8.11.1 → edx_enterprise_data-8.12.0}/enterprise_data/api/v1/urls.py

@@ -15,10 +15,7 @@ from enterprise_data.api.v1.views.analytics_engagements import (
     AdvanceAnalyticsEngagementStatsView,
     AdvanceAnalyticsIndividualEngagementsView,
 )
-from enterprise_data.api.v1.views.analytics_enrollments import (
-    AdvanceAnalyticsEnrollmentStatsView,
-    AdvanceAnalyticsIndividualEnrollmentsView,
-)
+from enterprise_data.api.v1.views.analytics_enrollments import AdvanceAnalyticsEnrollmentsView
 from enterprise_data.api.v1.views.analytics_leaderboard import AdvanceAnalyticsLeaderboardView
 from enterprise_data.constants import UUID4_REGEX
 
@@ -69,12 +66,12 @@ urlpatterns = [
     ),
     re_path(
         fr'^admin/analytics/(?P<enterprise_uuid>{UUID4_REGEX})/enrollments/stats$',
-        AdvanceAnalyticsEnrollmentStatsView.as_view(),
+        AdvanceAnalyticsEnrollmentsView.as_view({'get': 'stats'}),
         name='enterprise-admin-analytics-enrollments-stats'
     ),
     re_path(
         fr'^admin/analytics/(?P<enterprise_uuid>{UUID4_REGEX})/enrollments$',
-        AdvanceAnalyticsIndividualEnrollmentsView.as_view(),
+        AdvanceAnalyticsEnrollmentsView.as_view({'get': 'list'}),
         name='enterprise-admin-analytics-enrollments'
     ),
     re_path(

edx_enterprise_data-8.12.0/enterprise_data/api/v1/views/analytics_enrollments.py

@@ -0,0 +1,146 @@
+"""
+Advance Analytics for API endpoints to fetch enterprise enrollments data.
+"""
+from datetime import datetime
+from logging import getLogger
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.viewsets import ViewSet
+
+from django.http import StreamingHttpResponse
+
+from enterprise_data.admin_analytics.constants import ResponseType
+from enterprise_data.admin_analytics.database.tables import FactEnrollmentAdminDashTable
+from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
+from enterprise_data.api.v1.serializers import (
+    AdvanceAnalyticsEnrollmentStatsSerializer,
+    AdvanceAnalyticsQueryParamSerializer,
+)
+from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
+from enterprise_data.renderers import IndividualEnrollmentsCSVRenderer
+from enterprise_data.utils import timer
+
+LOGGER = getLogger(__name__)
+
+
+class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
+    """
+    View to handle requests for enterprise enrollments data.
+
+    Here is the list of URLs that are handled by this view:
+    1. `enterprise_data_api_v1.enterprise-learner-enrollment-list`: Get individual enrollment data.
+    2. `enterprise_data_api_v1.enterprise-learner-enrollment-stats`: Get enrollment stats data.
+    """
+    authentication_classes = (JwtAuthentication,)
+    pagination_class = AdvanceAnalyticsPagination
+    http_method_names = ('get', )
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
+    def list(self, request, enterprise_uuid):
+        """
+        Get individual enrollments data for the enterprise.
+        """
+        serializer = AdvanceAnalyticsQueryParamSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+        min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(
+            enterprise_uuid,
+        )
+
+        # get values from query params or use default values
+        start_date = serializer.data.get('start_date', min_enrollment_date)
+        end_date = serializer.data.get('end_date', datetime.now())
+        page = serializer.data.get('page', 1)
+        page_size = serializer.data.get('page_size', 100)
+        enrollments = FactEnrollmentAdminDashTable().get_all_enrollments(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+            limit=page_size,
+            offset=(page - 1) * page_size,
+        )
+        total_count = FactEnrollmentAdminDashTable().get_enrollment_count(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+        )
+        response_type = request.query_params.get('response_type', ResponseType.JSON.value)
+
+        LOGGER.info(
+            "Individual enrollments data requested for enterprise [%s] from [%s] to [%s]",
+            enterprise_uuid,
+            start_date,
+            end_date,
+        )
+
+        if response_type == ResponseType.CSV.value:
+            filename = f"""individual_enrollments, {start_date} - {end_date}.csv"""
+
+            return StreamingHttpResponse(
+                IndividualEnrollmentsCSVRenderer().render(self._stream_serialized_data(
+                    enterprise_uuid, start_date, end_date, total_count
+                )),
+                content_type="text/csv",
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+
+        return self.get_paginated_response(
+            request=request,
+            records=enrollments,
+            page=page,
+            page_size=page_size,
+            total_count=total_count,
+        )
+
+    @staticmethod
+    def _stream_serialized_data(enterprise_uuid, start_date, end_date, total_count, page_size=50000):
+        """
+        Stream the serialized data.
+        """
+        offset = 0
+        while offset < total_count:
+            enrollments = FactEnrollmentAdminDashTable().get_all_enrollments(
+                enterprise_customer_uuid=enterprise_uuid,
+                start_date=start_date,
+                end_date=end_date,
+                limit=page_size,
+                offset=offset,
+            )
+            yield from enrollments
+            offset += page_size
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
+    @action(detail=False, methods=['get'], name='Charts Data', url_path='stats')
+    def stats(self, request, enterprise_uuid):
+        """
+        Get data to populate enterprise enrollment charts.
+
+        Here is the list of the charts and their corresponding data:
+        1. `enrollments_over_time`: This will show time series data of enrollments over time.
+        2. `top_courses_by_enrollments`: This will show the top courses by enrollments.
+        3. `top_subjects_by_enrollments`: This will show the top subjects by enrollments.
+        """
+        serializer = AdvanceAnalyticsEnrollmentStatsSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+
+        min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(
+            enterprise_uuid,
+        )
+        # get values from query params or use default
+        start_date = serializer.data.get('start_date', min_enrollment_date)
+        end_date = serializer.data.get('end_date', datetime.now())
+        with timer('construct_enrollment_all_stats'):
+            data = {
+                'enrollments_over_time': FactEnrollmentAdminDashTable().get_enrolment_time_series_data(
+                    enterprise_uuid, start_date, end_date
+                ),
+                'top_courses_by_enrollments': FactEnrollmentAdminDashTable().get_top_courses_by_enrollments(
+                    enterprise_uuid, start_date, end_date,
+                ),
+                'top_subjects_by_enrollments': FactEnrollmentAdminDashTable().get_top_subjects_by_enrollments(
+                    enterprise_uuid, start_date, end_date,
+                ),
+            }
+        return Response(data)