edx-enterprise-data 9.0.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

enterprise_data/api/v1/views/analytics_engagements.py

@@ -1,395 +1,150 @@
-"""Advance Analytics for Engagements"""
+"""
+Views for handling REST endpoints related to Engagements analytics.
+"""
+
 from datetime import datetime
+from logging import getLogger
 
 from edx_rbac.decorators import permission_required
 from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
-from rest_framework import status as rest_status
+from rest_framework.decorators import action
 from rest_framework.response import Response
-from rest_framework.views import APIView
+from rest_framework.viewsets import ViewSet
 
-from django.http import HttpResponse, StreamingHttpResponse
+from django.http import StreamingHttpResponse
 
-from enterprise_data.admin_analytics.constants import Calculation, EngagementChart, Granularity, ResponseType
-from enterprise_data.admin_analytics.utils import (
-    calculation_aggregation,
-    fetch_and_cache_engagements_data,
-    fetch_and_cache_enrollments_data,
-    fetch_enrollments_cache_expiry_timestamp,
-    granularity_aggregation,
-)
-from enterprise_data.api.v1 import serializers
+from enterprise_data.admin_analytics.constants import ResponseType
+from enterprise_data.admin_analytics.database.tables import FactEngagementAdminDashTable, FactEnrollmentAdminDashTable
 from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
+from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
+from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
 from enterprise_data.renderers import IndividualEngagementsCSVRenderer
-from enterprise_data.utils import date_filter
+from enterprise_data.utils import timer
 
+LOGGER = getLogger(__name__)
 
-class AdvanceAnalyticsIndividualEngagementsView(APIView):
-    """
-    API for getting the advance analytics individual engagements data.
+
+class AdvanceAnalyticsEngagementView(AnalyticsPaginationMixin, ViewSet):
     """
+    View to handle requests for enterprise engagement data.
 
+    Here is the list of URLs that are handled by this view:
+    1. `enterprise_data_api_v1.enterprise-learner-engagement-list`: Get individual engagement data.
+    2. `enterprise_data_api_v1.enterprise-learner-engagement-stats`: Get engagement stats data.
+    """
     authentication_classes = (JwtAuthentication,)
     pagination_class = AdvanceAnalyticsPagination
-    http_method_names = ["get"]
+    http_method_names = ('get', )
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
-    def get(self, request, enterprise_uuid):
+    def list(self, request, enterprise_uuid):
         """
-        HTTP GET endpoint to retrieve the enterprise engagements data.
+        Get individual engagements data for the enterprise.
         """
-        serializer = serializers.AdvanceAnalyticsEngagementStatsSerializer(data=request.GET)
+        # Remove hyphens from the UUID
+        enterprise_uuid = enterprise_uuid.replace('-', '')
+
+        serializer = AdvanceAnalyticsQueryParamSerializer(data=request.GET)
         serializer.is_valid(raise_exception=True)
-        cache_expiry = fetch_enrollments_cache_expiry_timestamp()
+        min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(
+            enterprise_uuid,
+        )
 
-        enrollment_df = fetch_and_cache_enrollments_data(enterprise_uuid, cache_expiry).copy()
-        engagement_df = fetch_and_cache_engagements_data(enterprise_uuid, cache_expiry).copy()
-        # Use start and end date if provided by the client, if client has not provided then use
-        # 1. minimum enrollment date from the data as the start_date
-        # 2. today's date as the end_date
-        start_date = serializer.data.get('start_date', enrollment_df.enterprise_enrollment_date.min())
+        # get values from query params or use default values
+        start_date = serializer.data.get('start_date', min_enrollment_date)
         end_date = serializer.data.get('end_date', datetime.now())
+        page = serializer.data.get('page', 1)
+        page_size = serializer.data.get('page_size', 100)
+        engagements = FactEngagementAdminDashTable().get_all_engagements(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+            limit=page_size,
+            offset=(page - 1) * page_size,
+        )
+        total_count = FactEngagementAdminDashTable().get_engagement_count(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+        )
         response_type = request.query_params.get('response_type', ResponseType.JSON.value)
-        # Date filtering.
-        engagements = date_filter(
-            start=start_date, end=end_date, data_frame=engagement_df.copy(), date_column='activity_date'
+
+        LOGGER.info(
+            "Individual engagements data requested for enterprise [%s] from [%s] to [%s]",
+            enterprise_uuid,
+            start_date,
+            end_date,
         )
-        engagements["learning_time_hours"] = engagements["learning_time_seconds"] / 60 / 60
-        engagements = engagements[engagements["learning_time_hours"] > 0]
-        engagements["learning_time_hours"] = round(engagements["learning_time_hours"].astype(float), 1)
 
-        # Select only the columns that will be in the table.
-        engagements = engagements[
-            [
-                "email",
-                "course_title",
-                "activity_date",
-                "course_subject",
-                "learning_time_hours",
-            ]
-        ]
-        engagements["activity_date"] = engagements["activity_date"].dt.date
-        engagements = engagements.sort_values(by="activity_date", ascending=False).reset_index(drop=True)
         if response_type == ResponseType.CSV.value:
-            response = StreamingHttpResponse(
-                IndividualEngagementsCSVRenderer().render(self._stream_serialized_data(engagements)),
-                content_type="text/csv"
+            filename = f"""individual_engagements, {start_date} - {end_date}.csv"""
+
+            return StreamingHttpResponse(
+                IndividualEngagementsCSVRenderer().render(self._stream_serialized_data(
+                    enterprise_uuid, start_date, end_date, total_count
+                )),
+                content_type="text/csv",
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
             )
-            start_date = start_date.strftime('%Y/%m/%d')
-            end_date = end_date.strftime('%Y/%m/%d')
-            filename = f"""Individual Engagements, {start_date} - {end_date}.csv"""
-            response['Content-Disposition'] = f'attachment; filename="{filename}"'
-            response['Access-Control-Expose-Headers'] = 'Content-Disposition'
-            return response
 
-        paginator = self.pagination_class()
-        page = paginator.paginate_queryset(engagements, request)
-        serialized_data = page.data.to_dict(orient='records')
-        response = paginator.get_paginated_response(serialized_data)
-
-        return response
+        return self.get_paginated_response(
+            request=request,
+            records=engagements,
+            page=page,
+            page_size=page_size,
+            total_count=total_count,
+        )
 
-    def _stream_serialized_data(self, engagements, chunk_size=50000):
+    @staticmethod
+    def _stream_serialized_data(enterprise_uuid, start_date, end_date, total_count, page_size=50000):
         """
         Stream the serialized data.
         """
-        total_rows = engagements.shape[0]
-        for start_index in range(0, total_rows, chunk_size):
-            end_index = min(start_index + chunk_size, total_rows)
-            chunk = engagements.iloc[start_index:end_index]
-            yield from chunk.to_dict(orient='records')
-
-
-class AdvanceAnalyticsEngagementStatsView(APIView):
-    """
-    API for getting the advance analytics engagements statistics data.
-    """
-
-    authentication_classes = (JwtAuthentication,)
-    http_method_names = ["get"]
+        offset = 0
+        while offset < total_count:
+            engagements = FactEngagementAdminDashTable().get_all_engagements(
+                enterprise_customer_uuid=enterprise_uuid,
+                start_date=start_date,
+                end_date=end_date,
+                limit=page_size,
+                offset=offset,
+            )
+            yield from engagements
+            offset += page_size
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
-    def get(self, request, enterprise_uuid):
+    @action(detail=False, methods=['get'], name='Enterprise engagements data for charts', url_path='stats')
+    def stats(self, request, enterprise_uuid):
         """
-        HTTP GET endpoint to retrieve the enterprise engagements statistics data.
+        Get data to populate enterprise engagement charts.
+
+        Here is the list of the charts and their corresponding data:
+        1. `engagement_over_time`: This will show time series data of engagements over time.
+        2. `top_courses_by_engagement`: This will show the top courses by engagements.
+        3. `top_subjects_by_engagement`: This will show the top subjects by engagements.
         """
-        serializer = serializers.AdvanceAnalyticsEngagementStatsSerializer(data=request.GET)
-        serializer.is_valid(raise_exception=True)
+        # Remove hyphens from the UUID
+        enterprise_uuid = enterprise_uuid.replace('-', '')
 
-        cache_expiry = fetch_enrollments_cache_expiry_timestamp()
+        serializer = AdvanceAnalyticsQueryParamSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
 
-        enrollment_df = fetch_and_cache_enrollments_data(enterprise_uuid, cache_expiry).copy()
-        engagement_df = fetch_and_cache_engagements_data(enterprise_uuid, cache_expiry).copy()
-        # Use start and end date if provided by the client, if client has not provided then use
-        # 1. minimum enrollment date from the data as the start_date
-        # 2. today's date as the end_date
-        start_date = serializer.data.get('start_date', enrollment_df.enterprise_enrollment_date.min())
+        min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(
+            enterprise_uuid,
+        )
+        # get values from query params or use default
+        start_date = serializer.data.get('start_date', min_enrollment_date)
         end_date = serializer.data.get('end_date', datetime.now())
-        granularity = serializer.data.get('granularity', Granularity.DAILY.value)
-        calculation = serializer.data.get('calculation', Calculation.TOTAL.value)
-        chart_type = serializer.data.get('chart_type')
-
-        if chart_type is None:
+        with timer('construct_engagement_all_stats'):
             data = {
-                "engagements_over_time": self.construct_engagements_over_time(
-                    engagement_df.copy(),
-                    start_date,
-                    end_date,
-                    granularity,
-                    calculation,
+                'engagement_over_time': FactEngagementAdminDashTable().get_engagement_time_series_data(
+                    enterprise_uuid, start_date, end_date
                 ),
-                "top_courses_by_engagement": self.construct_top_courses_by_engagements(
-                    engagement_df.copy(),
-                    start_date,
-                    end_date,
+                'top_courses_by_engagement': FactEngagementAdminDashTable().get_top_courses_by_engagement(
+                    enterprise_uuid, start_date, end_date,
                 ),
-                "top_subjects_by_engagement": self.construct_top_subjects_by_engagements(
-                    engagement_df.copy(),
-                    start_date,
-                    end_date,
+                'top_subjects_by_engagement': FactEngagementAdminDashTable().get_top_subjects_by_engagement(
+                    enterprise_uuid, start_date, end_date,
                 ),
             }
-            return Response(data)
-        elif chart_type == EngagementChart.ENGAGEMENTS_OVER_TIME.value:
-            return self.construct_engagements_over_time_csv(
-                engagement_df.copy(),
-                start_date,
-                end_date,
-                granularity,
-                calculation,
-            )
-        elif chart_type == EngagementChart.TOP_COURSES_BY_ENGAGEMENTS.value:
-            return self.construct_top_courses_by_engagements_csv(
-                engagement_df.copy(),
-                start_date,
-                end_date,
-            )
-        elif chart_type == EngagementChart.TOP_SUBJECTS_BY_ENGAGEMENTS.value:
-            return self.construct_top_subjects_by_engagements_csv(
-                engagement_df.copy(),
-                start_date,
-                end_date,
-            )
-        return Response(data='Not Found', status=rest_status.HTTP_400_BAD_REQUEST)
-
-    def engagements_over_time_common(self, engagements_df, start_date, end_date, granularity, calculation):
-        """
-        Common method for constructing engagements over time data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-            granularity {str} -- Granularity of the data. One of Granularity choices
-            calculation {str} -- Calculation of the data. One of Calculation choices
-        """
-        engagements_df["learning_time_hours"] = engagements_df["learning_time_seconds"] / 60 / 60
-        engagements_df = engagements_df[engagements_df["learning_time_hours"] > 0]
-        engagements_df["learning_time_hours"] = round(engagements_df["learning_time_hours"].astype(float), 1)
-
-        engagements_df = engagements_df[["activity_date", "enroll_type", "learning_time_hours"]]
-
-        # Date filtering.
-        engagements_df = date_filter(
-            start=start_date, end=end_date, data_frame=engagements_df, date_column="activity_date"
-        )
-
-        # Date aggregation.
-        engagements_df = granularity_aggregation(
-            level=granularity,
-            group=["activity_date", "enroll_type"],
-            date="activity_date",
-            data_frame=engagements_df,
-            aggregation_type="sum",
-        )
-
-        # Calculating metric.
-        engagements = calculation_aggregation(calc=calculation, aggregation_type="sum", data_frame=engagements_df)
-        return engagements
-
-    def construct_engagements_over_time(self, engagements_df, start_date, end_date, granularity, calculation):
-        """
-        Construct engagements over time data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-            granularity {str} -- Granularity of the data. One of Granularity choices
-            calculation {str} -- Calculation of the data. One of Calculation choices
-        """
-        engagements = self.engagements_over_time_common(engagements_df, start_date, end_date, granularity, calculation)
-        # convert dataframe to a list of records
-        return engagements.to_dict(orient='records')
-
-    def construct_engagements_over_time_csv(self, engagements_df, start_date, end_date, granularity, calculation):
-        """
-        Construct engagements over time CSV.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-            granularity {str} -- Granularity of the data. One of Granularity choices
-            calculation {str} -- Calculation of the data. One of Calculation choices
-        """
-        engagements = self.engagements_over_time_common(engagements_df, start_date, end_date, granularity, calculation)
-
-        engagements = engagements.pivot(
-            index="activity_date", columns="enroll_type", values="sum"
-        )
-
-        filename = f"Engagement Timeseries, {start_date} - {end_date} ({granularity} {calculation}).csv"
-        return self.construct_csv_response(engagements, filename)
-
-    def top_courses_by_engagements_common(self, engagements_df, start_date, end_date):
-        """
-        Common method for constructing top courses by engagements data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-            group_by_columns {list} -- List of columns to group by
-            columns {list} -- List of column for the final result
-        """
-        engagements_df["learning_time_hours"] = engagements_df["learning_time_seconds"] / 60 / 60
-        engagements_df["learning_time_hours"] = engagements_df["learning_time_hours"].astype("float")
-
-        # Date filtering.
-        engagements = date_filter(
-            start=start_date, end=end_date, data_frame=engagements_df, date_column="activity_date"
-        )
-
-        courses = list(
-            engagements.groupby(["course_key"])
-            .learning_time_hours.sum()
-            .sort_values(ascending=False)[:10]
-            .index
-        )
-
-        engagements = (
-            engagements_df[engagements_df.course_key.isin(courses)]
-            .groupby(["course_key", "course_title", "enroll_type"])
-            .learning_time_hours.sum()
-            .reset_index()
-        )
-
-        engagements.columns = ["course_key", "course_title", "enroll_type", "count"]
-
-        return engagements
-
-    def construct_top_courses_by_engagements(self, engagements_df, start_date, end_date):
-        """
-        Construct top courses by engagements data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-        """
-        engagements = self.top_courses_by_engagements_common(
-            engagements_df,
-            start_date,
-            end_date
-        )
-
-        # convert dataframe to a list of records
-        return engagements.to_dict(orient='records')
-
-    def construct_top_courses_by_engagements_csv(self, engagements_df, start_date, end_date):
-        """
-        Construct top courses by engagements CSV.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-        """
-        engagements = self.top_courses_by_engagements_common(
-            engagements_df,
-            start_date,
-            end_date
-        )
-
-        engagements = engagements.pivot(
-            index=["course_key", "course_title"], columns="enroll_type", values="count"
-        )
-
-        filename = f"Top 10 Courses by Learning Hours, {start_date} - {end_date}.csv"
-        return self.construct_csv_response(engagements, filename)
-
-    def top_subjects_by_engagements_common(self, engagements_df, start_date, end_date):
-        """
-        Common method for constructing top subjects by engagements data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-        """
-        engagements_df["learning_time_hours"] = engagements_df["learning_time_seconds"] / 60 / 60
-        engagements_df["learning_time_hours"] = engagements_df["learning_time_hours"].astype("float")
-
-        # Date filtering.
-        engagements = date_filter(
-            start=start_date, end=end_date, data_frame=engagements_df, date_column="activity_date"
-        )
-
-        subjects = list(
-            engagements.groupby(["course_subject"])
-            .learning_time_hours.sum()
-            .sort_values(ascending=False)[:10]
-            .index
-        )
-
-        engagements = (
-            engagements[engagements.course_subject.isin(subjects)]
-            .groupby(["course_subject", "enroll_type"])
-            .learning_time_hours.sum()
-            .reset_index()
-        )
-        engagements.columns = ["course_subject", "enroll_type", "count"]
-
-        return engagements
-
-    def construct_top_subjects_by_engagements(self, engagements_df, start_date, end_date):
-        """
-        Construct top subjects by engagements data.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-        """
-        engagements = self.top_subjects_by_engagements_common(engagements_df, start_date, end_date)
-        # convert dataframe to a list of records
-        return engagements.to_dict(orient='records')
-
-    def construct_top_subjects_by_engagements_csv(self, engagements_df, start_date, end_date):
-        """
-        Construct top subjects by engagements CSV.
-
-        Arguments:
-            engagements_df {DataFrame} -- DataFrame of engagements
-            start_date {datetime} -- Engagement start date in the format 'YYYY-MM-DD'
-            end_date {datetime} -- Engagement end date in the format 'YYYY-MM-DD'
-        """
-        engagements = self.top_subjects_by_engagements_common(engagements_df, start_date, end_date)
-        engagements = engagements.pivot(index="course_subject", columns="enroll_type", values="count")
-        filename = f"Top 10 Subjects by Learning Hours, {start_date} - {end_date}.csv"
-        return self.construct_csv_response(engagements, filename)
-
-    def construct_csv_response(self, engagements, filename):
-        """
-        Construct CSV response.
-
-        Arguments:
-            engagements {DataFrame} -- DataFrame of engagements
-            filename {str} -- Filename for the CSV
-        """
-        response = HttpResponse(content_type='text/csv')
-        response['Content-Disposition'] = f'attachment; filename="{filename}"'
-        response['Access-Control-Expose-Headers'] = 'Content-Disposition'
-        engagements.to_csv(path_or_buf=response)
-
-        return response
+        return Response(data)

enterprise_data/api/v1/views/analytics_enrollments.py

@@ -15,10 +15,7 @@ from django.http import StreamingHttpResponse
 from enterprise_data.admin_analytics.constants import ResponseType
 from enterprise_data.admin_analytics.database.tables import FactEnrollmentAdminDashTable
 from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
-from enterprise_data.api.v1.serializers import (
-    AdvanceAnalyticsEnrollmentStatsSerializer,
-    AdvanceAnalyticsQueryParamSerializer,
-)
+from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
 from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
 from enterprise_data.renderers import IndividualEnrollmentsCSVRenderer
 from enterprise_data.utils import timer
@@ -115,7 +112,7 @@ class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
             offset += page_size
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
-    @action(detail=False, methods=['get'], name='Charts Data', url_path='stats')
+    @action(detail=False, methods=['get'], name='Enterprise enrollments data for charts', url_path='stats')
     def stats(self, request, enterprise_uuid):
         """
         Get data to populate enterprise enrollment charts.
@@ -128,7 +125,7 @@ class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
         # Remove hyphens from the UUID
         enterprise_uuid = enterprise_uuid.replace('-', '')
 
-        serializer = AdvanceAnalyticsEnrollmentStatsSerializer(data=request.GET)
+        serializer = AdvanceAnalyticsQueryParamSerializer(data=request.GET)
         serializer.is_valid(raise_exception=True)
 
         min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(

enterprise_data/renderers.py

@@ -45,17 +45,17 @@ class IndividualEnrollmentsCSVRenderer(CSVStreamingRenderer):
     ]
 
 
-class LeaderboardCSVRenderer(CSVStreamingRenderer):
+class IndividualCompletionsCSVRenderer(CSVStreamingRenderer):
     """
-    Custom streaming csv renderer for advance analytics leaderboard data.
+    Custom streaming csv renderer for advance analytics individual completions data.
     """
 
     header = [
         'email',
-        'learning_time_hours',
-        'daily_sessions',
-        'average_session_length',
-        'course_completions',
+        'course_title',
+        'course_subject',
+        'enroll_type',
+        'passed_date',
     ]
 
 
@@ -67,7 +67,22 @@ class IndividualEngagementsCSVRenderer(CSVStreamingRenderer):
     header = [
         'email',
         'course_title',
-        'activity_date',
         'course_subject',
+        'enroll_type',
+        'activity_date',
+        'learning_time_hours',
+    ]
+
+
+class LeaderboardCSVRenderer(CSVStreamingRenderer):
+    """
+    Custom streaming csv renderer for advance analytics leaderboard data.
+    """
+
+    header = [
+        'email',
         'learning_time_hours',
+        'daily_sessions',
+        'average_session_length',
+        'course_completions',
     ]