edx-enterprise-data 8.5.0__py3-none-any.whl → 8.6.1__py3-none-any.whl

enterprise_data/api/v1/views/analytics_enrollments.py

@@ -0,0 +1,375 @@
+"""Advance Analytics for Enrollments"""
+from datetime import datetime, timedelta
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from django.http import HttpResponse, StreamingHttpResponse
+
+from enterprise_data.admin_analytics.constants import CALCULATION, ENROLLMENT_CSV, GRANULARITY
+from enterprise_data.admin_analytics.data_loaders import fetch_max_enrollment_datetime
+from enterprise_data.admin_analytics.utils import (
+    calculation_aggregation,
+    fetch_and_cache_enrollments_data,
+    granularity_aggregation,
+)
+from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
+from enterprise_data.api.v1.serializers import (
+    AdvanceAnalyticsEnrollmentSerializer,
+    AdvanceAnalyticsEnrollmentStatsSerializer,
+)
+from enterprise_data.renderers import IndividualEnrollmentsCSVRenderer
+from enterprise_data.utils import date_filter
+
+
+def fetch_enrollments_cache_expiry_timestamp():
+    """Calculate cache expiry timestamp"""
+    # TODO: Implement correct cache expiry logic for `enrollments` data.
+    #       Current cache expiry logic is based on `enterprise_learner_enrollment` table,
+    #       Which has nothing to do with the `enrollments` data. Instead cache expiry should
+    #       be based on `fact_enrollment_admin_dash` table. Currently we have no timestamp in
+    #       `fact_enrollment_admin_dash` table that can be used for cache expiry. Add a new
+    #       column in the table for this purpose and then use that column for cache expiry.
+    last_updated_at = fetch_max_enrollment_datetime()
+    cache_expiry = (
+        last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+    )
+    return cache_expiry
+
+
+class AdvanceAnalyticsIndividualEnrollmentsView(APIView):
+    """
+    API for getting the advance analytics individual enrollments data.
+    """
+    authentication_classes = (JwtAuthentication,)
+    pagination_class = AdvanceAnalyticsPagination
+    http_method_names = ['get']
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
+    def get(self, request, enterprise_uuid):
+        """Get individual enrollments data"""
+        serializer = AdvanceAnalyticsEnrollmentSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+
+        cache_expiry = fetch_enrollments_cache_expiry_timestamp()
+        enrollments_df = fetch_and_cache_enrollments_data(enterprise_uuid, cache_expiry)
+
+        # get values from query params or use default values
+        start_date = serializer.data.get('start_date', enrollments_df.enterprise_enrollment_date.min())
+        end_date = serializer.data.get('end_date', datetime.now())
+        csv_type = request.query_params.get('csv_type')
+
+        # filter enrollments by date
+        enrollments = date_filter(start_date, end_date, enrollments_df, "enterprise_enrollment_date")
+
+        # select only the columns that will be in the table.
+        enrollments = enrollments[
+            [
+                "email",
+                "course_title",
+                "course_subject",
+                "enroll_type",
+                "enterprise_enrollment_date",
+            ]
+        ]
+        enrollments["enterprise_enrollment_date"] = enrollments["enterprise_enrollment_date"].dt.date
+        enrollments = enrollments.sort_values(by="enterprise_enrollment_date", ascending=False).reset_index(drop=True)
+
+        if csv_type == ENROLLMENT_CSV.INDIVIDUAL_ENROLLMENTS.value:
+            return StreamingHttpResponse(
+                IndividualEnrollmentsCSVRenderer().render(self._stream_serialized_data(enrollments)),
+                content_type="text/csv",
+                headers={"Content-Disposition": 'attachment; filename="individual_enrollments.csv"'},
+            )
+
+        paginator = self.pagination_class()
+        page = paginator.paginate_queryset(enrollments, request)
+        serialized_data = page.data.to_dict(orient='records')
+        response = paginator.get_paginated_response(serialized_data)
+
+        return response
+
+    def _stream_serialized_data(self, enrollments, chunk_size=50000):
+        """
+        Stream the serialized data.
+        """
+        total_rows = enrollments.shape[0]
+        for start_index in range(0, total_rows, chunk_size):
+            end_index = min(start_index + chunk_size, total_rows)
+            chunk = enrollments.iloc[start_index:end_index]
+            yield from chunk.to_dict(orient='records')
+
+
+class AdvanceAnalyticsEnrollmentStatsView(APIView):
+    """
+    API for getting the advance analytics enrollment chart stats.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ['get']
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
+    def get(self, request, enterprise_uuid):  # lint-amnesty, pylint: disable=inconsistent-return-statements
+        """Get enrollment chart stats"""
+        serializer = AdvanceAnalyticsEnrollmentStatsSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+
+        cache_expiry = fetch_enrollments_cache_expiry_timestamp()
+        enrollments_df = fetch_and_cache_enrollments_data(enterprise_uuid, cache_expiry)
+
+        # get values from query params or use default
+        start_date = serializer.data.get('start_date', enrollments_df.enterprise_enrollment_date.min())
+        end_date = serializer.data.get('end_date', datetime.now())
+        granularity = serializer.data.get('granularity', GRANULARITY.DAILY.value)
+        calculation = serializer.data.get('calculation', CALCULATION.TOTAL.value)
+        csv_type = serializer.data.get('csv_type')
+
+        if csv_type is None:
+            data = {
+                "enrollments_over_time": self.construct_enrollments_over_time(
+                    enrollments_df.copy(),
+                    start_date,
+                    end_date,
+                    granularity,
+                    calculation,
+                ),
+                "top_courses_by_enrollments": self.construct_top_courses_by_enrollments(
+                    enrollments_df.copy(),
+                    start_date,
+                    end_date,
+                ),
+                "top_subjects_by_enrollments": self.construct_top_subjects_by_enrollments(
+                    enrollments_df.copy(),
+                    start_date,
+                    end_date,
+                ),
+            }
+            return Response(data)
+        elif csv_type == ENROLLMENT_CSV.ENROLLMENTS_OVER_TIME.value:
+            return self.construct_enrollments_over_time_csv(
+                enrollments_df.copy(),
+                start_date,
+                end_date,
+                granularity,
+                calculation,
+            )
+        elif csv_type == ENROLLMENT_CSV.TOP_COURSES_BY_ENROLLMENTS.value:
+            return self.construct_top_courses_by_enrollments_csv(
+                enrollments_df.copy(),
+                start_date,
+                end_date,
+            )
+        elif csv_type == ENROLLMENT_CSV.TOP_SUBJECTS_BY_ENROLLMENTS.value:
+            return self.construct_top_subjects_by_enrollments_csv(
+                enrollments_df.copy(),
+                start_date,
+                end_date,
+            )
+
+    def enrollments_over_time_common(self, enrollments_df, start_date, end_date, granularity, calculation):
+        """
+        Common method for constructing enrollments over time data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+            granularity {str} -- Granularity of the data. One of GRANULARITY choices
+            calculation {str} -- Calculation of the data. One of CALCULATION choices
+        """
+        # filter enrollments by date
+        enrollments = date_filter(start_date, end_date, enrollments_df, "enterprise_enrollment_date")
+
+        # aggregate enrollments by granularity
+        enrollments = granularity_aggregation(
+            level=granularity,
+            group=["enterprise_enrollment_date", "enroll_type"],
+            date="enterprise_enrollment_date",
+            data_frame=enrollments,
+        )
+
+        # aggregate enrollments by calculation
+        enrollments = calculation_aggregation(calc=calculation, data_frame=enrollments)
+
+        return enrollments
+
+    def construct_enrollments_over_time(self, enrollments_df, start_date, end_date, granularity, calculation):
+        """
+        Construct enrollments over time data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+            granularity {str} -- Granularity of the data. One of GRANULARITY choices
+            calculation {str} -- Calculation of the data. One of CALCULATION choices
+        """
+        enrollments = self.enrollments_over_time_common(enrollments_df, start_date, end_date, granularity, calculation)
+
+        # convert dataframe to a list of records
+        return enrollments.to_dict(orient='records')
+
+    def construct_enrollments_over_time_csv(self, enrollments_df, start_date, end_date, granularity, calculation):
+        """
+        Construct enrollments over time CSV.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+            granularity {str} -- Granularity of the data. One of GRANULARITY choices
+            calculation {str} -- Calculation of the data. One of CALCULATION choices
+        """
+        enrollments = self.enrollments_over_time_common(enrollments_df, start_date, end_date, granularity, calculation)
+
+        enrollments = enrollments.pivot(
+            index="enterprise_enrollment_date", columns="enroll_type", values="count"
+        )
+
+        filename = f"Enrollment Timeseries, {start_date} - {end_date} ({granularity} {calculation}).csv"
+        return self.construct_csv_response(enrollments, filename)
+
+    def top_courses_by_enrollments_common(self, enrollments_df, start_date, end_date, group_by_columns, columns):
+        """
+        Common method for constructing top courses by enrollments data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+            group_by_columns {list} -- List of columns to group by
+            columns {list} -- List of column for the final result
+        """
+        # filter enrollments by date
+        enrollments = date_filter(start_date, end_date, enrollments_df, "enterprise_enrollment_date")
+
+        courses = list(
+            enrollments.groupby(["course_key"]).size().sort_values(ascending=False)[:10].index
+        )
+
+        enrollments = (
+            enrollments[enrollments.course_key.isin(courses)]
+            .groupby(group_by_columns)
+            .size()
+            .reset_index()
+        )
+        enrollments.columns = columns
+
+        return enrollments
+
+    def construct_top_courses_by_enrollments(self, enrollments_df, start_date, end_date):
+        """
+        Construct top courses by enrollments data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+        """
+        group_by_columns = ["course_key", "enroll_type"]
+        columns = ["course_key", "enroll_type", "count"]
+        enrollments = self.top_courses_by_enrollments_common(
+            enrollments_df,
+            start_date,
+            end_date,
+            group_by_columns,
+            columns
+        )
+
+        # convert dataframe to a list of records
+        return enrollments.to_dict(orient='records')
+
+    def construct_top_courses_by_enrollments_csv(self, enrollments_df, start_date, end_date):
+        """
+        Construct top courses by enrollments CSV.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+        """
+        group_by_columns = ["course_key", "course_title", "enroll_type"]
+        columns = ["course_key", "course_title", "enroll_type", "count"]
+        enrollments = self.top_courses_by_enrollments_common(
+            enrollments_df,
+            start_date,
+            end_date,
+            group_by_columns,
+            columns
+        )
+
+        enrollments = enrollments.pivot(
+            index=["course_key", "course_title"], columns="enroll_type", values="count"
+        )
+
+        filename = f"Top 10 Courses, {start_date} - {end_date}.csv"
+        return self.construct_csv_response(enrollments, filename)
+
+    def top_subjects_by_enrollments_common(self, enrollments_df, start_date, end_date):
+        """
+        Common method for constructing top subjects by enrollments data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+        """
+        # filter enrollments by date
+        enrollments = date_filter(start_date, end_date, enrollments_df, "enterprise_enrollment_date")
+
+        subjects = list(
+            enrollments.groupby(["course_subject"]).size().sort_values(ascending=False)[:10].index
+        )
+
+        enrollments = (
+            enrollments[enrollments.course_subject.isin(subjects)]
+            .groupby(["course_subject", "enroll_type"])
+            .size()
+            .reset_index()
+        )
+        enrollments.columns = ["course_subject", "enroll_type", "count"]
+
+        return enrollments
+
+    def construct_top_subjects_by_enrollments(self, enrollments_df, start_date, end_date):
+        """
+        Construct top subjects by enrollments data.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+        """
+        enrollments = self.top_subjects_by_enrollments_common(enrollments_df, start_date, end_date)
+        # convert dataframe to a list of records
+        return enrollments.to_dict(orient='records')
+
+    def construct_top_subjects_by_enrollments_csv(self, enrollments_df, start_date, end_date):
+        """
+        Construct top subjects by enrollments CSV.
+
+        Arguments:
+            enrollments_df {DataFrame} -- DataFrame of enrollments
+            start_date {datetime} -- Enrollment start date in the format 'YYYY-MM-DD'
+            end_date {datetime} -- Enrollment end date in the format 'YYYY-MM-DD'
+        """
+        enrollments = self.top_subjects_by_enrollments_common(enrollments_df, start_date, end_date)
+        enrollments = enrollments.pivot(index="course_subject", columns="enroll_type", values="count")
+
+        filename = f"Top 10 Subjects by Enrollment, {start_date} - {end_date}.csv"
+        return self.construct_csv_response(enrollments, filename)
+
+    def construct_csv_response(self, enrollments, filename):
+        """
+        Construct CSV response.
+
+        Arguments:
+            enrollments {DataFrame} -- DataFrame of enrollments
+            filename {str} -- Filename for the CSV
+        """
+        response = HttpResponse(content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+        enrollments.to_csv(path_or_buf=response)
+
+        return response

enterprise_data/renderers.py

@@ -29,3 +29,17 @@ class EnrollmentsCSVRenderer(CSVStreamingRenderer):
         'enterprise_sso_uid', 'created', 'course_api_url', 'total_learning_time_hours', 'is_subsidy',
         'course_product_line', 'budget_id', 'enterprise_group_name', 'enterprise_group_uuid',
     ]
+
+
+class IndividualEnrollmentsCSVRenderer(CSVStreamingRenderer):
+    """
+    Custom streaming csv renderer for advance analytics individual enrollments data.
+    """
+
+    header = [
+        'email',
+        'course_title',
+        'course_subject',
+        'enroll_type',
+        'enterprise_enrollment_date',
+    ]

enterprise_data/tests/admin_analytics/mock_enrollments.py

@@ -0,0 +1,169 @@
+"""Mock data for enrollments"""
+
+import pandas as pd
+
+from enterprise_data.admin_analytics.constants import ENROLLMENT_CSV
+
+ENROLLMENTS = [
+    {
+        "enterprise_customer_name": "Hill Ltd",
+        "enterprise_customer_uuid": "33ce656295e04ecfa2a77d407eb96f69",
+        "lms_enrollment_id": 1013,
+        "user_id": 8907,
+        "email": "rebeccanelson@example.com",
+        "course_key": "hEmW+tvk03",
+        "courserun_key": "course-v1:hEmW+tvk03+1T9889",
+        "course_id": "1681",
+        "course_subject": "business-management",
+        "course_title": "Re-engineered tangible approach",
+        "enterprise_enrollment_date": "2021-07-04",
+        "lms_enrollment_mode": "verified",
+        "enroll_type": "certificate",
+        "program_title": "Non-Program",
+        "date_certificate_awarded": "2021-08-25",
+        "grade_percent": 0.99,
+        "cert_awarded": 1,
+        "date_certificate_created_raw": "2021-08-25",
+        "passed_date_raw": "2021-08-25",
+        "passed_date": "2021-08-25",
+        "has_passed": 1,
+    },
+    {
+        "enterprise_customer_name": "Hill Ltd",
+        "enterprise_customer_uuid": "33ce656295e04ecfa2a77d407eb96f69",
+        "lms_enrollment_id": 9172,
+        "user_id": 8369,
+        "email": "taylorjames@example.com",
+        "course_key": "hEmW+tvk03",
+        "courserun_key": "course-v1:hEmW+tvk03+1T9889",
+        "course_id": "1681",
+        "course_subject": "business-management",
+        "course_title": "Re-engineered tangible approach",
+        "enterprise_enrollment_date": "2021-07-03",
+        "lms_enrollment_mode": "verified",
+        "enroll_type": "certificate",
+        "program_title": "Non-Program",
+        "date_certificate_awarded": "2021-09-01",
+        "grade_percent": 0.93,
+        "cert_awarded": 1,
+        "date_certificate_created_raw": "2021-09-01",
+        "passed_date_raw": "2021-09-01",
+        "passed_date": "2021-09-01",
+        "has_passed": 1,
+    },
+    {
+        "enterprise_customer_name": "Hill Ltd",
+        "enterprise_customer_uuid": "33ce656295e04ecfa2a77d407eb96f69",
+        "lms_enrollment_id": 9552,
+        "user_id": 8719,
+        "email": "ssmith@example.com",
+        "course_key": "qZJC+KFX86",
+        "courserun_key": "course-v1:qZJC+KFX86+1T8918",
+        "course_id": "1725",
+        "course_subject": "medicine",
+        "course_title": "Secured static capability",
+        "enterprise_enrollment_date": "2021-05-11",
+        "lms_enrollment_mode": "verified",
+        "enroll_type": "certificate",
+        "program_title": "Non-Program",
+        "date_certificate_awarded": None,
+        "grade_percent": 0.0,
+        "cert_awarded": 0,
+        "date_certificate_created_raw": None,
+        "passed_date_raw": None,
+        "passed_date": None,
+        "has_passed": 0,
+    },
+    {
+        "enterprise_customer_name": "Hill Ltd",
+        "enterprise_customer_uuid": "33ce656295e04ecfa2a77d407eb96f69",
+        "lms_enrollment_id": 3436,
+        "user_id": 3125,
+        "email": "kathleenmartin@example.com",
+        "course_key": "QWXx+Jqz64",
+        "courserun_key": "course-v1:QWXx+Jqz64+1T9449",
+        "course_id": "4878",
+        "course_subject": "social-sciences",
+        "course_title": "Horizontal solution-oriented hub",
+        "enterprise_enrollment_date": "2020-04-03",
+        "lms_enrollment_mode": "verified",
+        "enroll_type": "certificate",
+        "program_title": "Non-Program",
+        "date_certificate_awarded": None,
+        "grade_percent": 0.0,
+        "cert_awarded": 0,
+        "date_certificate_created_raw": None,
+        "passed_date_raw": None,
+        "passed_date": None,
+        "has_passed": 0,
+    },
+    {
+        "enterprise_customer_name": "Hill Ltd",
+        "enterprise_customer_uuid": "33ce656295e04ecfa2a77d407eb96f69",
+        "lms_enrollment_id": 5934,
+        "user_id": 4853,
+        "email": "amber79@example.com",
+        "course_key": "NOGk+UVD31",
+        "courserun_key": "course-v1:NOGk+UVD31+1T4956",
+        "course_id": "4141",
+        "course_subject": "communication",
+        "course_title": "Streamlined zero-defect attitude",
+        "enterprise_enrollment_date": "2020-04-08",
+        "lms_enrollment_mode": "verified",
+        "enroll_type": "certificate",
+        "program_title": "Non-Program",
+        "date_certificate_awarded": None,
+        "grade_percent": 0.0,
+        "cert_awarded": 0,
+        "date_certificate_created_raw": None,
+        "passed_date_raw": None,
+        "passed_date": None,
+        "has_passed": 0,
+    },
+]
+
+ENROLLMENT_STATS_CSVS = {
+    ENROLLMENT_CSV.ENROLLMENTS_OVER_TIME.value: (
+        b'enterprise_enrollment_date,certificate\n'
+        b'2020-04-03,1\n'
+        b'2020-04-08,1\n'
+        b'2021-05-11,1\n'
+        b'2021-07-03,1\n'
+        b'2021-07-04,1\n'
+    ),
+    ENROLLMENT_CSV.TOP_COURSES_BY_ENROLLMENTS.value: (
+        b'course_key,course_title,certificate\n'
+        b'NOGk+UVD31,Streamlined zero-defect attitude,1\n'
+        b'QWXx+Jqz64,Horizontal solution-oriented hub,1\n'
+        b'hEmW+tvk03,Re-engineered tangible approach,2\n'
+        b'qZJC+KFX86,Secured static capability,1\n'
+    ),
+    ENROLLMENT_CSV.TOP_SUBJECTS_BY_ENROLLMENTS.value: (
+        b'course_subject,certificate\nbusiness-management,2\ncommunication,1\nmedicine,1\nsocial-sciences,1\n'
+    )
+}
+
+
+def enrollments_dataframe():
+    """Return a DataFrame of enrollments."""
+    enrollments = pd.DataFrame(ENROLLMENTS)
+
+    enrollments['enterprise_enrollment_date'] = enrollments['enterprise_enrollment_date'].astype('datetime64[ns]')
+    enrollments['date_certificate_awarded'] = enrollments['date_certificate_awarded'].astype('datetime64[ns]')
+    enrollments['date_certificate_created_raw'] = enrollments['date_certificate_created_raw'].astype('datetime64[ns]')
+    enrollments['passed_date_raw'] = enrollments['passed_date_raw'].astype('datetime64[ns]')
+    enrollments['passed_date'] = enrollments['passed_date'].astype('datetime64[ns]')
+
+    return enrollments
+
+
+def enrollments_csv_content():
+    """Return the CSV content of enrollments."""
+    return (
+        b'email,course_title,course_subject,enroll_type,enterprise_enrollment_date\r\n'
+        b'rebeccanelson@example.com,Re-engineered tangible approach,business-management,certificate,2021-07-04\r\n'
+        b'taylorjames@example.com,Re-engineered tangible approach,business-management,certificate,2021-07-03\r\n'
+        b'ssmith@example.com,Secured static capability,medicine,certificate,2021-05-11\r\n'
+        b'amber79@example.com,Streamlined zero-defect attitude,communication,certificate,2020-04-08\r\n'
+        b'kathleenmartin@example.com,Horizontal solution-oriented hub,social-sciences,certificate,2020-04-03\r\n'
+    )