edx-enterprise-data 9.1.1__tar.gz → 9.2.1__tar.gz

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/CHANGELOG.rst

@@ -16,6 +16,14 @@ Unreleased
 
 =========================
 
+[9.2.1] - 2024-09-25
+---------------------
+  * fix: Added temporary cache logging on EnterpriseLearnerEnrollmentViewSet.
+
+[9.2.0] - 2024-09-25
+---------------------
+  * refactor: Performance optimizations for leaderboard API endpoints
+
 [9.1.1] - 2024-09-24
 ---------------------
   * fix: disable caching for EnterpriseLearnerEnrollmentViewSet

{edx_enterprise_data-9.1.1/edx_enterprise_data.egg-info → edx_enterprise_data-9.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 9.1.1
+Version: 9.2.1
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 9.1.1
+Version: 9.2.1
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/edx_enterprise_data.egg-info/SOURCES.txt

@@ -23,7 +23,6 @@ enterprise_data/utils.py
 enterprise_data/admin_analytics/__init__.py
 enterprise_data/admin_analytics/constants.py
 enterprise_data/admin_analytics/data_loaders.py
-enterprise_data/admin_analytics/utils.py
 enterprise_data/admin_analytics/database/__init__.py
 enterprise_data/admin_analytics/database/utils.py
 enterprise_data/admin_analytics/database/queries/__init__.py
@@ -42,7 +41,6 @@ enterprise_data/api/v0/serializers.py
 enterprise_data/api/v0/urls.py
 enterprise_data/api/v0/views.py
 enterprise_data/api/v1/__init__.py
-enterprise_data/api/v1/paginators.py
 enterprise_data/api/v1/serializers.py
 enterprise_data/api/v1/urls.py
 enterprise_data/api/v1/views/__init__.py
@@ -132,7 +130,6 @@ enterprise_data/tests/admin_analytics/test_analytics_enrollments.py
 enterprise_data/tests/admin_analytics/test_analytics_leaderboard.py
 enterprise_data/tests/admin_analytics/test_data_loaders.py
 enterprise_data/tests/admin_analytics/test_enterprise_completions.py
-enterprise_data/tests/admin_analytics/test_utils.py
 enterprise_data/tests/api/__init__.py
 enterprise_data/tests/api/v0/__init__.py
 enterprise_data/tests/api/v0/test_serializers.py

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "9.1.1"
+__version__ = "9.2.1"

edx_enterprise_data-9.2.1/enterprise_data/admin_analytics/constants.py

@@ -0,0 +1,11 @@
+"""
+Constants for admin analytics app.
+"""
+
+from enum import Enum
+
+
+class ResponseType(Enum):
+    """Response type choices"""
+    JSON = 'json'
+    CSV = 'csv'

edx_enterprise_data-9.2.1/enterprise_data/admin_analytics/data_loaders.py

@@ -0,0 +1,24 @@
+"""
+Utility functions for fetching data from the database.
+"""
+from logging import getLogger
+
+import pandas
+
+from enterprise_data.admin_analytics.database import run_query
+
+LOGGER = getLogger(__name__)
+
+
+def fetch_max_enrollment_datetime():
+    """
+    Fetch the latest created date from the enterprise_learner_enrollment table.
+
+    created will be same for all records as this is added at the time of data load. Which is when the async process
+    populates the data in the table. We can use this to get the latest data load time.
+    """
+    query = "SELECT MAX(created) FROM enterprise_learner_enrollment"
+    results = run_query(query)
+    if not results:
+        return None
+    return pandas.to_datetime(results[0][0])

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/admin_analytics/database/queries/fact_engagement_admin_dash.py

@@ -109,3 +109,88 @@ class FactEngagementAdminDashQueries:
             GROUP BY activity_date, enroll_type
             ORDER BY activity_date;
         """
+
+    @staticmethod
+    def get_all_leaderboard_data_query():
+        """
+        Get the query to fetch the leaderboard data.
+
+        Query should fetch the leaderboard data for the enterprise customer to show in the data table.
+
+        Returns:
+            (str): Query to fetch the leaderboard data.
+        """
+        return """
+            WITH Engagement AS (
+                SELECT
+                    email,
+                    ROUND(SUM(learning_time_seconds) / 3600, 1) as learning_time_hours,
+                    SUM(is_engaged) as session_count,
+                    CASE
+                        WHEN SUM(is_engaged) = 0 THEN 0.0
+                        ELSE ROUND(SUM(learning_time_seconds) / 3600 / SUM(is_engaged), 1)
+                    END AS average_session_length
+                FROM fact_enrollment_engagement_day_admin_dash
+                WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                    (activity_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                    is_engaged = 1
+                GROUP BY email
+            ),
+            Completions AS (
+                SELECT email, count(course_key) as course_completion_count
+                FROM fact_enrollment_admin_dash
+                WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                    (passed_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                    has_passed = 1
+                GROUP BY email
+            )
+            SELECT
+                Engagement.email,
+                Engagement.learning_time_hours,
+                Engagement.session_count,
+                Engagement.average_session_length,
+                Completions.course_completion_count
+            FROM Engagement
+            LEFT JOIN Completions
+            ON Engagement.email = Completions.email
+            ORDER BY
+                Engagement.learning_time_hours DESC,
+                Engagement.session_count DESC,
+                Completions.course_completion_count DESC
+            LIMIT %(limit)s OFFSET %(offset)s;
+        """
+
+    @staticmethod
+    def get_leaderboard_data_count_query():
+        """
+        Get the query to fetch the leaderboard row count.
+
+        Query should fetch the count of rows for the leaderboard data for the enterprise customer.
+
+        Returns:
+            (str): Query to fetch the leaderboard row count.
+        """
+        return """
+            WITH Engagement AS (
+                SELECT
+                    email
+                FROM fact_enrollment_engagement_day_admin_dash
+                WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                    (activity_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                    is_engaged = 1
+                GROUP BY email
+            ),
+            Completions AS (
+                SELECT email, count(course_key) as course_completion_count
+                FROM fact_enrollment_admin_dash
+                WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                    (passed_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                    has_passed = 1
+                GROUP BY email
+            )
+            SELECT
+                count(*)
+            FROM Engagement
+            LEFT JOIN Completions
+            ON Engagement.email = Completions.email
+        """

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/admin_analytics/database/tables/fact_engagement_admin_dash.py

@@ -152,3 +152,53 @@ class FactEngagementAdminDashTable(BaseTable):
             },
             as_dict=True,
         )
+
+    def get_all_leaderboard_data(
+            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+    ):
+        """
+        Get the leaderboard data for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+            limit (int): The maximum number of records to return.
+            offset (int): The number of records to skip.
+
+        Returns:
+            list[dict]: The leaderboard data.
+        """
+        return run_query(
+            query=self.queries.get_all_leaderboard_data_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+                'limit': limit,
+                'offset': offset,
+            },
+            as_dict=True,
+        )
+
+    def get_leaderboard_data_count(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
+        """
+        Get the total number of leaderboard records for the given enterprise customer.
+
+        Arguments:
+            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            start_date (date): The start date.
+            end_date (date): The end date.
+
+        Returns:
+            (int): The total number of leaderboard records.
+        """
+        results = run_query(
+            query=self.queries.get_leaderboard_data_count_query(),
+            params={
+                'enterprise_customer_uuid': enterprise_customer_uuid,
+                'start_date': start_date,
+                'end_date': end_date,
+            }
+        )
+        return results[0][0]

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/serializers.py

@@ -5,7 +5,7 @@ from uuid import UUID
 
 from rest_framework import serializers
 
-from enterprise_data.admin_analytics.constants import Calculation, Granularity, ResponseType
+from enterprise_data.admin_analytics.constants import ResponseType
 from enterprise_data.models import (
     EnterpriseAdminLearnerProgress,
     EnterpriseAdminSummarizeInsights,
@@ -241,23 +241,8 @@ class AdvanceAnalyticsQueryParamSerializer(serializers.Serializer):  # pylint: d
         ResponseType.JSON.value,
         ResponseType.CSV.value
     ]
-    GRANULARITY_CHOICES = [
-        Granularity.DAILY.value,
-        Granularity.WEEKLY.value,
-        Granularity.MONTHLY.value,
-        Granularity.QUARTERLY.value
-    ]
-    CALCULATION_CHOICES = [
-        Calculation.TOTAL.value,
-        Calculation.RUNNING_TOTAL.value,
-        Calculation.MOVING_AVERAGE_3_PERIOD.value,
-        Calculation.MOVING_AVERAGE_7_PERIOD.value
-    ]
-
     start_date = serializers.DateField(required=False)
     end_date = serializers.DateField(required=False)
-    granularity = serializers.CharField(required=False)
-    calculation = serializers.CharField(required=False)
     response_type = serializers.CharField(required=False)
     page = serializers.IntegerField(required=False, min_value=1)
     page_size = serializers.IntegerField(required=False, min_value=2)
@@ -287,25 +272,3 @@ class AdvanceAnalyticsQueryParamSerializer(serializers.Serializer):  # pylint: d
         if value not in self.RESPONSE_TYPES:
             raise serializers.ValidationError(f"response_type must be one of {self.RESPONSE_TYPES}")
         return value
-
-    def validate_granularity(self, value):
-        """
-        Validate the granularity value.
-
-        Raises:
-            serializers.ValidationError: If granularity is not one of the valid choices.
-        """
-        if value not in self.GRANULARITY_CHOICES:
-            raise serializers.ValidationError(f"Granularity must be one of {self.GRANULARITY_CHOICES}")
-        return value
-
-    def validate_calculation(self, value):
-        """
-        Validate the calculation value.
-
-        Raises:
-            serializers.ValidationError: If calculation is not one of the valid choices
-        """
-        if value not in self.CALCULATION_CHOICES:
-            raise serializers.ValidationError(f"Calculation must be one of {self.CALCULATION_CHOICES}")
-        return value

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/urls.py

@@ -58,8 +58,8 @@ urlpatterns = [
     ),
     re_path(
         fr'^admin/analytics/(?P<enterprise_uuid>{UUID4_REGEX})/leaderboard$',
-        AdvanceAnalyticsLeaderboardView.as_view(),
-        name='enterprise-admin-analytics-leaderboard'
+        AdvanceAnalyticsLeaderboardView.as_view({'get': 'list'}),
+        name='enterprise-admin-analytics-leaderboard-list'
     ),
     re_path(
         fr'^admin/analytics/(?P<enterprise_uuid>{UUID4_REGEX})/enrollments/stats$',

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/views/analytics_completions.py

@@ -15,7 +15,6 @@ from django.http import StreamingHttpResponse
 
 from enterprise_data.admin_analytics.constants import ResponseType
 from enterprise_data.admin_analytics.database.tables import FactEnrollmentAdminDashTable
-from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
 from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
 from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
 from enterprise_data.renderers import IndividualCompletionsCSVRenderer
@@ -33,7 +32,6 @@ class AdvanceAnalyticsCompletionsView(AnalyticsPaginationMixin, ViewSet):
     2. `enterprise_data_api_v1.enterprise-learner-completion-stats`: Get completion stats data.
     """
     authentication_classes = (JwtAuthentication,)
-    pagination_class = AdvanceAnalyticsPagination
     http_method_names = ('get', )
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/views/analytics_engagements.py

@@ -15,7 +15,6 @@ from django.http import StreamingHttpResponse
 
 from enterprise_data.admin_analytics.constants import ResponseType
 from enterprise_data.admin_analytics.database.tables import FactEngagementAdminDashTable, FactEnrollmentAdminDashTable
-from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
 from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
 from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
 from enterprise_data.renderers import IndividualEngagementsCSVRenderer
@@ -33,7 +32,6 @@ class AdvanceAnalyticsEngagementView(AnalyticsPaginationMixin, ViewSet):
     2. `enterprise_data_api_v1.enterprise-learner-engagement-stats`: Get engagement stats data.
     """
     authentication_classes = (JwtAuthentication,)
-    pagination_class = AdvanceAnalyticsPagination
     http_method_names = ('get', )
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/views/analytics_enrollments.py

@@ -14,7 +14,6 @@ from django.http import StreamingHttpResponse
 
 from enterprise_data.admin_analytics.constants import ResponseType
 from enterprise_data.admin_analytics.database.tables import FactEnrollmentAdminDashTable
-from enterprise_data.api.v1.paginators import AdvanceAnalyticsPagination
 from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
 from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
 from enterprise_data.renderers import IndividualEnrollmentsCSVRenderer
@@ -32,7 +31,6 @@ class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
     2. `enterprise_data_api_v1.enterprise-learner-enrollment-stats`: Get enrollment stats data.
     """
     authentication_classes = (JwtAuthentication,)
-    pagination_class = AdvanceAnalyticsPagination
     http_method_names = ('get', )
 
     @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)

edx_enterprise_data-9.2.1/enterprise_data/api/v1/views/analytics_leaderboard.py

@@ -0,0 +1,107 @@
+"""
+Views for fetching leaderboard data.
+"""
+
+from datetime import datetime
+from logging import getLogger
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework.viewsets import ViewSet
+
+from django.http import StreamingHttpResponse
+
+from enterprise_data.admin_analytics.constants import ResponseType
+from enterprise_data.admin_analytics.database.tables import FactEngagementAdminDashTable, FactEnrollmentAdminDashTable
+from enterprise_data.api.v1.serializers import AdvanceAnalyticsQueryParamSerializer
+from enterprise_data.api.v1.views.base import AnalyticsPaginationMixin
+from enterprise_data.renderers import LeaderboardCSVRenderer
+
+LOGGER = getLogger(__name__)
+
+
+class AdvanceAnalyticsLeaderboardView(AnalyticsPaginationMixin, ViewSet):
+    """
+    View to handle requests for enterprise leaderboard data.
+
+    Here is the list of URLs that are handled by this view:
+    1. `enterprise_data_api_v1.enterprise-admin-analytics-leaderboard-list`: Get leaderboard data.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ('get', )
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_uuid: enterprise_uuid)
+    def list(self, request, enterprise_uuid):
+        """
+        Get individual leaderboard data for the enterprise.
+        """
+        # Remove hyphens from the UUID
+        enterprise_uuid = enterprise_uuid.replace('-', '')
+
+        serializer = AdvanceAnalyticsQueryParamSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+        min_enrollment_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(
+            enterprise_uuid,
+        )
+
+        # get values from query params or use default values
+        start_date = serializer.data.get('start_date', min_enrollment_date)
+        end_date = serializer.data.get('end_date', datetime.now())
+        page = serializer.data.get('page', 1)
+        page_size = serializer.data.get('page_size', 100)
+        leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+            limit=page_size,
+            offset=(page - 1) * page_size,
+        )
+        total_count = FactEngagementAdminDashTable().get_leaderboard_data_count(
+            enterprise_customer_uuid=enterprise_uuid,
+            start_date=start_date,
+            end_date=end_date,
+        )
+        response_type = request.query_params.get('response_type', ResponseType.JSON.value)
+
+        LOGGER.info(
+            'Leaderboard data requested for enterprise [%s] from [%s] to [%s]',
+            enterprise_uuid,
+            start_date,
+            end_date,
+        )
+
+        if response_type == ResponseType.CSV.value:
+            filename = f'Leaderboard, {start_date} - {end_date}.csv'
+
+            return StreamingHttpResponse(
+                LeaderboardCSVRenderer().render(self._stream_serialized_data(
+                    enterprise_uuid, start_date, end_date, total_count
+                )),
+                content_type='text/csv',
+                headers={'Content-Disposition': f'attachment; filename="{filename}"'},
+            )
+
+        return self.get_paginated_response(
+            request=request,
+            records=leaderboard,
+            page=page,
+            page_size=page_size,
+            total_count=total_count,
+        )
+
+    @staticmethod
+    def _stream_serialized_data(enterprise_uuid, start_date, end_date, total_count, page_size=50000):
+        """
+        Stream the serialized data.
+        """
+        offset = 0
+        while offset < total_count:
+            leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
+                enterprise_customer_uuid=enterprise_uuid,
+                start_date=start_date,
+                end_date=end_date,
+                limit=page_size,
+                offset=offset,
+            )
+            yield from leaderboard
+            offset += page_size

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/api/v1/views/enterprise_learner.py

@@ -6,6 +6,7 @@ from datetime import date, timedelta
 from logging import getLogger
 from uuid import UUID
 
+from edx_django_utils.cache import TieredCache
 from rest_framework import filters, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -18,12 +19,13 @@ from django.db.models.functions import Coalesce
 from django.http import StreamingHttpResponse
 from django.utils import timezone
 
+from enterprise_data.admin_analytics.database.utils import LOGGER
 from enterprise_data.api.v1 import serializers
 from enterprise_data.filters import AuditEnrollmentsFilterBackend, AuditUsersEnrollmentFilterBackend
 from enterprise_data.models import EnterpriseLearner, EnterpriseLearnerEnrollment
 from enterprise_data.paginators import EnterpriseEnrollmentsPagination
 from enterprise_data.renderers import EnrollmentsCSVRenderer
-from enterprise_data.utils import subtract_one_month
+from enterprise_data.utils import get_cache_key, subtract_one_month
 
 from .base import EnterpriseViewSetMixin
 
@@ -82,19 +84,26 @@ class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOn
         enterprise_customer_uuid = self.kwargs['enterprise_id']
 
         # TODO: Created a ticket ENT0-9531 to fix the cache issue
-        # Reason for Comenting cache: Remove the cache for this ViewSet
-        # becuae the cache is not working as expected
-        # cache_key = get_cache_key(
-        #     resource='enterprise-learner',
-        #     enterprise_customer=enterprise_customer_uuid,
-        # )
-        # cached_response = TieredCache.get_cached_response(cache_key)
-        # if cached_response.is_found:
-        #     return cached_response.value
-        # else:
+        # Temporary logging to test cache issue
+        try:
+            LOGGER.info("Trying to get Learner Enrollment data from Cache")
+            cache_key = get_cache_key(
+                resource='enterprise-learner',
+                enterprise_customer=enterprise_customer_uuid,
+            )
+            cached_response = TieredCache.get_cached_response(cache_key)
+            if cached_response.is_found:
+                LOGGER.info("Learner Enrollment data found in Cache for Enterprise: [%s]", enterprise_customer_uuid)
+            else:
+                LOGGER.info("Learner Enrollment data not found in Cache for Enterprise: [%s]", enterprise_customer_uuid)
+                TieredCache.set_all_tiers(cache_key, 'dummy_enrollments', DEFAULT_LEARNER_CACHE_TIMEOUT)
+                LOGGER.info("Cache set for Enterprise: [%s] whose key is: [%s]", enterprise_customer_uuid, cache_key)
+        except Exception as e:
+            LOGGER.error("Memcached connection test failed: %s", e)
+
         enrollments = EnterpriseLearnerEnrollment.objects.filter(enterprise_customer_uuid=enterprise_customer_uuid)
         enrollments = self.apply_filters(enrollments)
-        # TieredCache.set_all_tiers(cache_key, enrollments, DEFAULT_LEARNER_CACHE_TIMEOUT)
+
         return enrollments
 
     def list(self, request, *args, **kwargs):

{edx_enterprise_data-9.1.1 → edx_enterprise_data-9.2.1}/enterprise_data/renderers.py

@@ -82,7 +82,7 @@ class LeaderboardCSVRenderer(CSVStreamingRenderer):
     header = [
         'email',
         'learning_time_hours',
-        'daily_sessions',
+        'session_count',
         'average_session_length',
-        'course_completions',
+        'course_completion_count',
     ]