edx-enterprise-data 10.0.1__tar.gz → 10.2.0__tar.gz

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/CHANGELOG.rst

@@ -16,6 +16,15 @@ Unreleased
 
 =========================
 
+[10.2.0] - 2024-11-12
+---------------------
+  * Fixed null email issue for leaderboard.
+
+
+[10.1.0] - 2024-10-29
+---------------------
+  * Added management command to pre-warm analytics data.
+
 [10.0.1] - 2024-10-25
 ---------------------
   * Same as ``10.0.0`` 

{edx_enterprise_data-10.0.1/edx_enterprise_data.egg-info → edx_enterprise_data-10.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 10.0.1
+Version: 10.2.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 10.0.1
+Version: 10.2.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/edx_enterprise_data.egg-info/SOURCES.txt

@@ -65,12 +65,14 @@ enterprise_data/management/commands/create_enterprise_learner_enrollment_lpr_v1.
 enterprise_data/management/commands/create_enterprise_learner_lpr_v1.py
 enterprise_data/management/commands/create_enterprise_offer.py
 enterprise_data/management/commands/create_enterprise_user.py
+enterprise_data/management/commands/pre_warm_analytics_cache.py
 enterprise_data/management/commands/tests/__init__.py
 enterprise_data/management/commands/tests/test_create_dummy_data_lpr_v1.py
 enterprise_data/management/commands/tests/test_create_enterprise_enrollment.py
 enterprise_data/management/commands/tests/test_create_enterprise_learner_enrollment_lpr_v1.py
 enterprise_data/management/commands/tests/test_create_enterprise_learner_lpr_v1.py
 enterprise_data/management/commands/tests/test_create_enterprise_user.py
+enterprise_data/management/commands/tests/test_pre_warm_analytics_cache.py
 enterprise_data/migrations/0001_initial.py
 enterprise_data/migrations/0002_auto_20180430_1358.py
 enterprise_data/migrations/0003_auto_20180501_0603.py

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "10.0.1"
+__version__ = "10.2.0"

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/queries/fact_engagement_admin_dash.py

@@ -163,6 +163,55 @@ class FactEngagementAdminDashQueries:
             GROUP BY email;
         """
 
+    @staticmethod
+    def get_engagement_data_for_leaderboard_null_email_only_query():
+        """
+        Get the query to fetch the engagement data for leaderboard for NULL emails only.
+
+        Query should fetch the engagement data for like learning time, session length of
+        the enterprise learners to show in the leaderboard.
+
+        Returns:
+            (str): Query to fetch the engagement data for leaderboard.
+        """
+        return """
+            SELECT
+                email,
+                ROUND(SUM(learning_time_seconds) / 3600, 1) as learning_time_hours,
+                SUM(is_engaged) as session_count,
+                CASE
+                    WHEN SUM(is_engaged) = 0 THEN 0.0
+                    ELSE ROUND(SUM(learning_time_seconds) / 3600 / SUM(is_engaged), 1)
+                END AS average_session_length
+            FROM fact_enrollment_engagement_day_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                (activity_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                is_engaged = 1 AND
+                email is NULL
+            GROUP BY email;
+        """
+
+    @staticmethod
+    def get_completion_data_for_leaderboard_null_email_only_query():
+        """
+        Get the query to fetch the completions data for leaderboard for NULL emails.
+
+        Query should fetch the completion data for like course completion count of
+        the enterprise learners to show in the leaderboard.
+
+        Returns:
+            (list<str>): Query to fetch the completions data for leaderboard.
+        """
+        return """
+            SELECT email, count(course_key) as course_completion_count
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                (passed_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                has_passed = 1 AND
+                email is NULL
+            GROUP BY email;
+        """
+
     @staticmethod
     def get_leaderboard_data_count_query():
         """

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/queries/fact_enrollment_admin_dash.py

@@ -7,6 +7,18 @@ class FactEnrollmentAdminDashQueries:
     """
     Queries related to the fact_enrollment_admin_dash table.
     """
+    @staticmethod
+    def get_top_enterprises_query(count=10):
+        """
+        Get the query to fetch the top enterprises by enrollments.
+        """
+        return f"""
+            SELECT enterprise_customer_uuid
+            FROM fact_enrollment_admin_dash
+            GROUP BY enterprise_customer_uuid
+            ORDER BY COUNT(enterprise_customer_uuid) DESC LIMIT {count};
+        """
+
     @staticmethod
     def get_enrollment_count_query():
         """

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/tables/fact_engagement_admin_dash.py

@@ -5,6 +5,7 @@ from datetime import date
 from uuid import UUID
 
 from enterprise_data.cache.decorators import cache_it
+from enterprise_data.utils import find_first
 
 from ..queries import FactEngagementAdminDashQueries
 from ..utils import run_query
@@ -168,7 +169,14 @@ class FactEngagementAdminDashTable(BaseTable):
 
     @cache_it()
     def _get_engagement_data_for_leaderboard(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            limit: int,
+            offset: int,
+            include_null_email: bool,
+
     ):
         """
         Get the engagement data for leaderboard.
@@ -182,11 +190,12 @@ class FactEngagementAdminDashTable(BaseTable):
             end_date (date): The end date.
             limit (int): The maximum number of records to return.
             offset (int): The number of records to skip.
+            include_null_email (bool): If True, only fetch data for NULL emails.
 
         Returns:
             list[dict]: The engagement data for leaderboard.
         """
-        return run_query(
+        engagements = run_query(
             query=self.queries.get_engagement_data_for_leaderboard_query(),
             params={
                 'enterprise_customer_uuid': enterprise_customer_uuid,
@@ -198,9 +207,27 @@ class FactEngagementAdminDashTable(BaseTable):
             as_dict=True,
         )
 
+        if include_null_email:
+            engagement_for_null_email = run_query(
+                query=self.queries.get_engagement_data_for_leaderboard_null_email_only_query(),
+                params={
+                    'enterprise_customer_uuid': enterprise_customer_uuid,
+                    'start_date': start_date,
+                    'end_date': end_date,
+                },
+                as_dict=True,
+            )
+            engagements += engagement_for_null_email
+        return engagements
+
     @cache_it()
     def _get_completion_data_for_leaderboard_query(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, email_list: list
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            email_list: list,
+            include_null_email: bool,
     ):
         """
         Get the completion data for leaderboard.
@@ -213,11 +240,13 @@ class FactEngagementAdminDashTable(BaseTable):
             start_date (date): The start date.
             end_date (date): The end date.
             email_list (list<str>): List of emails of the enterprise learners.
+            include_null_email (bool): If True, only fetch data for NULL emails.
 
         Returns:
             list[dict]: The engagement data for leaderboard.
         """
-        return run_query(
+
+        completions = run_query(
             query=self.queries.get_completion_data_for_leaderboard_query(email_list),
             params={
                 'enterprise_customer_uuid': enterprise_customer_uuid,
@@ -227,8 +256,28 @@ class FactEngagementAdminDashTable(BaseTable):
             as_dict=True,
         )
 
+        if include_null_email:
+            completions_for_null_email = run_query(
+                query=self.queries.get_completion_data_for_leaderboard_null_email_only_query(),
+                params={
+                    'enterprise_customer_uuid': enterprise_customer_uuid,
+                    'start_date': start_date,
+                    'end_date': end_date,
+                },
+                as_dict=True,
+            )
+            completions += completions_for_null_email
+
+        return completions
+
     def get_all_leaderboard_data(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            limit: int,
+            offset: int,
+            total_count: int,
     ):
         """
         Get the leaderboard data for the given enterprise customer.
@@ -239,32 +288,48 @@ class FactEngagementAdminDashTable(BaseTable):
             end_date (date): The end date.
             limit (int): The maximum number of records to return.
             offset (int): The number of records to skip.
+            total_count (int): The total number of records.
 
         Returns:
             list[dict]: The leaderboard data.
         """
+        include_null_email = False
+        # If this is the last or only page, we need to include NULL emails record.
+        if total_count <= offset + limit:
+            include_null_email = True
+
         engagement_data = self._get_engagement_data_for_leaderboard(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
             limit=limit,
             offset=offset,
+            include_null_email=include_null_email,
         )
         # If there is no data, no need to proceed.
         if not engagement_data:
             return []
 
-        engagement_data_dict = {engagement['email']: engagement for engagement in engagement_data}
+        engagement_data_dict = {
+            engagement['email']: engagement for engagement in engagement_data if engagement['email']
+        }
         completion_data = self._get_completion_data_for_leaderboard_query(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
             email_list=list(engagement_data_dict.keys()),
+            include_null_email=include_null_email,
         )
         for completion in completion_data:
             email = completion['email']
             engagement_data_dict[email]['course_completion_count'] = completion['course_completion_count']
 
+        if include_null_email:
+            engagement_data_dict['None'] = find_first(engagement_data, lambda x: x['email'] is None) or {}
+            completion = find_first(completion_data, lambda x: x['email'] is None) or \
+                {'course_completion_count': 'Unknown'}
+            engagement_data_dict['None']['course_completion_count'] = completion['course_completion_count']
+
         return list(engagement_data_dict.values())
 
     @cache_it()

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/tables/fact_enrollment_admin_dash.py

@@ -17,6 +17,22 @@ class FactEnrollmentAdminDashTable(BaseTable):
     """
     queries = FactEnrollmentAdminDashQueries()
 
+    def get_top_enterprises(self, count=10):
+        """
+        Get the top enterprises by enrollments.
+
+        Arguments:
+            count (int): The number of enterprises to return.
+
+        Returns:
+            list<str>: A list of enterprise UUIDs.
+        """
+        result = run_query(
+            query=self.queries.get_top_enterprises_query(count),
+            as_dict=False,
+        )
+        return [row[0] for row in result]
+
     @cache_it()
     def get_enrollment_count(self, enterprise_customer_uuid: UUID, start_date: date, end_date: date):
         """
@@ -77,7 +93,7 @@ class FactEnrollmentAdminDashTable(BaseTable):
         Get the enrollment date range for the given enterprise customer.
 
         Arguments:
-            enterprise_customer_uuid (UUID): The UUID of the enterprise customer.
+            enterprise_customer_uuid (UUID | str): The UUID of the enterprise customer.
 
         Returns:
             (tuple<date, date>): The minimum and maximum enrollment dates.

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/analytics_completions.py

@@ -2,7 +2,7 @@
 Views for enterprise admin completions analytics.
 """
 
-from datetime import datetime
+from datetime import date
 from logging import getLogger
 
 from edx_rbac.decorators import permission_required
@@ -50,7 +50,7 @@ class AdvanceAnalyticsCompletionsView(AnalyticsPaginationMixin, ViewSet):
 
         # get values from query params or use default values
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         page = serializer.data.get('page', 1)
         page_size = serializer.data.get('page_size', 100)
         completions = FactEnrollmentAdminDashTable().get_all_completions(
@@ -132,7 +132,7 @@ class AdvanceAnalyticsCompletionsView(AnalyticsPaginationMixin, ViewSet):
         )
         # get values from query params or use default
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         with timer('construct_completion_all_stats'):
             data = {
                 'completions_over_time': FactEnrollmentAdminDashTable().get_completions_time_series_data(

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/analytics_engagements.py

@@ -2,7 +2,7 @@
 Views for handling REST endpoints related to Engagements analytics.
 """
 
-from datetime import datetime
+from datetime import date
 from logging import getLogger
 
 from edx_rbac.decorators import permission_required
@@ -50,7 +50,7 @@ class AdvanceAnalyticsEngagementView(AnalyticsPaginationMixin, ViewSet):
 
         # get values from query params or use default values
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         page = serializer.data.get('page', 1)
         page_size = serializer.data.get('page_size', 100)
         engagements = FactEngagementAdminDashTable().get_all_engagements(
@@ -132,7 +132,7 @@ class AdvanceAnalyticsEngagementView(AnalyticsPaginationMixin, ViewSet):
         )
         # get values from query params or use default
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         with timer('construct_engagement_all_stats'):
             data = {
                 'engagement_over_time': FactEngagementAdminDashTable().get_engagement_time_series_data(

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/analytics_enrollments.py

@@ -1,7 +1,7 @@
 """
 Advance Analytics for API endpoints to fetch enterprise enrollments data.
 """
-from datetime import datetime
+from datetime import date
 from logging import getLogger
 
 from edx_rbac.decorators import permission_required
@@ -49,7 +49,7 @@ class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
 
         # get values from query params or use default values
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         page = serializer.data.get('page', 1)
         page_size = serializer.data.get('page_size', 100)
         enrollments = FactEnrollmentAdminDashTable().get_all_enrollments(
@@ -131,7 +131,7 @@ class AdvanceAnalyticsEnrollmentsView(AnalyticsPaginationMixin, ViewSet):
         )
         # get values from query params or use default
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         with timer('construct_enrollment_all_stats'):
             data = {
                 'enrollments_over_time': FactEnrollmentAdminDashTable().get_enrolment_time_series_data(

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/analytics_leaderboard.py

@@ -2,7 +2,7 @@
 Views for fetching leaderboard data.
 """
 
-from datetime import datetime
+from datetime import date
 from logging import getLogger
 
 from edx_rbac.decorators import permission_required
@@ -46,20 +46,21 @@ class AdvanceAnalyticsLeaderboardView(AnalyticsPaginationMixin, ViewSet):
 
         # get values from query params or use default values
         start_date = serializer.data.get('start_date', min_enrollment_date)
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
         page = serializer.data.get('page', 1)
         page_size = serializer.data.get('page_size', 100)
-        leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
+        total_count = FactEngagementAdminDashTable().get_leaderboard_data_count(
             enterprise_customer_uuid=enterprise_uuid,
             start_date=start_date,
             end_date=end_date,
-            limit=page_size,
-            offset=(page - 1) * page_size,
         )
-        total_count = FactEngagementAdminDashTable().get_leaderboard_data_count(
+        leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
             enterprise_customer_uuid=enterprise_uuid,
             start_date=start_date,
             end_date=end_date,
+            limit=page_size,
+            offset=(page - 1) * page_size,
+            total_count=total_count,
         )
         response_type = request.query_params.get('response_type', ResponseType.JSON.value)
 
@@ -102,6 +103,7 @@ class AdvanceAnalyticsLeaderboardView(AnalyticsPaginationMixin, ViewSet):
                 end_date=end_date,
                 limit=page_size,
                 offset=offset,
+                total_count=total_count,
             )
             yield from leaderboard
             offset += page_size

{edx_enterprise_data-10.0.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/enterprise_admin.py

@@ -1,7 +1,7 @@
 """
 Views for enterprise admin api v1.
 """
-from datetime import datetime
+from datetime import date, datetime
 
 from edx_rbac.decorators import permission_required
 from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
@@ -160,7 +160,7 @@ class EnterpriseAdminAnalyticsSkillsView(APIView):
         if (start_date := serializer.data.get('start_date')) is None:
             start_date, _ = FactEnrollmentAdminDashTable().get_enrollment_date_range(enterprise_id)
 
-        end_date = serializer.data.get('end_date', datetime.now())
+        end_date = serializer.data.get('end_date', date.today())
 
         with timer('top_skills'):
             skills = SkillsDailyRollupAdminDashTable().get_top_skills(enterprise_id, start_date, end_date)