edx-enterprise-data 9.7.1__tar.gz → 10.2.0__tar.gz

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/CHANGELOG.rst

@@ -16,6 +16,25 @@ Unreleased
 
 =========================
 
+[10.2.0] - 2024-11-12
+---------------------
+  * Fixed null email issue for leaderboard.
+
+
+[10.1.0] - 2024-10-29
+---------------------
+  * Added management command to pre-warm analytics data.
+
+[10.0.1] - 2024-10-25
+---------------------
+  * Same as ``10.0.0`` 
+  * Bumping the version so a new tag can be created in the GitHub 
+
+[10.0.0] - 2024-10-25
+---------------------
+  * feat!: Python 3.12 Upgrade
+  * Dropped support for ``Python<3.12``
+
 [9.7.0] - 2024-10-23
 ---------------------
   * feat: Add API to fetch enterprise budgets information

{edx_enterprise_data-9.7.1/edx_enterprise_data.egg-info → edx_enterprise_data-10.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 9.7.1
+Version: 10.2.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX
@@ -9,7 +9,6 @@ License: AGPL 3.0
 Classifier: Framework :: Django
 Classifier: Framework :: Django :: 4.2
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.12
 License-File: LICENSE
 Requires-Dist: Django
@@ -24,8 +23,8 @@ Requires-Dist: edx-rbac
 Requires-Dist: edx-rest-api-client
 Requires-Dist: factory_boy
 Requires-Dist: mysql-connector-python
-Requires-Dist: numpy<=1.24.4
-Requires-Dist: pandas<=2.0.3
+Requires-Dist: numpy
+Requires-Dist: pandas
 Requires-Dist: requests
 Requires-Dist: rules
 Provides-Extra: reporting

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 9.7.1
+Version: 10.2.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX
@@ -9,7 +9,6 @@ License: AGPL 3.0
 Classifier: Framework :: Django
 Classifier: Framework :: Django :: 4.2
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.12
 License-File: LICENSE
 Requires-Dist: Django
@@ -24,8 +23,8 @@ Requires-Dist: edx-rbac
 Requires-Dist: edx-rest-api-client
 Requires-Dist: factory_boy
 Requires-Dist: mysql-connector-python
-Requires-Dist: numpy<=1.24.4
-Requires-Dist: pandas<=2.0.3
+Requires-Dist: numpy
+Requires-Dist: pandas
 Requires-Dist: requests
 Requires-Dist: rules
 Provides-Extra: reporting

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/edx_enterprise_data.egg-info/requires.txt

@@ -10,8 +10,8 @@ edx-rbac
 edx-rest-api-client
 factory_boy
 mysql-connector-python
-numpy<=1.24.4
-pandas<=2.0.3
+numpy
+pandas
 requests
 rules
 

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "9.7.1"
+__version__ = "10.2.0"

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/queries/fact_engagement_admin_dash.py

@@ -163,6 +163,55 @@ class FactEngagementAdminDashQueries:
             GROUP BY email;
         """
 
+    @staticmethod
+    def get_engagement_data_for_leaderboard_null_email_only_query():
+        """
+        Get the query to fetch the engagement data for leaderboard for NULL emails only.
+
+        Query should fetch the engagement data for like learning time, session length of
+        the enterprise learners to show in the leaderboard.
+
+        Returns:
+            (str): Query to fetch the engagement data for leaderboard.
+        """
+        return """
+            SELECT
+                email,
+                ROUND(SUM(learning_time_seconds) / 3600, 1) as learning_time_hours,
+                SUM(is_engaged) as session_count,
+                CASE
+                    WHEN SUM(is_engaged) = 0 THEN 0.0
+                    ELSE ROUND(SUM(learning_time_seconds) / 3600 / SUM(is_engaged), 1)
+                END AS average_session_length
+            FROM fact_enrollment_engagement_day_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                (activity_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                is_engaged = 1 AND
+                email is NULL
+            GROUP BY email;
+        """
+
+    @staticmethod
+    def get_completion_data_for_leaderboard_null_email_only_query():
+        """
+        Get the query to fetch the completions data for leaderboard for NULL emails.
+
+        Query should fetch the completion data for like course completion count of
+        the enterprise learners to show in the leaderboard.
+
+        Returns:
+            (list<str>): Query to fetch the completions data for leaderboard.
+        """
+        return """
+            SELECT email, count(course_key) as course_completion_count
+            FROM fact_enrollment_admin_dash
+            WHERE enterprise_customer_uuid=%(enterprise_customer_uuid)s AND
+                (passed_date BETWEEN %(start_date)s AND %(end_date)s) AND
+                has_passed = 1 AND
+                email is NULL
+            GROUP BY email;
+        """
+
     @staticmethod
     def get_leaderboard_data_count_query():
         """

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/admin_analytics/database/tables/fact_engagement_admin_dash.py

@@ -5,6 +5,7 @@ from datetime import date
 from uuid import UUID
 
 from enterprise_data.cache.decorators import cache_it
+from enterprise_data.utils import find_first
 
 from ..queries import FactEngagementAdminDashQueries
 from ..utils import run_query
@@ -168,7 +169,14 @@ class FactEngagementAdminDashTable(BaseTable):
 
     @cache_it()
     def _get_engagement_data_for_leaderboard(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            limit: int,
+            offset: int,
+            include_null_email: bool,
+
     ):
         """
         Get the engagement data for leaderboard.
@@ -182,11 +190,12 @@ class FactEngagementAdminDashTable(BaseTable):
             end_date (date): The end date.
             limit (int): The maximum number of records to return.
             offset (int): The number of records to skip.
+            include_null_email (bool): If True, only fetch data for NULL emails.
 
         Returns:
             list[dict]: The engagement data for leaderboard.
         """
-        return run_query(
+        engagements = run_query(
             query=self.queries.get_engagement_data_for_leaderboard_query(),
             params={
                 'enterprise_customer_uuid': enterprise_customer_uuid,
@@ -198,9 +207,27 @@ class FactEngagementAdminDashTable(BaseTable):
             as_dict=True,
         )
 
+        if include_null_email:
+            engagement_for_null_email = run_query(
+                query=self.queries.get_engagement_data_for_leaderboard_null_email_only_query(),
+                params={
+                    'enterprise_customer_uuid': enterprise_customer_uuid,
+                    'start_date': start_date,
+                    'end_date': end_date,
+                },
+                as_dict=True,
+            )
+            engagements += engagement_for_null_email
+        return engagements
+
     @cache_it()
     def _get_completion_data_for_leaderboard_query(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, email_list: list
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            email_list: list,
+            include_null_email: bool,
     ):
         """
         Get the completion data for leaderboard.
@@ -213,11 +240,13 @@ class FactEngagementAdminDashTable(BaseTable):
             start_date (date): The start date.
             end_date (date): The end date.
             email_list (list<str>): List of emails of the enterprise learners.
+            include_null_email (bool): If True, only fetch data for NULL emails.
 
         Returns:
             list[dict]: The engagement data for leaderboard.
         """
-        return run_query(
+
+        completions = run_query(
             query=self.queries.get_completion_data_for_leaderboard_query(email_list),
             params={
                 'enterprise_customer_uuid': enterprise_customer_uuid,
@@ -227,8 +256,28 @@ class FactEngagementAdminDashTable(BaseTable):
             as_dict=True,
         )
 
+        if include_null_email:
+            completions_for_null_email = run_query(
+                query=self.queries.get_completion_data_for_leaderboard_null_email_only_query(),
+                params={
+                    'enterprise_customer_uuid': enterprise_customer_uuid,
+                    'start_date': start_date,
+                    'end_date': end_date,
+                },
+                as_dict=True,
+            )
+            completions += completions_for_null_email
+
+        return completions
+
     def get_all_leaderboard_data(
-            self, enterprise_customer_uuid: UUID, start_date: date, end_date: date, limit: int, offset: int
+            self,
+            enterprise_customer_uuid: UUID,
+            start_date: date,
+            end_date: date,
+            limit: int,
+            offset: int,
+            total_count: int,
     ):
         """
         Get the leaderboard data for the given enterprise customer.
@@ -239,32 +288,48 @@ class FactEngagementAdminDashTable(BaseTable):
             end_date (date): The end date.
             limit (int): The maximum number of records to return.
             offset (int): The number of records to skip.
+            total_count (int): The total number of records.
 
         Returns:
             list[dict]: The leaderboard data.
         """
+        include_null_email = False
+        # If this is the last or only page, we need to include NULL emails record.
+        if total_count <= offset + limit:
+            include_null_email = True
+
         engagement_data = self._get_engagement_data_for_leaderboard(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
             limit=limit,
             offset=offset,
+            include_null_email=include_null_email,
         )
         # If there is no data, no need to proceed.
         if not engagement_data:
             return []
 
-        engagement_data_dict = {engagement['email']: engagement for engagement in engagement_data}
+        engagement_data_dict = {
+            engagement['email']: engagement for engagement in engagement_data if engagement['email']
+        }
         completion_data = self._get_completion_data_for_leaderboard_query(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
             email_list=list(engagement_data_dict.keys()),
+            include_null_email=include_null_email,
         )
         for completion in completion_data:
             email = completion['email']
             engagement_data_dict[email]['course_completion_count'] = completion['course_completion_count']
 
+        if include_null_email:
+            engagement_data_dict['None'] = find_first(engagement_data, lambda x: x['email'] is None) or {}
+            completion = find_first(completion_data, lambda x: x['email'] is None) or \
+                {'course_completion_count': 'Unknown'}
+            engagement_data_dict['None']['course_completion_count'] = completion['course_completion_count']
+
         return list(engagement_data_dict.values())
 
     @cache_it()

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/api/v1/views/analytics_leaderboard.py

@@ -49,17 +49,18 @@ class AdvanceAnalyticsLeaderboardView(AnalyticsPaginationMixin, ViewSet):
         end_date = serializer.data.get('end_date', date.today())
         page = serializer.data.get('page', 1)
         page_size = serializer.data.get('page_size', 100)
-        leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
+        total_count = FactEngagementAdminDashTable().get_leaderboard_data_count(
             enterprise_customer_uuid=enterprise_uuid,
             start_date=start_date,
             end_date=end_date,
-            limit=page_size,
-            offset=(page - 1) * page_size,
         )
-        total_count = FactEngagementAdminDashTable().get_leaderboard_data_count(
+        leaderboard = FactEngagementAdminDashTable().get_all_leaderboard_data(
             enterprise_customer_uuid=enterprise_uuid,
             start_date=start_date,
             end_date=end_date,
+            limit=page_size,
+            offset=(page - 1) * page_size,
+            total_count=total_count,
         )
         response_type = request.query_params.get('response_type', ResponseType.JSON.value)
 
@@ -102,6 +103,7 @@ class AdvanceAnalyticsLeaderboardView(AnalyticsPaginationMixin, ViewSet):
                 end_date=end_date,
                 limit=page_size,
                 offset=offset,
+                total_count=total_count,
             )
             yield from leaderboard
             offset += page_size

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/management/commands/pre_warm_analytics_cache.py

@@ -161,17 +161,18 @@ class Command(BaseCommand):
             start_date=start_date,
             end_date=end_date,
         )
-        enterprise_engagement_table.get_all_leaderboard_data(
+        total_count = enterprise_engagement_table.get_leaderboard_data_count(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
-            limit=page_size,
-            offset=0,
         )
-        enterprise_engagement_table.get_leaderboard_data_count(
+        enterprise_engagement_table.get_all_leaderboard_data(
             enterprise_customer_uuid=enterprise_customer_uuid,
             start_date=start_date,
             end_date=end_date,
+            limit=page_size,
+            offset=0,
+            total_count=total_count,
         )
 
     @staticmethod

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/management/commands/tests/test_pre_warm_analytics_cache.py

@@ -30,9 +30,18 @@ class Test(TestCase):
         get_enrollment_date_range_patcher.start()
         self.addCleanup(get_enrollment_date_range_patcher.stop)
 
-    @patch('enterprise_data.admin_analytics.database.tables.fact_engagement_admin_dash.run_query', MagicMock())
-    @patch('enterprise_data.admin_analytics.database.tables.fact_enrollment_admin_dash.run_query', MagicMock())
-    @patch('enterprise_data.admin_analytics.database.tables.skills_daily_rollup_admin_dash.run_query', MagicMock())
+    @patch(
+        'enterprise_data.admin_analytics.database.tables.fact_engagement_admin_dash.run_query',
+        MagicMock(return_value=[])
+    )
+    @patch(
+        'enterprise_data.admin_analytics.database.tables.fact_enrollment_admin_dash.run_query',
+        MagicMock(return_value=[])
+    )
+    @patch(
+        'enterprise_data.admin_analytics.database.tables.skills_daily_rollup_admin_dash.run_query',
+        MagicMock(return_value=[])
+    )
     @patch('enterprise_data.api.v1.views.analytics_enrollments.FactEnrollmentAdminDashTable.get_top_enterprises')
     @patch('enterprise_data.cache.decorators.cache.set')
     @patch('enterprise_data.cache.decorators.cache.get')
@@ -47,5 +56,5 @@ class Test(TestCase):
 
         call_command('pre_warm_analytics_cache')
 
-        assert mock_get_cache.call_count == 24
-        assert mock_set_cache.call_count == 24
+        assert mock_get_cache.call_count == 23
+        assert mock_set_cache.call_count == 23

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data/utils.py

@@ -97,3 +97,20 @@ def primary_subject_truncate(x):
         return x
     else:
         return "other"
+
+
+def find_first(iterable, condition):
+    """
+    Find the first item in an iterable that satisfies the condition.
+
+    Arguments:
+        iterable (iterable): The iterable to search.
+        condition (function): The condition to satisfy.
+
+    Returns:
+        The first item that satisfies the condition, or None if no item satisfies the condition.
+    """
+    try:
+        return next(item for item in iterable if condition(item))
+    except StopIteration:
+        return None

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/enterprise_data_roles/rules.py

@@ -57,5 +57,5 @@ def request_user_has_explicit_access(*args, **kwargs):
 
 rules.add_perm(
     'can_access_enterprise',
-    request_user_has_implicit_access | request_user_has_explicit_access  # pylint: disable=unsupported-binary-operation
+    request_user_has_implicit_access | request_user_has_explicit_access
 )

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/requirements/base.in

@@ -13,6 +13,6 @@ django-model-utils
 edx-rbac
 rules
 factory_boy
-numpy<=1.24.4
-pandas<=2.0.3
+numpy
+pandas
 mysql-connector-python

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/requirements/base.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    make upgrade
@@ -10,15 +10,15 @@ asgiref==3.8.1
     # via django
 asn1crypto==1.5.1
     # via snowflake-connector-python
-awscli==1.34.29
+awscli==1.35.20
     # via -r requirements/reporting.in
 bcrypt==4.2.0
     # via paramiko
 billiard==4.2.1
     # via celery
-boto3==1.35.29
+boto3==1.35.54
     # via -r requirements/reporting.in
-botocore==1.35.29
+botocore==1.35.54
     # via
     #   awscli
     #   boto3
@@ -35,7 +35,7 @@ cffi==1.17.1
     #   cryptography
     #   pynacl
     #   snowflake-connector-python
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via
     #   requests
     #   snowflake-connector-python
@@ -54,7 +54,7 @@ click-repl==0.3.0
     # via celery
 colorama==0.4.6
     # via awscli
-cryptography==43.0.1
+cryptography==43.0.3
     # via
     #   -r requirements/reporting.in
     #   django-fernet-fields-v2
@@ -100,18 +100,18 @@ djangorestframework==3.15.2
     #   edx-drf-extensions
 djangorestframework-csv==3.0.2
     # via -r requirements/base.in
-dnspython==2.6.1
+dnspython==2.7.0
     # via pymongo
 docutils==0.16
     # via awscli
 drf-jwt==1.19.2
     # via edx-drf-extensions
-edx-django-utils==5.15.0
+edx-django-utils==7.0.0
     # via
     #   -r requirements/base.in
     #   edx-drf-extensions
     #   edx-rest-api-client
-edx-drf-extensions==10.4.0
+edx-drf-extensions==10.5.0
     # via
     #   -r requirements/base.in
     #   edx-rbac
@@ -125,7 +125,7 @@ edx-rest-api-client==6.0.0
     # via -r requirements/base.in
 factory-boy==3.3.1
     # via -r requirements/base.in
-faker==30.0.0
+faker==30.8.2
     # via factory-boy
 filelock==3.16.1
     # via snowflake-connector-python
@@ -143,24 +143,21 @@ kombu==5.4.2
     # via celery
 monotonic==1.6
     # via py2neo
-mysql-connector-python==9.0.0
+mysql-connector-python==9.1.0
     # via -r requirements/base.in
-newrelic==10.0.0
+newrelic==10.2.0
     # via edx-django-utils
-numpy==1.24.4
+numpy==2.1.3
     # via
-    #   -c requirements/constraints.txt
     #   -r requirements/base.in
     #   pandas
 packaging==24.1
     # via
     #   py2neo
     #   snowflake-connector-python
-pandas==2.0.3
-    # via
-    #   -c requirements/constraints.txt
-    #   -r requirements/base.in
-pansi==2020.7.3
+pandas==2.2.3
+    # via -r requirements/base.in
+pansi==2024.11.0
     # via py2neo
 paramiko==3.5.0
     # via -r requirements/reporting.in
@@ -168,11 +165,13 @@ pbr==6.1.0
     # via stevedore
 pgpy==0.6.0
     # via -r requirements/reporting.in
+pillow==11.0.0
+    # via pansi
 platformdirs==4.3.6
     # via snowflake-connector-python
 prompt-toolkit==3.0.48
     # via click-repl
-psutil==6.0.0
+psutil==6.1.0
     # via edx-django-utils
 py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz
     # via -r requirements/reporting.in
@@ -192,7 +191,7 @@ pyjwt[crypto]==2.9.0
     #   snowflake-connector-python
 pyminizip==0.2.6
     # via -r requirements/reporting.in
-pymongo==4.9.1
+pymongo==4.10.1
     # via edx-opaque-keys
 pynacl==1.5.0
     # via
@@ -224,7 +223,7 @@ rsa==4.7.2
     # via awscli
 rules==3.5
     # via -r requirements/base.in
-s3transfer==0.10.2
+s3transfer==0.10.3
     # via
     #   awscli
     #   boto3
@@ -234,11 +233,10 @@ six==1.16.0
     # via
     #   edx-rbac
     #   interchange
-    #   pansi
     #   py2neo
     #   python-dateutil
     #   vertica-python
-snowflake-connector-python==3.12.2
+snowflake-connector-python==3.12.3
     # via -r requirements/reporting.in
 sortedcontainers==2.4.0
     # via snowflake-connector-python
@@ -253,6 +251,7 @@ tomlkit==0.13.2
 typing-extensions==4.12.2
     # via
     #   edx-opaque-keys
+    #   faker
     #   snowflake-connector-python
 tzdata==2024.2
     # via
@@ -261,9 +260,8 @@ tzdata==2024.2
     #   pandas
 unicodecsv==0.14.1
     # via -r requirements/reporting.in
-urllib3==1.26.20
+urllib3==2.2.3
     # via
-    #   -c requirements/constraints.txt
     #   botocore
     #   py2neo
     #   requests

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/requirements/ci.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    make upgrade
@@ -10,9 +10,9 @@ chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
-coverage==7.6.1
+coverage==7.6.4
     # via -r requirements/ci.in
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
 filelock==3.16.1
     # via
@@ -30,7 +30,7 @@ pluggy==1.5.0
     # via tox
 pyproject-api==1.8.0
     # via tox
-tox==4.20.0
+tox==4.23.2
     # via -r requirements/ci.in
-virtualenv==20.26.6
+virtualenv==20.27.1
     # via tox

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/requirements/common_constraints.txt

@@ -30,3 +30,7 @@ django-simple-history==3.0.0
 # This can be unpinned once https://github.com/openedx/edx-platform/issues/34586
 # has been resolved and edx-platform is running with pymongo>=4.4.0
 event-tracking<2.4.1
+
+# Cause: https://github.com/openedx/edx-lint/issues/458
+# This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved.
+pip<24.3

{edx_enterprise_data-9.7.1 → edx_enterprise_data-10.2.0}/requirements/constraints.txt

@@ -12,18 +12,3 @@
 
 # Common constraints for edx repos
 -c common_constraints.txt
-
-# Needed for Django 4.2 + Python 3.12 compatibility
-backports.zoneinfo; python_version<"3.9"
-
-# botocore 1.34.145 depends on urllib3<1.27 and >=1.25.4; python_version < "3.10"
-urllib3<2.0.0
-
-numpy<=1.24.4
-pandas<=2.0.3
-
-# this is the last supported version for python 3.8.x
-astroid<=3.2.4
-
-# upgrade to 5.4.0 causes issues with `importlib`
-edx-lint<=5.3.7