edx-enterprise-data 8.0.0__tar.gz → 8.3.0__tar.gz

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/CHANGELOG.rst

@@ -15,6 +15,18 @@ Unreleased
 ----------
 
 =========================
+[8.3.0] - 2024-07-25
+---------------------
+  * refactor: Refactor code to avoid error conditions.
+
+[8.2.0] - 2024-07-25
+---------------------
+  * Added a new API endpoint to get admin analytics aggregated data on user enrollment and engagement.
+
+[8.1.0] - 2024-07-22
+---------------------
+  * Upgrade python requirements
+
 [8.0.0] - 2024-07-18
 ---------------------
   * Fix migration for EnterpriseLearnerEnrollment model

{edx_enterprise_data-8.0.0/edx_enterprise_data.egg-info → edx_enterprise_data-8.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.0.0
+Version: 8.3.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX
@@ -23,6 +23,9 @@ Requires-Dist: edx-opaque-keys
 Requires-Dist: edx-rbac
 Requires-Dist: edx-rest-api-client
 Requires-Dist: factory_boy
+Requires-Dist: mysql-connector-python
+Requires-Dist: numpy<=1.24.4
+Requires-Dist: pandas<=2.0.3
 Requires-Dist: requests
 Requires-Dist: rules
 Provides-Extra: reporting

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.0.0
+Version: 8.3.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX
@@ -23,6 +23,9 @@ Requires-Dist: edx-opaque-keys
 Requires-Dist: edx-rbac
 Requires-Dist: edx-rest-api-client
 Requires-Dist: factory_boy
+Requires-Dist: mysql-connector-python
+Requires-Dist: numpy<=1.24.4
+Requires-Dist: pandas<=2.0.3
 Requires-Dist: requests
 Requires-Dist: rules
 Provides-Extra: reporting

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/edx_enterprise_data.egg-info/SOURCES.txt

@@ -20,6 +20,10 @@ enterprise_data/renderers.py
 enterprise_data/signals.py
 enterprise_data/urls.py
 enterprise_data/utils.py
+enterprise_data/admin_analytics/__init__.py
+enterprise_data/admin_analytics/data_loaders.py
+enterprise_data/admin_analytics/database.py
+enterprise_data/admin_analytics/utils.py
 enterprise_data/api/__init__.py
 enterprise_data/api/urls.py
 enterprise_data/api/v0/__init__.py
@@ -29,7 +33,11 @@ enterprise_data/api/v0/views.py
 enterprise_data/api/v1/__init__.py
 enterprise_data/api/v1/serializers.py
 enterprise_data/api/v1/urls.py
-enterprise_data/api/v1/views.py
+enterprise_data/api/v1/views/__init__.py
+enterprise_data/api/v1/views/base.py
+enterprise_data/api/v1/views/enterprise_admin.py
+enterprise_data/api/v1/views/enterprise_learner.py
+enterprise_data/api/v1/views/enterprise_offers.py
 enterprise_data/fixtures/enterprise_enrollment.json
 enterprise_data/fixtures/enterprise_user.json
 enterprise_data/management/__init__.py
@@ -99,12 +107,17 @@ enterprise_data/tests/test_filters.py
 enterprise_data/tests/test_models.py
 enterprise_data/tests/test_utils.py
 enterprise_data/tests/test_views.py
+enterprise_data/tests/admin_analytics/__init__.py
+enterprise_data/tests/admin_analytics/test_data_loaders.py
+enterprise_data/tests/admin_analytics/test_utils.py
 enterprise_data/tests/api/__init__.py
 enterprise_data/tests/api/v0/__init__.py
 enterprise_data/tests/api/v0/test_serializers.py
 enterprise_data/tests/api/v1/__init__.py
 enterprise_data/tests/api/v1/test_serializers.py
 enterprise_data/tests/api/v1/test_views.py
+enterprise_data/tests/api/v1/views/__init__.py
+enterprise_data/tests/api/v1/views/test_enterprise_admin.py
 enterprise_data_roles/__init__.py
 enterprise_data_roles/admin.py
 enterprise_data_roles/apps.py

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/edx_enterprise_data.egg-info/requires.txt

@@ -9,6 +9,9 @@ edx-opaque-keys
 edx-rbac
 edx-rest-api-client
 factory_boy
+mysql-connector-python
+numpy<=1.24.4
+pandas<=2.0.3
 requests
 rules
 

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "8.0.0"
+__version__ = "8.3.0"

edx_enterprise_data-8.3.0/enterprise_data/admin_analytics/data_loaders.py

@@ -0,0 +1,137 @@
+"""
+Utility functions for fetching data from the database.
+"""
+import numpy
+import pandas
+
+from django.http import Http404
+
+from enterprise_data.admin_analytics.database import run_query
+
+
+def get_select_query(table: str, columns: list, enterprise_uuid: str) -> str:
+    """
+    Generate a SELECT query for the given table and columns.
+
+    Arguments:
+        table (str): The table to query.
+        columns (list): The columns to select.
+        enterprise_uuid (str): The UUID of the enterprise customer.
+
+    Returns:
+        (str): The SELECT query.
+    """
+    return f'SELECT {", ".join(columns)} FROM {table} WHERE enterprise_customer_uuid = "{enterprise_uuid}"'
+
+
+def fetch_enrollment_data(enterprise_uuid: str):
+    """
+    Fetch enrollment data from the database for the given enterprise customer.
+
+    Arguments:
+        enterprise_uuid (str): The UUID of the enterprise customer.
+
+    Returns:
+        (pandas.DataFrame): The enrollment data.
+    """
+    enterprise_uuid = enterprise_uuid.replace('-', '')
+
+    columns = [
+        'enterprise_customer_name',
+        'enterprise_customer_uuid',
+        'lms_enrollment_id',
+        'user_id',
+        'email',
+        'course_key',
+        'courserun_key',
+        'course_id',
+        'course_subject',
+        'course_title',
+        'enterprise_enrollment_date',
+        'lms_enrollment_mode',
+        'enroll_type',
+        'program_title',
+        'date_certificate_awarded',
+        'grade_percent',
+        'cert_awarded',
+        'date_certificate_created_raw',
+        'passed_date_raw',
+        'passed_date',
+        'has_passed',
+    ]
+    query = get_select_query(
+        table='fact_enrollment_admin_dash',
+        columns=columns,
+        enterprise_uuid=enterprise_uuid,
+    )
+
+    results = run_query(query=query)
+    if not results:
+        raise Http404(f'No enrollment data found for enterprise {enterprise_uuid}')
+
+    enrollments = pandas.DataFrame(numpy.array(results), columns=columns)
+
+    # Convert date columns to datetime.
+    enrollments['enterprise_enrollment_date'] = enrollments['enterprise_enrollment_date'].astype('datetime64[ns]')
+    enrollments['date_certificate_awarded'] = enrollments['date_certificate_awarded'].astype('datetime64[ns]')
+    enrollments['date_certificate_created_raw'] = enrollments['date_certificate_created_raw'].astype('datetime64[ns]')
+    enrollments['passed_date_raw'] = enrollments['passed_date_raw'].astype('datetime64[ns]')
+    enrollments['passed_date'] = enrollments['passed_date'].astype('datetime64[ns]')
+
+    return enrollments
+
+
+def fetch_engagement_data(enterprise_uuid: str):
+    """
+    Fetch engagement data from the database for the given enterprise customer.
+
+    Arguments:
+        enterprise_uuid (str): The UUID of the enterprise customer.
+
+    Returns:
+        (pandas.DataFrame): The engagement data.
+    """
+    enterprise_uuid = enterprise_uuid.replace('-', '')
+
+    columns = [
+        'user_id',
+        'email',
+        'enterprise_customer_uuid',
+        'course_key',
+        'enroll_type',
+        'activity_date',
+        'course_title',
+        'course_subject',
+        'is_engaged',
+        'is_engaged_video',
+        'is_engaged_forum',
+        'is_engaged_problem',
+        'is_active',
+        'learning_time_seconds',
+    ]
+    query = get_select_query(
+        table='fact_enrollment_engagement_day_admin_dash', columns=columns, enterprise_uuid=enterprise_uuid
+    )
+
+    results = run_query(query=query)
+    if not results:
+        raise Http404(f'No engagement data found for enterprise {enterprise_uuid}')
+
+    engagement = pandas.DataFrame(numpy.array(results), columns=columns)
+    engagement['activity_date'] = engagement['activity_date'].astype('datetime64[ns]')
+
+    return engagement
+
+
+def fetch_max_enrollment_datetime():
+    """
+    Fetch the latest created date from the enterprise_learner_enrollment table.
+
+    created will be same for all records as this is added at the time of data load. Which is when the async process
+    populates the data in the table. We can use this to get the latest data load time.
+    """
+    query = "SELECT MAX(created) FROM enterprise_learner_enrollment"
+    results = run_query(query)
+    if not results:
+        return None
+    return pandas.to_datetime(results[0][0])

edx_enterprise_data-8.3.0/enterprise_data/admin_analytics/database.py

@@ -0,0 +1,50 @@
+"""
+Utility functions for interacting with the database.
+"""
+from contextlib import closing
+from logging import getLogger
+
+from mysql.connector import connect
+
+from django.conf import settings
+
+from enterprise_data.utils import timeit
+
+LOGGER = getLogger(__name__)
+
+
+def get_db_connection(database=settings.ENTERPRISE_REPORTING_DB_ALIAS):
+    """
+    Get a connection to the database.
+
+    Returns:
+        (mysql.connector.connection.MySQLConnection): The database connection.
+    """
+    return connect(
+        host=settings.DATABASES[database]['HOST'],
+        port=settings.DATABASES[database]['PORT'],
+        database=settings.DATABASES[database]['NAME'],
+        user=settings.DATABASES[database]['USER'],
+        password=settings.DATABASES[database]['PASSWORD'],
+    )
+
+
+@timeit
+def run_query(query):
+    """
+    Run a query on the database and return the results.
+
+    Arguments:
+        query (str): The query to run.
+
+    Returns:
+        (list): The results of the query.
+    """
+    try:
+        with closing(get_db_connection()) as connection:
+            with closing(connection.cursor()) as cursor:
+                cursor.execute(query)
+                return cursor.fetchall()
+    except Exception:
+        LOGGER.exception(f'[run_query]: run_query failed for query "{query}".')
+        raise

edx_enterprise_data-8.3.0/enterprise_data/admin_analytics/utils.py

@@ -0,0 +1,81 @@
+"""
+Utility functions for fetching data from the database.
+"""
+from datetime import datetime
+
+from edx_django_utils.cache import TieredCache, get_cache_key
+
+from enterprise_data.admin_analytics.data_loaders import fetch_engagement_data, fetch_enrollment_data
+
+
+def get_cache_timeout(cache_expiry):
+    """
+    Helper method to calculate cache timeout in seconds.
+
+    Arguments:
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (int): Cache timeout in seconds.
+    """
+    now = datetime.now()
+    cache_timeout = 0
+    if cache_expiry > now:
+        # Calculate cache expiry in seconds from now.
+        cache_timeout = (cache_expiry - now).seconds
+
+    return cache_timeout
+
+
+def fetch_and_cache_enrollments_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache enrollments data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The enrollments data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-enrollments',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        enrollments = fetch_enrollment_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, enrollments, get_cache_timeout(cache_expiry)
+        )
+        return enrollments
+
+
+def fetch_and_cache_engagements_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache engagements data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The engagements data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-engagements',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        engagements = fetch_engagement_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, engagements, get_cache_timeout(cache_expiry)
+        )
+        return engagements

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/enterprise_data/api/v1/serializers.py

@@ -196,3 +196,23 @@ class EnterpriseAdminSummarizeInsightsSerializer(serializers.ModelSerializer):
     class Meta:
         model = EnterpriseAdminSummarizeInsights
         fields = '__all__'
+
+
+class AdminAnalyticsAggregatesQueryParamsSerializer(serializers.Serializer):  # pylint: disable=abstract-method
+    """
+    Serializer for validating admin analytics query params.
+    """
+    start_date = serializers.DateField(required=False)
+    end_date = serializers.DateField(required=False)
+
+    def validate(self, attrs):
+        """
+        Validate the query params.
+
+        Raises:
+            serializers.ValidationError: If start_date is greater than end_date.
+        """
+        if 'start_date' in attrs and 'end_date' in attrs:
+            if attrs['start_date'] > attrs['end_date']:
+                raise serializers.ValidationError("start_date should be less than or equal to end_date.")
+        return attrs

{edx_enterprise_data-8.0.0 → edx_enterprise_data-8.3.0}/enterprise_data/api/v1/urls.py

@@ -7,7 +7,9 @@ from rest_framework.routers import DefaultRouter
 
 from django.urls import re_path
 
-from enterprise_data.api.v1 import views
+from enterprise_data.api.v1.views import enterprise_admin as enterprise_admin_views
+from enterprise_data.api.v1.views import enterprise_learner as enterprise_learner_views
+from enterprise_data.api.v1.views import enterprise_offers as enterprise_offers_views
 from enterprise_data.constants import UUID4_REGEX
 
 app_name = 'enterprise_data_api_v1'
@@ -15,31 +17,36 @@ app_name = 'enterprise_data_api_v1'
 router = DefaultRouter()
 router.register(
     r'enterprise/(?P<enterprise_id>.+)/enrollments',
-    views.EnterpriseLearnerEnrollmentViewSet,
+    enterprise_learner_views.EnterpriseLearnerEnrollmentViewSet,
     'enterprise-learner-enrollment',
 )
 router.register(
     r'enterprise/(?P<enterprise_id>.+)/offers',
-    views.EnterpriseOfferViewSet,
+    enterprise_offers_views.EnterpriseOfferViewSet,
     'enterprise-offers',
 )
 router.register(
     r'enterprise/(?P<enterprise_id>.+)/users',
-    views.EnterpriseLearnerViewSet,
+    enterprise_learner_views.EnterpriseLearnerViewSet,
     'enterprise-learner',
 )
 router.register(
     r'enterprise/(?P<enterprise_id>.+)/learner_completed_courses',
-    views.EnterpriseLearnerCompletedCoursesViewSet,
+    enterprise_learner_views.EnterpriseLearnerCompletedCoursesViewSet,
     'enterprise-learner-completed-courses',
 )
 
 urlpatterns = [
     re_path(
         fr'^admin/insights/(?P<enterprise_id>{UUID4_REGEX})$',
-        views.EnterpriseAdminInsightsView.as_view(),
+        enterprise_admin_views.EnterpriseAdminInsightsView.as_view(),
         name='enterprise-admin-insights'
     ),
+    re_path(
+        fr'^admin/anlaytics/(?P<enterprise_id>{UUID4_REGEX})$',
+        enterprise_admin_views.EnterpriseAdminAnalyticsAggregatesView.as_view(),
+        name='enterprise-admin-analytics-aggregates'
+    ),
 ]
 
 urlpatterns += router.urls

edx_enterprise_data-8.3.0/enterprise_data/api/v1/views/base.py

@@ -0,0 +1,26 @@
+"""
+Base views for enterprise data api v1.
+"""
+from edx_rbac.mixins import PermissionRequiredMixin
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from edx_rest_framework_extensions.paginators import DefaultPagination
+
+from enterprise_data.constants import ANALYTICS_API_VERSION_1
+
+
+class EnterpriseViewSetMixin(PermissionRequiredMixin):
+    """
+    Base class for all Enterprise view sets.
+    """
+    authentication_classes = (JwtAuthentication,)
+    pagination_class = DefaultPagination
+    permission_required = 'can_access_enterprise'
+    API_VERSION = ANALYTICS_API_VERSION_1
+
+    def paginate_queryset(self, queryset):
+        """
+        Allows no_page query param to skip pagination
+        """
+        if 'no_page' in self.request.query_params:
+            return None
+        return super().paginate_queryset(queryset)

edx_enterprise_data-8.3.0/enterprise_data/api/v1/views/enterprise_admin.py

@@ -0,0 +1,109 @@
+"""
+Views for enterprise admin api v1.
+"""
+from datetime import datetime, timedelta
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework.response import Response
+from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND
+from rest_framework.views import APIView
+
+from enterprise_data.admin_analytics.data_loaders import fetch_max_enrollment_datetime
+from enterprise_data.admin_analytics.utils import fetch_and_cache_engagements_data, fetch_and_cache_enrollments_data
+from enterprise_data.api.v1 import serializers
+from enterprise_data.models import EnterpriseAdminLearnerProgress, EnterpriseAdminSummarizeInsights
+from enterprise_data.utils import date_filter
+
+
+class EnterpriseAdminInsightsView(APIView):
+    """
+    API for getting the enterprise admin insights.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ['get']
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id)
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin insights
+        """
+        response_data = {}
+        learner_progress = {}
+        learner_engagement = {}
+
+        try:
+            learner_progress = EnterpriseAdminLearnerProgress.objects.get(enterprise_customer_uuid=enterprise_id)
+            learner_progress = serializers.EnterpriseAdminLearnerProgressSerializer(learner_progress).data
+            response_data['learner_progress'] = learner_progress
+        except EnterpriseAdminLearnerProgress.DoesNotExist:
+            pass
+
+        try:
+            learner_engagement = EnterpriseAdminSummarizeInsights.objects.get(enterprise_customer_uuid=enterprise_id)
+            learner_engagement = serializers.EnterpriseAdminSummarizeInsightsSerializer(learner_engagement).data
+            response_data['learner_engagement'] = learner_engagement
+        except EnterpriseAdminSummarizeInsights.DoesNotExist:
+            pass
+
+        status = HTTP_200_OK
+        if learner_progress == {} and learner_engagement == {}:
+            status = HTTP_404_NOT_FOUND
+
+        return Response(data=response_data, status=status)
+
+
+class EnterpriseAdminAnalyticsAggregatesView(APIView):
+    """
+    API for getting the enterprise admin analytics aggregates.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ['get']
+
+    @permission_required('can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id)
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin aggregate data.
+        """
+        serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+
+        last_updated_at = fetch_max_enrollment_datetime()
+        cache_expiry = last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+
+        enrollment = fetch_and_cache_enrollments_data(enterprise_id, cache_expiry).copy()
+        engagement = fetch_and_cache_engagements_data(enterprise_id, cache_expiry).copy()
+        # Use start and end date if provided by the client, if client has not provided then use
+        # 1. minimum enrollment date from the data as the start_date
+        # 2. today's date as the end_date
+        start_date = serializer.data.get('start_date', enrollment.enterprise_enrollment_date.min())
+        end_date = serializer.data.get('end_date', datetime.now())
+
+        # Date filtering.
+        dff = date_filter(
+            start=start_date, end=end_date, data_frame=enrollment.copy(), date_column='enterprise_enrollment_date'
+        )
+
+        enrolls = len(dff)
+        courses = len(dff.course_key.unique())
+
+        dff = date_filter(start=start_date, end=end_date, data_frame=enrollment.copy(), date_column='passed_date')
+
+        completions = dff.has_passed.sum()
+
+        # Date filtering.
+        dff = date_filter(start=start_date, end=end_date, data_frame=engagement.copy(), date_column='activity_date')
+
+        hours = round(dff.learning_time_seconds.sum() / 60 / 60, 1)
+        sessions = dff.is_engaged.sum()
+
+        return Response(data={
+            'enrolls': enrolls,
+            'courses': courses,
+            'completions': completions,
+            'hours': hours,
+            'sessions': sessions,
+            'last_updated_at': last_updated_at.date() if last_updated_at else None,
+            'min_enrollment_date': enrollment.enterprise_enrollment_date.min().date(),
+            'max_enrollment_date': enrollment.enterprise_enrollment_date.max().date(),
+        }, status=HTTP_200_OK)