edx-enterprise-data 8.3.1__tar.gz → 8.5.0__tar.gz

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/CHANGELOG.rst

@@ -15,6 +15,15 @@ Unreleased
 ----------
 
 =========================
+
+[8.5.0] - 2024-08-12
+---------------------
+  * Added a new model and REST endpoint to get Exec Ed LC Module Performance data.
+
+[8.4.0] - 2024-08-09
+---------------------
+  * feat: endpoint to get skills aggregated data for an enterprise customer
+
 [8.3.1] - 2024-08-06
 ---------------------
   * Dependency updates

{edx_enterprise_data-8.3.1/edx_enterprise_data.egg-info → edx_enterprise_data-8.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.3.1
+Version: 8.5.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.3.1
+Version: 8.5.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/edx_enterprise_data.egg-info/SOURCES.txt

@@ -96,6 +96,7 @@ enterprise_data/migrations/0037_alter_enterpriseenrollment_consent_granted.py
 enterprise_data/migrations/0038_enterpriseoffer_export_timestamp.py
 enterprise_data/migrations/0039_auto_20240212_1403.py
 enterprise_data/migrations/0040_auto_20240718_0536_squashed_0043_alter_enterpriselearnerenrollment_enterprise_enrollment_id.py
+enterprise_data/migrations/0044_enterpriseexecedlcmoduleperformance.py
 enterprise_data/migrations/__init__.py
 enterprise_data/settings/__init__.py
 enterprise_data/settings/test.py

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "8.3.1"
+__version__ = "8.5.0"

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/enterprise_data/admin_analytics/data_loaders.py

@@ -135,3 +135,48 @@ def fetch_max_enrollment_datetime():
     if not results:
         return None
     return pandas.to_datetime(results[0][0])
+
+
+def fetch_skills_data(enterprise_uuid: str):
+    """
+    Fetch skills data from the database for the given enterprise customer.
+
+    Arguments:
+        enterprise_uuid (str): The UUID of the enterprise customer.
+
+    Returns:
+        (pandas.DataFrame): The skills data.
+    """
+
+    enterprise_uuid = enterprise_uuid.replace('-', '')
+
+    cols = [
+        'course_number',
+        'skill_type',
+        'skill_name',
+        'skill_url',
+        'confidence',
+        'skill_rank',
+        'course_title',
+        'course_key',
+        'level_type',
+        'primary_subject_name',
+        'date',
+        'enterprise_customer_uuid',
+        'enterprise_customer_name',
+        'enrolls',
+        'completions',
+    ]
+    query = get_select_query(
+        table='skills_daily_rollup_admin_dash', columns=cols, enterprise_uuid=enterprise_uuid
+    )
+
+    skills = run_query(query=query)
+
+    if not skills:
+        raise Http404(f'No skills data found for enterprise {enterprise_uuid}')
+
+    skills = pandas.DataFrame(numpy.array(skills), columns=cols)
+    skills['date'] = skills['date'].astype('datetime64[ns]')
+
+    return skills

edx_enterprise_data-8.5.0/enterprise_data/admin_analytics/utils.py

@@ -0,0 +1,257 @@
+"""
+Utility functions for fetching data from the database.
+"""
+from datetime import datetime
+from enum import Enum
+
+from edx_django_utils.cache import TieredCache, get_cache_key
+
+from enterprise_data.admin_analytics.data_loaders import fetch_engagement_data, fetch_enrollment_data, fetch_skills_data
+from enterprise_data.utils import date_filter, primary_subject_truncate
+
+
+class ChartType(Enum):
+    """
+    Chart types.
+    """
+    BUBBLE = 'bubble'
+    TOP_SKILLS_ENROLLMENT = 'top_skills_enrollment'
+    TOP_SKILLS_COMPLETION = 'top_skills_completion'
+
+
+def get_cache_timeout(cache_expiry):
+    """
+    Helper method to calculate cache timeout in seconds.
+
+    Arguments:
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (int): Cache timeout in seconds.
+    """
+    now = datetime.now()
+    cache_timeout = 0
+    if cache_expiry > now:
+        # Calculate cache expiry in seconds from now.
+        cache_timeout = (cache_expiry - now).seconds
+
+    return cache_timeout
+
+
+def fetch_and_cache_enrollments_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache enrollments data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The enrollments data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-enrollments',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        enrollments = fetch_enrollment_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, enrollments, get_cache_timeout(cache_expiry)
+        )
+        return enrollments
+
+
+def fetch_and_cache_engagements_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache engagements data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The engagements data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-engagements',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        engagements = fetch_engagement_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, engagements, get_cache_timeout(cache_expiry)
+        )
+        return engagements
+
+
+def fetch_and_cache_skills_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache skills data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The skills data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregate-skills',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        skills = fetch_skills_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, skills, get_cache_timeout(cache_expiry)
+        )
+        return skills
+
+
+def get_skills_bubble_chart_df(skills_filtered):
+    """ Get the skills data for the bubble chart.
+
+    Args:
+        skills_filtered (list): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The skills data for the bubble chart.
+    """
+
+    # Group by skill_name and skill_type, and aggregate enrolls and completions
+    skills_aggregated = (
+        skills_filtered.groupby(['skill_name', 'skill_type'], as_index=False)
+        .agg(enrolls=('enrolls', 'sum'), completions=('completions', 'sum'))
+    )
+
+    # Convert enrolls and completions to integers
+    skills_aggregated['enrolls'] = skills_aggregated['enrolls'].astype(int)
+    skills_aggregated['completions'] = skills_aggregated['completions'].astype(int)
+
+    # Sort the dataframe by enrolls and completions in descending order
+    skills_aggregated = skills_aggregated.sort_values(by=['enrolls', 'completions'], ascending=False)
+
+    return skills_aggregated
+
+
+def get_top_skills_enrollment(skills_filtered):
+    """ Get the top skills by enrolls.
+
+    Args:
+        skills_filtered (pandas.DataFrame): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The top skills by enrolls data
+    """
+
+    # Get the top 10 skills by enrolls
+    top_skills = (
+        skills_filtered.groupby('skill_name')
+        .enrolls.sum()
+        .sort_values(ascending=False)
+        .head(10)
+        .index
+    )
+
+    # Apply primary_subject_truncate to the primary_subject_name column
+    skills_filtered['primary_subject_name'] = skills_filtered['primary_subject_name'].apply(primary_subject_truncate)
+
+    # Filter data for the top skills and aggregate enrolls by skill_name and primary_subject_name
+    top_skills_enrollment_data = (
+        skills_filtered[skills_filtered.skill_name.isin(top_skills)]
+        .groupby(['skill_name', 'primary_subject_name'], as_index=False)
+        .agg(count=('enrolls', 'sum'))
+    )
+
+    # Sort the dataframe by primary_subject_name
+    top_skills_enrollment_data = top_skills_enrollment_data.sort_values(by="primary_subject_name")
+
+    return top_skills_enrollment_data
+
+
+def get_top_skills_completion(skills_filtered):
+    """ Get the top skills by completions.
+
+    Args:
+        skills_filtered (pandas.DataFrame): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The top skills by completions
+    """
+
+    # Get the top 10 skills by completions
+    top_skills = (
+        skills_filtered.groupby('skill_name')
+        .completions.sum()
+        .sort_values(ascending=False)
+        .head(10)
+        .index
+    )
+
+    # Apply primary_subject_truncate to the primary_subject_name column
+    skills_filtered['primary_subject_name'] = skills_filtered['primary_subject_name'].apply(primary_subject_truncate)
+
+    # Filter data for the top skills and aggregate completions by skill_name and primary_subject_name
+    top_skills_completion_data = (
+        skills_filtered[skills_filtered.skill_name.isin(top_skills)]
+        .groupby(['skill_name', 'primary_subject_name'], as_index=False)
+        .agg(count=('completions', 'sum'))
+    )
+
+    # Sort the dataframe by primary_subject_name
+    top_skills_completion_data = top_skills_completion_data.sort_values(by='primary_subject_name')
+
+    return top_skills_completion_data
+
+
+def get_skills_chart_data(chart_type, start_date, end_date, skills):
+    """
+    Get chart data for skill charts.
+
+    Arguments:
+        chart_type (ChartType): The type of chart to generate.
+        start_date (datetime): The start date for the date filter.
+        end_date (datetime): The end date for the date filter.
+        skills (pandas.DataFrame): The skills data.
+    """
+    skills_filtered = date_filter(start=start_date, end=end_date, data_frame=skills.copy(), date_column='date')
+    if chart_type == ChartType.BUBBLE:
+        return get_skills_bubble_chart_df(skills_filtered=skills_filtered.copy())
+    elif chart_type == ChartType.TOP_SKILLS_ENROLLMENT:
+        return get_top_skills_enrollment(skills_filtered=skills_filtered.copy())
+    elif chart_type == ChartType.TOP_SKILLS_COMPLETION:
+        return get_top_skills_completion(skills_filtered=skills_filtered.copy())
+    else:
+        raise ValueError(f"Invalid chart type: {chart_type}")
+
+
+def get_top_skills_csv_data(skills, start_date, end_date):
+    """ Get the top skills data for CSV download.
+
+    Args:
+        skills (pandas.DataFrame): The skills data.
+        start_date (str): The start date for the date filter.
+        end_date (str): The end date for the date filter.
+
+    Returns:
+        (pandas.DataFrame): The top skills data for CSV download.
+    """
+    dff = get_skills_chart_data(
+        chart_type=ChartType.BUBBLE,
+        start_date=start_date,
+        end_date=end_date,
+        skills=skills.copy(),
+    )
+    dff = dff.sort_values(by='enrolls', ascending=False)
+    return dff

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/enterprise_data/api/v1/serializers.py

@@ -8,6 +8,7 @@ from rest_framework import serializers
 from enterprise_data.models import (
     EnterpriseAdminLearnerProgress,
     EnterpriseAdminSummarizeInsights,
+    EnterpriseExecEdLCModulePerformance,
     EnterpriseLearner,
     EnterpriseLearnerEnrollment,
     EnterpriseOffer,
@@ -216,3 +217,13 @@ class AdminAnalyticsAggregatesQueryParamsSerializer(serializers.Serializer):  #
             if attrs['start_date'] > attrs['end_date']:
                 raise serializers.ValidationError("start_date should be less than or equal to end_date.")
         return attrs
+
+
+class EnterpriseExecEdLCModulePerformanceSerializer(serializers.ModelSerializer):
+    """
+    Serializer for EnterpriseExecEdLCModulePerformance model.
+    """
+
+    class Meta:
+        model = EnterpriseExecEdLCModulePerformance
+        fields = '__all__'

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.5.0}/enterprise_data/api/v1/urls.py

@@ -35,6 +35,11 @@ router.register(
     enterprise_learner_views.EnterpriseLearnerCompletedCoursesViewSet,
     'enterprise-learner-completed-courses',
 )
+router.register(
+    r'enterprise/(?P<enterprise_id>.+)/module-performance',
+    enterprise_admin_views.EnterpriseExecEdLCModulePerformanceViewSet,
+    'enterprise-admin-module-performance',
+)
 
 urlpatterns = [
     re_path(
@@ -47,6 +52,11 @@ urlpatterns = [
         enterprise_admin_views.EnterpriseAdminAnalyticsAggregatesView.as_view(),
         name='enterprise-admin-analytics-aggregates'
     ),
+    re_path(
+        fr'^admin/anlaytics/(?P<enterprise_id>{UUID4_REGEX})/skills/stats',
+        enterprise_admin_views.EnterpriseAdminAnalyticsSkillsView.as_view(),
+        name='enterprise-admin-analytics-skills'
+    ),
 ]
 
 urlpatterns += router.urls

edx_enterprise_data-8.5.0/enterprise_data/api/v1/views/enterprise_admin.py

@@ -0,0 +1,261 @@
+"""
+Views for enterprise admin api v1.
+"""
+from datetime import datetime, timedelta
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework import filters, viewsets
+from rest_framework.response import Response
+from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND
+from rest_framework.views import APIView
+
+from django.http import HttpResponse
+
+from enterprise_data.admin_analytics.data_loaders import fetch_max_enrollment_datetime
+from enterprise_data.admin_analytics.utils import (
+    ChartType,
+    fetch_and_cache_engagements_data,
+    fetch_and_cache_enrollments_data,
+    fetch_and_cache_skills_data,
+    get_skills_chart_data,
+    get_top_skills_csv_data,
+)
+from enterprise_data.api.v1 import serializers
+from enterprise_data.models import (
+    EnterpriseAdminLearnerProgress,
+    EnterpriseAdminSummarizeInsights,
+    EnterpriseExecEdLCModulePerformance,
+)
+from enterprise_data.utils import date_filter
+
+from .base import EnterpriseViewSetMixin
+
+
+class EnterpriseAdminInsightsView(APIView):
+    """
+    API for getting the enterprise admin insights.
+    """
+
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ["get"]
+
+    @permission_required(
+        "can_access_enterprise", fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin insights
+        """
+        response_data = {}
+        learner_progress = {}
+        learner_engagement = {}
+
+        try:
+            learner_progress = EnterpriseAdminLearnerProgress.objects.get(
+                enterprise_customer_uuid=enterprise_id
+            )
+            learner_progress = serializers.EnterpriseAdminLearnerProgressSerializer(
+                learner_progress
+            ).data
+            response_data["learner_progress"] = learner_progress
+        except EnterpriseAdminLearnerProgress.DoesNotExist:
+            pass
+
+        try:
+            learner_engagement = EnterpriseAdminSummarizeInsights.objects.get(
+                enterprise_customer_uuid=enterprise_id
+            )
+            learner_engagement = serializers.EnterpriseAdminSummarizeInsightsSerializer(
+                learner_engagement
+            ).data
+            response_data["learner_engagement"] = learner_engagement
+        except EnterpriseAdminSummarizeInsights.DoesNotExist:
+            pass
+
+        status = HTTP_200_OK
+        if learner_progress == {} and learner_engagement == {}:
+            status = HTTP_404_NOT_FOUND
+
+        return Response(data=response_data, status=status)
+
+
+class EnterpriseAdminAnalyticsAggregatesView(APIView):
+    """
+    API for getting the enterprise admin analytics aggregates.
+    """
+
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ["get"]
+
+    @permission_required(
+        "can_access_enterprise", fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin aggregate data.
+        """
+        serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(
+            data=request.GET
+        )
+        serializer.is_valid(raise_exception=True)
+
+        last_updated_at = fetch_max_enrollment_datetime()
+        cache_expiry = (
+            last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+        )
+
+        enrollment = fetch_and_cache_enrollments_data(
+            enterprise_id, cache_expiry
+        ).copy()
+        engagement = fetch_and_cache_engagements_data(
+            enterprise_id, cache_expiry
+        ).copy()
+        # Use start and end date if provided by the client, if client has not provided then use
+        # 1. minimum enrollment date from the data as the start_date
+        # 2. today's date as the end_date
+        start_date = serializer.data.get(
+            'start_date', enrollment.enterprise_enrollment_date.min()
+        )
+        end_date = serializer.data.get('end_date', datetime.now())
+
+        # Date filtering.
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=enrollment.copy(),
+            date_column='enterprise_enrollment_date',
+        )
+
+        enrolls = len(dff)
+        courses = len(dff.course_key.unique())
+
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=enrollment.copy(),
+            date_column='passed_date',
+        )
+
+        completions = dff.has_passed.sum()
+
+        # Date filtering.
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=engagement.copy(),
+            date_column='activity_date',
+        )
+
+        hours = round(dff.learning_time_seconds.sum() / 60 / 60, 1)
+        sessions = dff.is_engaged.sum()
+
+        return Response(
+            data={
+                'enrolls': enrolls,
+                'courses': courses,
+                'completions': completions,
+                'hours': hours,
+                'sessions': sessions,
+                'last_updated_at': last_updated_at.date() if last_updated_at else None,
+                'min_enrollment_date': enrollment.enterprise_enrollment_date.min().date(),
+                'max_enrollment_date': enrollment.enterprise_enrollment_date.max().date(),
+            },
+            status=HTTP_200_OK,
+        )
+
+
+class EnterpriseAdminAnalyticsSkillsView(APIView):
+    """
+    API for getting the enterprise admin analytics skills data.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ['get']
+
+    @permission_required(
+        'can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """HTTP GET endpoint to retrieve the enterprise admin skills aggregated data.
+
+        Args:
+            request (HttpRequest): request object
+            enterprise_id (str): UUID of the enterprise customer
+
+        Returns:
+            response(HttpResponse): response object
+        """
+        serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(
+            data=request.GET
+        )
+        serializer.is_valid(raise_exception=True)
+
+        start_date = serializer.data.get("start_date")
+        end_date = serializer.data.get("end_date", datetime.now())
+
+        last_updated_at = fetch_max_enrollment_datetime()
+        cache_expiry = (
+            last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+        )
+        skills = fetch_and_cache_skills_data(enterprise_id, cache_expiry).copy()
+
+        if request.GET.get("format") == "csv":
+            csv_data = get_top_skills_csv_data(skills, start_date, end_date)
+            response = HttpResponse(content_type='text/csv')
+            filename = f"Skills by Enrollment and Completion, {start_date} - {end_date}.csv"
+            response['Content-Disposition'] = f'attachment; filename="{filename}"'
+            csv_data.to_csv(path_or_buf=response, index=False)
+            return response
+
+        top_skills = get_skills_chart_data(
+            chart_type=ChartType.BUBBLE,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+        top_skills_enrollments = get_skills_chart_data(
+            chart_type=ChartType.TOP_SKILLS_ENROLLMENT,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+        top_skills_by_completions = get_skills_chart_data(
+            chart_type=ChartType.TOP_SKILLS_COMPLETION,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+
+        response_data = {
+            "top_skills": top_skills.to_dict(orient="records"),
+            "top_skills_by_enrollments": top_skills_enrollments.to_dict(
+                orient="records"
+            ),
+            "top_skills_by_completions": top_skills_by_completions.to_dict(
+                orient="records"
+            ),
+        }
+
+        return Response(data=response_data, status=HTTP_200_OK)
+
+
+class EnterpriseExecEdLCModulePerformanceViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet):
+    """
+    View to for getting enterprise exec ed learner module performance records.
+    """
+    serializer_class = serializers.EnterpriseExecEdLCModulePerformanceSerializer
+    filter_backends = (filters.OrderingFilter, filters.SearchFilter)
+    ordering_fields = '__all__'
+    ordering = ('last_access',)
+    search_fields = (
+        'username',
+        'course_name'
+    )
+
+    def get_queryset(self):
+        """
+        Return the queryset of EnterpriseExecEdLCModulePerformance objects.
+        """
+        return EnterpriseExecEdLCModulePerformance.objects.filter(
+            enterprise_customer_uuid=self.kwargs['enterprise_id'],
+        )