edx-enterprise-data 8.3.1__tar.gz → 8.4.0__tar.gz

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0}/CHANGELOG.rst

@@ -15,6 +15,10 @@ Unreleased
 ----------
 
 =========================
+[8.4.0] - 2024-08-09
+---------------------
+  * feat: endpoint to get skills aggregated data for an enterprise customer
+
 [8.3.1] - 2024-08-06
 ---------------------
   * Dependency updates

{edx_enterprise_data-8.3.1/edx_enterprise_data.egg-info → edx_enterprise_data-8.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.3.1
+Version: 8.4.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0/edx_enterprise_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edx-enterprise-data
-Version: 8.3.1
+Version: 8.4.0
 Summary: Enterprise Reporting
 Home-page: https://github.com/openedx/edx-enterprise-data
 Author: edX

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0}/enterprise_data/__init__.py

@@ -2,4 +2,4 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "8.3.1"
+__version__ = "8.4.0"

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0}/enterprise_data/admin_analytics/data_loaders.py

@@ -135,3 +135,48 @@ def fetch_max_enrollment_datetime():
     if not results:
         return None
     return pandas.to_datetime(results[0][0])
+
+
+def fetch_skills_data(enterprise_uuid: str):
+    """
+    Fetch skills data from the database for the given enterprise customer.
+
+    Arguments:
+        enterprise_uuid (str): The UUID of the enterprise customer.
+
+    Returns:
+        (pandas.DataFrame): The skills data.
+    """
+
+    enterprise_uuid = enterprise_uuid.replace('-', '')
+
+    cols = [
+        'course_number',
+        'skill_type',
+        'skill_name',
+        'skill_url',
+        'confidence',
+        'skill_rank',
+        'course_title',
+        'course_key',
+        'level_type',
+        'primary_subject_name',
+        'date',
+        'enterprise_customer_uuid',
+        'enterprise_customer_name',
+        'enrolls',
+        'completions',
+    ]
+    query = get_select_query(
+        table='skills_daily_rollup_admin_dash', columns=cols, enterprise_uuid=enterprise_uuid
+    )
+
+    skills = run_query(query=query)
+
+    if not skills:
+        raise Http404(f'No skills data found for enterprise {enterprise_uuid}')
+
+    skills = pandas.DataFrame(numpy.array(skills), columns=cols)
+    skills['date'] = skills['date'].astype('datetime64[ns]')
+
+    return skills

edx_enterprise_data-8.4.0/enterprise_data/admin_analytics/utils.py

@@ -0,0 +1,257 @@
+"""
+Utility functions for fetching data from the database.
+"""
+from datetime import datetime
+from enum import Enum
+
+from edx_django_utils.cache import TieredCache, get_cache_key
+
+from enterprise_data.admin_analytics.data_loaders import fetch_engagement_data, fetch_enrollment_data, fetch_skills_data
+from enterprise_data.utils import date_filter, primary_subject_truncate
+
+
+class ChartType(Enum):
+    """
+    Chart types.
+    """
+    BUBBLE = 'bubble'
+    TOP_SKILLS_ENROLLMENT = 'top_skills_enrollment'
+    TOP_SKILLS_COMPLETION = 'top_skills_completion'
+
+
+def get_cache_timeout(cache_expiry):
+    """
+    Helper method to calculate cache timeout in seconds.
+
+    Arguments:
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (int): Cache timeout in seconds.
+    """
+    now = datetime.now()
+    cache_timeout = 0
+    if cache_expiry > now:
+        # Calculate cache expiry in seconds from now.
+        cache_timeout = (cache_expiry - now).seconds
+
+    return cache_timeout
+
+
+def fetch_and_cache_enrollments_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache enrollments data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The enrollments data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-enrollments',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        enrollments = fetch_enrollment_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, enrollments, get_cache_timeout(cache_expiry)
+        )
+        return enrollments
+
+
+def fetch_and_cache_engagements_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache engagements data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The engagements data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregates-engagements',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        engagements = fetch_engagement_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, engagements, get_cache_timeout(cache_expiry)
+        )
+        return engagements
+
+
+def fetch_and_cache_skills_data(enterprise_id, cache_expiry):
+    """
+    Helper method to fetch and cache skills data.
+
+    Arguments:
+        enterprise_id (str): UUID of the enterprise customer in string format.
+        cache_expiry (datetime): Datetime object denoting the cache expiry.
+
+    Returns:
+        (pandas.DataFrame): The skills data.
+    """
+    cache_key = get_cache_key(
+        resource='enterprise-admin-analytics-aggregate-skills',
+        enterprise_customer=enterprise_id,
+    )
+    cached_response = TieredCache.get_cached_response(cache_key)
+
+    if cached_response.is_found:
+        return cached_response.value
+    else:
+        skills = fetch_skills_data(enterprise_id)
+        TieredCache.set_all_tiers(
+            cache_key, skills, get_cache_timeout(cache_expiry)
+        )
+        return skills
+
+
+def get_skills_bubble_chart_df(skills_filtered):
+    """ Get the skills data for the bubble chart.
+
+    Args:
+        skills_filtered (list): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The skills data for the bubble chart.
+    """
+
+    # Group by skill_name and skill_type, and aggregate enrolls and completions
+    skills_aggregated = (
+        skills_filtered.groupby(['skill_name', 'skill_type'], as_index=False)
+        .agg(enrolls=('enrolls', 'sum'), completions=('completions', 'sum'))
+    )
+
+    # Convert enrolls and completions to integers
+    skills_aggregated['enrolls'] = skills_aggregated['enrolls'].astype(int)
+    skills_aggregated['completions'] = skills_aggregated['completions'].astype(int)
+
+    # Sort the dataframe by enrolls and completions in descending order
+    skills_aggregated = skills_aggregated.sort_values(by=['enrolls', 'completions'], ascending=False)
+
+    return skills_aggregated
+
+
+def get_top_skills_enrollment(skills_filtered):
+    """ Get the top skills by enrolls.
+
+    Args:
+        skills_filtered (pandas.DataFrame): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The top skills by enrolls data
+    """
+
+    # Get the top 10 skills by enrolls
+    top_skills = (
+        skills_filtered.groupby('skill_name')
+        .enrolls.sum()
+        .sort_values(ascending=False)
+        .head(10)
+        .index
+    )
+
+    # Apply primary_subject_truncate to the primary_subject_name column
+    skills_filtered['primary_subject_name'] = skills_filtered['primary_subject_name'].apply(primary_subject_truncate)
+
+    # Filter data for the top skills and aggregate enrolls by skill_name and primary_subject_name
+    top_skills_enrollment_data = (
+        skills_filtered[skills_filtered.skill_name.isin(top_skills)]
+        .groupby(['skill_name', 'primary_subject_name'], as_index=False)
+        .agg(count=('enrolls', 'sum'))
+    )
+
+    # Sort the dataframe by primary_subject_name
+    top_skills_enrollment_data = top_skills_enrollment_data.sort_values(by="primary_subject_name")
+
+    return top_skills_enrollment_data
+
+
+def get_top_skills_completion(skills_filtered):
+    """ Get the top skills by completions.
+
+    Args:
+        skills_filtered (pandas.DataFrame): The skills data.
+
+    Returns:
+        (pandas.DataFrame): The top skills by completions
+    """
+
+    # Get the top 10 skills by completions
+    top_skills = (
+        skills_filtered.groupby('skill_name')
+        .completions.sum()
+        .sort_values(ascending=False)
+        .head(10)
+        .index
+    )
+
+    # Apply primary_subject_truncate to the primary_subject_name column
+    skills_filtered['primary_subject_name'] = skills_filtered['primary_subject_name'].apply(primary_subject_truncate)
+
+    # Filter data for the top skills and aggregate completions by skill_name and primary_subject_name
+    top_skills_completion_data = (
+        skills_filtered[skills_filtered.skill_name.isin(top_skills)]
+        .groupby(['skill_name', 'primary_subject_name'], as_index=False)
+        .agg(count=('completions', 'sum'))
+    )
+
+    # Sort the dataframe by primary_subject_name
+    top_skills_completion_data = top_skills_completion_data.sort_values(by='primary_subject_name')
+
+    return top_skills_completion_data
+
+
+def get_skills_chart_data(chart_type, start_date, end_date, skills):
+    """
+    Get chart data for skill charts.
+
+    Arguments:
+        chart_type (ChartType): The type of chart to generate.
+        start_date (datetime): The start date for the date filter.
+        end_date (datetime): The end date for the date filter.
+        skills (pandas.DataFrame): The skills data.
+    """
+    skills_filtered = date_filter(start=start_date, end=end_date, data_frame=skills.copy(), date_column='date')
+    if chart_type == ChartType.BUBBLE:
+        return get_skills_bubble_chart_df(skills_filtered=skills_filtered.copy())
+    elif chart_type == ChartType.TOP_SKILLS_ENROLLMENT:
+        return get_top_skills_enrollment(skills_filtered=skills_filtered.copy())
+    elif chart_type == ChartType.TOP_SKILLS_COMPLETION:
+        return get_top_skills_completion(skills_filtered=skills_filtered.copy())
+    else:
+        raise ValueError(f"Invalid chart type: {chart_type}")
+
+
+def get_top_skills_csv_data(skills, start_date, end_date):
+    """ Get the top skills data for CSV download.
+
+    Args:
+        skills (pandas.DataFrame): The skills data.
+        start_date (str): The start date for the date filter.
+        end_date (str): The end date for the date filter.
+
+    Returns:
+        (pandas.DataFrame): The top skills data for CSV download.
+    """
+    dff = get_skills_chart_data(
+        chart_type=ChartType.BUBBLE,
+        start_date=start_date,
+        end_date=end_date,
+        skills=skills.copy(),
+    )
+    dff = dff.sort_values(by='enrolls', ascending=False)
+    return dff

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0}/enterprise_data/api/v1/urls.py

@@ -47,6 +47,11 @@ urlpatterns = [
         enterprise_admin_views.EnterpriseAdminAnalyticsAggregatesView.as_view(),
         name='enterprise-admin-analytics-aggregates'
     ),
+    re_path(
+        fr'^admin/anlaytics/(?P<enterprise_id>{UUID4_REGEX})/skills/stats',
+        enterprise_admin_views.EnterpriseAdminAnalyticsSkillsView.as_view(),
+        name='enterprise-admin-analytics-skills'
+    ),
 ]
 
 urlpatterns += router.urls

edx_enterprise_data-8.4.0/enterprise_data/api/v1/views/enterprise_admin.py

@@ -0,0 +1,233 @@
+"""
+Views for enterprise admin api v1.
+"""
+
+from datetime import datetime, timedelta
+
+from edx_rbac.decorators import permission_required
+from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
+from rest_framework.response import Response
+from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND
+from rest_framework.views import APIView
+
+from django.http import HttpResponse
+
+from enterprise_data.admin_analytics.data_loaders import fetch_max_enrollment_datetime
+from enterprise_data.admin_analytics.utils import (
+    ChartType,
+    fetch_and_cache_engagements_data,
+    fetch_and_cache_enrollments_data,
+    fetch_and_cache_skills_data,
+    get_skills_chart_data,
+    get_top_skills_csv_data,
+)
+from enterprise_data.api.v1 import serializers
+from enterprise_data.models import EnterpriseAdminLearnerProgress, EnterpriseAdminSummarizeInsights
+from enterprise_data.utils import date_filter
+
+
+class EnterpriseAdminInsightsView(APIView):
+    """
+    API for getting the enterprise admin insights.
+    """
+
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ["get"]
+
+    @permission_required(
+        "can_access_enterprise", fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin insights
+        """
+        response_data = {}
+        learner_progress = {}
+        learner_engagement = {}
+
+        try:
+            learner_progress = EnterpriseAdminLearnerProgress.objects.get(
+                enterprise_customer_uuid=enterprise_id
+            )
+            learner_progress = serializers.EnterpriseAdminLearnerProgressSerializer(
+                learner_progress
+            ).data
+            response_data["learner_progress"] = learner_progress
+        except EnterpriseAdminLearnerProgress.DoesNotExist:
+            pass
+
+        try:
+            learner_engagement = EnterpriseAdminSummarizeInsights.objects.get(
+                enterprise_customer_uuid=enterprise_id
+            )
+            learner_engagement = serializers.EnterpriseAdminSummarizeInsightsSerializer(
+                learner_engagement
+            ).data
+            response_data["learner_engagement"] = learner_engagement
+        except EnterpriseAdminSummarizeInsights.DoesNotExist:
+            pass
+
+        status = HTTP_200_OK
+        if learner_progress == {} and learner_engagement == {}:
+            status = HTTP_404_NOT_FOUND
+
+        return Response(data=response_data, status=status)
+
+
+class EnterpriseAdminAnalyticsAggregatesView(APIView):
+    """
+    API for getting the enterprise admin analytics aggregates.
+    """
+
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ["get"]
+
+    @permission_required(
+        "can_access_enterprise", fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """
+        HTTP GET endpoint to retrieve the enterprise admin aggregate data.
+        """
+        serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(
+            data=request.GET
+        )
+        serializer.is_valid(raise_exception=True)
+
+        last_updated_at = fetch_max_enrollment_datetime()
+        cache_expiry = (
+            last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+        )
+
+        enrollment = fetch_and_cache_enrollments_data(
+            enterprise_id, cache_expiry
+        ).copy()
+        engagement = fetch_and_cache_engagements_data(
+            enterprise_id, cache_expiry
+        ).copy()
+        # Use start and end date if provided by the client, if client has not provided then use
+        # 1. minimum enrollment date from the data as the start_date
+        # 2. today's date as the end_date
+        start_date = serializer.data.get(
+            "start_date", enrollment.enterprise_enrollment_date.min()
+        )
+        end_date = serializer.data.get("end_date", datetime.now())
+
+        # Date filtering.
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=enrollment.copy(),
+            date_column="enterprise_enrollment_date",
+        )
+
+        enrolls = len(dff)
+        courses = len(dff.course_key.unique())
+
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=enrollment.copy(),
+            date_column="passed_date",
+        )
+
+        completions = dff.has_passed.sum()
+
+        # Date filtering.
+        dff = date_filter(
+            start=start_date,
+            end=end_date,
+            data_frame=engagement.copy(),
+            date_column="activity_date",
+        )
+
+        hours = round(dff.learning_time_seconds.sum() / 60 / 60, 1)
+        sessions = dff.is_engaged.sum()
+
+        return Response(
+            data={
+                "enrolls": enrolls,
+                "courses": courses,
+                "completions": completions,
+                "hours": hours,
+                "sessions": sessions,
+                "last_updated_at": last_updated_at.date() if last_updated_at else None,
+                "min_enrollment_date": enrollment.enterprise_enrollment_date.min().date(),
+                "max_enrollment_date": enrollment.enterprise_enrollment_date.max().date(),
+            },
+            status=HTTP_200_OK,
+        )
+
+
+class EnterpriseAdminAnalyticsSkillsView(APIView):
+    """
+    API for getting the enterprise admin analytics skills data.
+    """
+    authentication_classes = (JwtAuthentication,)
+    http_method_names = ['get']
+
+    @permission_required(
+        'can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id
+    )
+    def get(self, request, enterprise_id):
+        """HTTP GET endpoint to retrieve the enterprise admin skills aggregated data.
+
+        Args:
+            request (HttpRequest): request object
+            enterprise_id (str): UUID of the enterprise customer
+
+        Returns:
+            response(HttpResponse): response object
+        """
+        serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(
+            data=request.GET
+        )
+        serializer.is_valid(raise_exception=True)
+
+        start_date = serializer.data.get("start_date")
+        end_date = serializer.data.get("end_date", datetime.now())
+
+        last_updated_at = fetch_max_enrollment_datetime()
+        cache_expiry = (
+            last_updated_at + timedelta(days=1) if last_updated_at else datetime.now()
+        )
+        skills = fetch_and_cache_skills_data(enterprise_id, cache_expiry).copy()
+
+        if request.GET.get("format") == "csv":
+            csv_data = get_top_skills_csv_data(skills, start_date, end_date)
+            response = HttpResponse(content_type='text/csv')
+            filename = f"Skills by Enrollment and Completion, {start_date} - {end_date}.csv"
+            response['Content-Disposition'] = f'attachment; filename="{filename}"'
+            csv_data.to_csv(path_or_buf=response, index=False)
+            return response
+
+        top_skills = get_skills_chart_data(
+            chart_type=ChartType.BUBBLE,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+        top_skills_enrollments = get_skills_chart_data(
+            chart_type=ChartType.TOP_SKILLS_ENROLLMENT,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+        top_skills_by_completions = get_skills_chart_data(
+            chart_type=ChartType.TOP_SKILLS_COMPLETION,
+            start_date=start_date,
+            end_date=end_date,
+            skills=skills,
+        )
+
+        response_data = {
+            "top_skills": top_skills.to_dict(orient="records"),
+            "top_skills_by_enrollments": top_skills_enrollments.to_dict(
+                orient="records"
+            ),
+            "top_skills_by_completions": top_skills_by_completions.to_dict(
+                orient="records"
+            ),
+        }
+
+        return Response(data=response_data, status=HTTP_200_OK)

{edx_enterprise_data-8.3.1 → edx_enterprise_data-8.4.0}/enterprise_data/tests/admin_analytics/test_data_loaders.py

@@ -13,8 +13,13 @@ from enterprise_data.admin_analytics.data_loaders import (
     fetch_engagement_data,
     fetch_enrollment_data,
     fetch_max_enrollment_datetime,
+    fetch_skills_data,
+)
+from enterprise_data.tests.test_utils import (
+    get_dummy_engagements_data,
+    get_dummy_enrollments_data,
+    get_dummy_skills_data,
 )
-from enterprise_data.tests.test_utils import get_dummy_engagements_data, get_dummy_enrollments_data
 
 
 class TestDataLoaders(TestCase):
@@ -84,3 +89,27 @@ class TestDataLoaders(TestCase):
             with pytest.raises(Http404) as error:
                 fetch_enrollment_data(enterprise_uuid)
             error.value.message = f'No enrollment data found for enterprise {enterprise_uuid}'
+
+    def test_fetch_skills_data(self):
+        """
+        Validate the fetch_skills_data function.
+        """
+        with patch('enterprise_data.admin_analytics.data_loaders.run_query') as mock_run_query:
+            enterprise_uuid = str(uuid4())
+            mock_run_query.return_value = [
+                list(item.values()) for item in get_dummy_skills_data(enterprise_uuid)
+            ]
+
+            skills_data = fetch_skills_data(enterprise_uuid)
+            self.assertEqual(skills_data.shape, (10, 15))
+
+    def test_fetch_skills_data_empty_data(self):
+        """
+        Validate the fetch_skills_data function behavior when no data is returned from the query.
+        """
+        with patch('enterprise_data.admin_analytics.data_loaders.run_query') as mock_run_query:
+            mock_run_query.return_value = []
+            enterprise_uuid = str(uuid4())
+            with pytest.raises(Http404) as error:
+                fetch_skills_data(enterprise_uuid)
+            error.value.message = f'No skills data found for enterprise {enterprise_uuid}'