openedx-learning 0.27.1__tar.gz → 0.28.0__tar.gz

{openedx_learning-0.27.1/openedx_learning.egg-info → openedx_learning-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openedx-learning
-Version: 0.27.1
+Version: 0.28.0
 Summary: Open edX Learning Core and Tagging.
 Home-page: https://github.com/openedx/openedx-learning
 Author: David Ormsbee
@@ -19,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.11
 License-File: LICENSE.txt
-Requires-Dist: djangorestframework<4.0
-Requires-Dist: tomlkit
 Requires-Dist: celery
+Requires-Dist: rules<4.0
 Requires-Dist: Django
-Requires-Dist: attrs
 Requires-Dist: edx-drf-extensions
-Requires-Dist: rules<4.0
+Requires-Dist: tomlkit
+Requires-Dist: attrs
+Requires-Dist: djangorestframework<4.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{openedx_learning-0.27.1 → openedx_learning-0.28.0}/openedx_learning/__init__.py

@@ -2,4 +2,4 @@
 Open edX Learning ("Learning Core").
 """
 
-__version__ = "0.27.1"
+__version__ = "0.28.0"

{openedx_learning-0.27.1 → openedx_learning-0.28.0}/openedx_learning/apps/authoring/backup_restore/toml.py

@@ -6,6 +6,7 @@ from datetime import datetime
 
 import tomlkit
 
+from openedx_learning.apps.authoring.publishing import api as publishing_api
 from openedx_learning.apps.authoring.publishing.models import PublishableEntity, PublishableEntityVersion
 from openedx_learning.apps.authoring.publishing.models.learning_package import LearningPackage
 
@@ -27,8 +28,8 @@ def toml_learning_package(learning_package: LearningPackage) -> str:
 def toml_publishable_entity(entity: PublishableEntity) -> str:
     """Create a TOML representation of a publishable entity."""
 
-    current_draft_version = getattr(entity, "draft", None)
-    current_published_version = getattr(entity, "published", None)
+    current_draft_version = publishing_api.get_draft_version(entity)
+    current_published_version = publishing_api.get_published_version(entity)
 
     doc = tomlkit.document()
     entity_table = tomlkit.table()
@@ -37,12 +38,12 @@ def toml_publishable_entity(entity: PublishableEntity) -> str:
 
     if current_draft_version:
         draft_table = tomlkit.table()
-        draft_table.add("version_num", current_draft_version.version.version_num)
+        draft_table.add("version_num", current_draft_version.version_num)
         entity_table.add("draft", draft_table)
 
     published_table = tomlkit.table()
     if current_published_version:
-        published_table.add("version_num", current_published_version.version.version_num)
+        published_table.add("version_num", current_published_version.version_num)
     else:
         published_table.add(tomlkit.comment("unpublished: no published_version_num"))
     entity_table.add("published", published_table)

openedx_learning-0.28.0/openedx_learning/apps/authoring/backup_restore/zipper.py

@@ -0,0 +1,242 @@
+"""
+This module provides functionality to create a zip file containing the learning package data,
+including a TOML representation of the learning package and its entities.
+"""
+import hashlib
+import zipfile
+from pathlib import Path
+from typing import List, Optional
+
+from django.db.models import Prefetch, QuerySet
+from django.utils.text import slugify
+
+from openedx_learning.api.authoring_models import (
+    ComponentVersion,
+    ComponentVersionContent,
+    Content,
+    LearningPackage,
+    PublishableEntity,
+    PublishableEntityVersion,
+)
+from openedx_learning.apps.authoring.backup_restore.toml import toml_learning_package, toml_publishable_entity
+from openedx_learning.apps.authoring.publishing import api as publishing_api
+
+TOML_PACKAGE_NAME = "package.toml"
+
+
+def slugify_hashed_filename(identifier: str) -> str:
+    """
+    Generate a filesystem-safe filename from an identifier.
+
+    Why:
+        Identifiers may contain characters that are invalid or ambiguous
+        in filesystems (e.g., slashes, colons, case differences).
+        Additionally, two different identifiers might normalize to the same
+        slug after cleaning. To avoid collisions and ensure uniqueness,
+        we append a short blake2b hash.
+
+    What:
+        - Slugify the identifier (preserves most characters, only strips
+          filesystem-invalid ones).
+        - Append a short hash for uniqueness.
+        - Result: human-readable but still unique and filesystem-safe filename.
+    """
+    slug = slugify(identifier, allow_unicode=True)
+    # Short digest ensures uniqueness without overly long filenames
+    short_hash = hashlib.blake2b(
+        identifier.encode("utf-8"),
+        digest_size=3,
+    ).hexdigest()
+    return f"{slug}_{short_hash}"
+
+
+class LearningPackageZipper:
+    """
+    A class to handle the zipping of learning content for backup and restore.
+    """
+
+    def __init__(self, learning_package: LearningPackage):
+        self.learning_package = learning_package
+        self.folders_already_created: set[Path] = set()
+
+    def create_folder(self, folder_path: Path, zip_file: zipfile.ZipFile) -> None:
+        """
+        Create a folder for the zip file structure.
+        Skips creating the folder if it already exists based on the folder path.
+        Args:
+            folder_path (Path): The path of the folder to create.
+        """
+        if folder_path not in self.folders_already_created:
+            zip_info = zipfile.ZipInfo(str(folder_path) + "/")
+            zip_file.writestr(zip_info, "")  # Add explicit empty directory entry
+            self.folders_already_created.add(folder_path)
+
+    def get_publishable_entities(self) -> QuerySet[PublishableEntity]:
+        """
+        Retrieve the publishable entities associated with the learning package.
+        Prefetches related data for efficiency.
+        """
+        lp_id = self.learning_package.pk
+        publishable_entities: QuerySet[PublishableEntity] = publishing_api.get_publishable_entities(lp_id)
+        return (
+            publishable_entities
+            .select_related(
+                "container",
+                "component__component_type",
+                "draft__version__componentversion",
+                "published__version__componentversion",
+            )
+            .prefetch_related(
+                # We should re-evaluate the prefetching strategy here,
+                # as the current approach may cause performance issues—
+                # especially with large libraries (up to 100K items),
+                # which is too large for this type of prefetch.
+                Prefetch(
+                    "draft__version__componentversion__componentversioncontent_set",
+                    queryset=ComponentVersionContent.objects.select_related("content"),
+                    to_attr="prefetched_contents",
+                ),
+                Prefetch(
+                    "published__version__componentversion__componentversioncontent_set",
+                    queryset=ComponentVersionContent.objects.select_related("content"),
+                    to_attr="prefetched_contents",
+                ),
+            )
+        )
+
+    def get_versions_to_write(self, entity: PublishableEntity):
+        """
+        Get the versions of a publishable entity that should be written to the zip file.
+        It retrieves both draft and published versions.
+        """
+        draft_version: Optional[PublishableEntityVersion] = publishing_api.get_draft_version(entity)
+        published_version: Optional[PublishableEntityVersion] = publishing_api.get_published_version(entity)
+
+        versions_to_write = [draft_version] if draft_version else []
+
+        if published_version and published_version != draft_version:
+            versions_to_write.append(published_version)
+        return versions_to_write
+
+    def create_zip(self, path: str) -> None:
+        """
+        Creates a zip file containing the learning package data.
+        Args:
+            path (str): The path where the zip file will be created.
+        Raises:
+            Exception: If the learning package cannot be found or if the zip creation fails.
+        """
+
+        with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
+            # Add the package.toml file
+            package_toml_content: str = toml_learning_package(self.learning_package)
+            zipf.writestr(TOML_PACKAGE_NAME, package_toml_content)
+
+            # Add the entities directory
+            entities_folder = Path("entities")
+            self.create_folder(entities_folder, zipf)
+
+            # Add the collections directory
+            collections_folder = Path("collections")
+            self.create_folder(collections_folder, zipf)
+
+            # ------ ENTITIES SERIALIZATION -------------
+
+            # get the publishable entities
+            publishable_entities: QuerySet[PublishableEntity] = self.get_publishable_entities()
+
+            for entity in publishable_entities:
+                # entity: PublishableEntity = entity  # Type hint for clarity
+
+                # Create a TOML representation of the entity
+                entity_toml_content: str = toml_publishable_entity(entity)
+
+                if hasattr(entity, 'container'):
+                    entity_slugify_hash = slugify_hashed_filename(entity.key)
+                    entity_toml_filename = f"{entity_slugify_hash}.toml"
+                    entity_toml_path = entities_folder / entity_toml_filename
+                    zipf.writestr(str(entity_toml_path), entity_toml_content)
+
+                if hasattr(entity, 'component'):
+                    # Create the component folder structure for the entity. The structure is as follows:
+                    # entities/
+                    #     xblock.v1/  (component namespace)
+                    #         html/  (component type)
+                    #             my_component.toml  (entity TOML file)
+                    #             my_component/  (component id)
+                    #                 component_versions/
+                    #                     v1/
+                    #                         static/
+
+                    # Generate the slugified hash for the component local key
+                    # Example: if the local key is "my_component", the slugified hash might be "my_component_123456"
+                    # It's a combination of the local key and a hash and should be unique
+                    entity_slugify_hash = slugify_hashed_filename(entity.component.local_key)
+
+                    # Create the component namespace folder
+                    # Example of component namespace is: "entities/xblock.v1/"
+                    component_namespace_folder = entities_folder / entity.component.component_type.namespace
+                    self.create_folder(component_namespace_folder, zipf)
+
+                    # Create the component type folder
+                    # Example of component type is: "entities/xblock.v1/html/"
+                    component_type_folder = component_namespace_folder / entity.component.component_type.name
+                    self.create_folder(component_type_folder, zipf)
+
+                    # Create the component id folder
+                    # Example of component id is: "entities/xblock.v1/html/my_component_123456/"
+                    component_id_folder = component_type_folder / entity_slugify_hash
+                    self.create_folder(component_id_folder, zipf)
+
+                    # Add the entity TOML file inside the component type folder as well
+                    # Example: "entities/xblock.v1/html/my_component_123456.toml"
+                    component_entity_toml_path = component_type_folder / f"{entity_slugify_hash}.toml"
+                    zipf.writestr(str(component_entity_toml_path), entity_toml_content)
+
+                    # Add component version folder into the component id folder
+                    # Example: "entities/xblock.v1/html/my_component_123456/component_versions/"
+                    component_version_folder = component_id_folder / "component_versions"
+                    self.create_folder(component_version_folder, zipf)
+
+                    # ------ COMPONENT VERSIONING -------------
+                    # Focusing on draft and published versions
+
+                    # Get the draft and published versions
+                    versions_to_write: List[PublishableEntityVersion] = self.get_versions_to_write(entity)
+
+                    for version in versions_to_write:
+                        # Create a folder for the version
+                        version_number = f"v{version.version_num}"
+                        version_folder = component_version_folder / version_number
+                        self.create_folder(version_folder, zipf)
+
+                        # Add static folder for the version
+                        static_folder = version_folder / "static"
+                        self.create_folder(static_folder, zipf)
+
+                        # ------ COMPONENT STATIC CONTENT -------------
+                        component_version: ComponentVersion = version.componentversion
+
+                        # Get content data associated with this version
+                        contents: QuerySet[
+                            ComponentVersionContent
+                        ] = component_version.prefetched_contents  # type: ignore[attr-defined]
+
+                        for component_version_content in contents:
+                            content: Content = component_version_content.content
+
+                            # Important: The component_version_content.key contains implicitly
+                            # the file name and the file extension
+                            file_path = version_folder / component_version_content.key
+
+                            if content.has_file and content.path:
+                                # If has_file, we pull it from the file system
+                                with content.read_file() as f:
+                                    file_data = f.read()
+                            elif not content.has_file and content.text:
+                                # Otherwise, we use the text content as the file data
+                                file_data = content.text
+                            else:
+                                # If no file and no text, we skip this content
+                                continue
+                            zipf.writestr(str(file_path), file_data)

{openedx_learning-0.27.1 → openedx_learning-0.28.0}/openedx_learning/apps/authoring/publishing/api.py

@@ -10,7 +10,7 @@ from contextlib import nullcontext
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from enum import Enum
-from typing import ContextManager, TypeVar
+from typing import ContextManager, Optional, TypeVar
 
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db.models import F, Q, QuerySet
@@ -58,9 +58,9 @@ __all__ = [
     "create_publishable_entity_version",
     "get_publishable_entity",
     "get_publishable_entity_by_key",
+    "get_publishable_entities",
     "get_last_publish",
     "get_all_drafts",
-    "get_entities",
     "get_entities_with_unpublished_changes",
     "get_entities_with_unpublished_deletes",
     "publish_all_drafts",
@@ -262,11 +262,18 @@ def get_all_drafts(learning_package_id: int, /) -> QuerySet[Draft]:
     )
 
 
-def get_entities(learning_package_id: int, /) -> QuerySet[PublishableEntity]:
+def get_publishable_entities(learning_package_id: int, /) -> QuerySet[PublishableEntity]:
     """
     Get all entities in a learning package.
     """
-    return PublishableEntity.objects.filter(learning_package_id=learning_package_id)
+    return (
+        PublishableEntity.objects
+        .filter(learning_package_id=learning_package_id)
+        .select_related(
+            "draft__version",
+            "published__version",
+        )
+    )
 
 
 def get_entities_with_unpublished_changes(
@@ -425,15 +432,22 @@ def publish_from_drafts(
     return publish_log
 
 
-def get_draft_version(publishable_entity_id: int, /) -> PublishableEntityVersion | None:
+def get_draft_version(publishable_entity_or_id: PublishableEntity | int, /) -> PublishableEntityVersion | None:
     """
     Return current draft PublishableEntityVersion for this PublishableEntity.
 
     This function will return None if there is no current draft.
     """
+    if isinstance(publishable_entity_or_id, PublishableEntity):
+        # Fetches the draft version for a given PublishableEntity.
+        # Gracefully handles cases where no draft is present.
+        draft: Optional[Draft] = getattr(publishable_entity_or_id, "draft", None)
+        if draft is None:
+            return None
+        return draft.version
     try:
         draft = Draft.objects.select_related("version").get(
-            entity_id=publishable_entity_id
+            entity_id=publishable_entity_or_id
         )
     except Draft.DoesNotExist:
         # No draft was ever created.
@@ -445,15 +459,22 @@ def get_draft_version(publishable_entity_id: int, /) -> PublishableEntityVersion
     return draft.version
 
 
-def get_published_version(publishable_entity_id: int, /) -> PublishableEntityVersion | None:
+def get_published_version(publishable_entity_or_id: PublishableEntity | int, /) -> PublishableEntityVersion | None:
     """
     Return current published PublishableEntityVersion for this PublishableEntity.
 
     This function will return None if there is no current published version.
     """
+    if isinstance(publishable_entity_or_id, PublishableEntity):
+        # Fetches the published version for a given PublishableEntity.
+        # Gracefully handles cases where no published version is present.
+        published: Optional[Published] = getattr(publishable_entity_or_id, "published", None)
+        if published is None:
+            return None
+        return published.version
     try:
         published = Published.objects.select_related("version").get(
-            entity_id=publishable_entity_id
+            entity_id=publishable_entity_or_id
         )
     except Published.DoesNotExist:
         return None

{openedx_learning-0.27.1 → openedx_learning-0.28.0/openedx_learning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openedx-learning
-Version: 0.27.1
+Version: 0.28.0
 Summary: Open edX Learning Core and Tagging.
 Home-page: https://github.com/openedx/openedx-learning
 Author: David Ormsbee
@@ -19,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.11
 License-File: LICENSE.txt
-Requires-Dist: djangorestframework<4.0
-Requires-Dist: tomlkit
 Requires-Dist: celery
+Requires-Dist: rules<4.0
 Requires-Dist: Django
-Requires-Dist: attrs
 Requires-Dist: edx-drf-extensions
-Requires-Dist: rules<4.0
+Requires-Dist: tomlkit
+Requires-Dist: attrs
+Requires-Dist: djangorestframework<4.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

openedx_learning-0.28.0/openedx_tagging/core/tagging/models/utils.py

@@ -0,0 +1,82 @@
+"""
+Utilities for tagging and taxonomy models
+"""
+from django.db import connection as db_connection
+from django.db.models import Aggregate, CharField, TextField
+from django.db.models.expressions import Combinable, Func
+
+RESERVED_TAG_CHARS = [
+    '\t',  # Used in the database to separate tag levels in the "lineage" field
+           # e.g. lineage="Earth\tNorth America\tMexico\tMexico City"
+    ' > ',  # Used in the search index and Instantsearch frontend to separate tag levels
+            # e.g. tags_level3="Earth > North America > Mexico > Mexico City"
+    ';',   # Used in CSV exports to separate multiple tags from the same taxonomy
+           # e.g. languages-v1: en;es;fr
+]
+TAGS_CSV_SEPARATOR = RESERVED_TAG_CHARS[2]
+
+
+class ConcatNull(Func):  # pylint: disable=abstract-method
+    """
+    Concatenate two arguments together. Like normal SQL but unlike Django's
+    "Concat", if either argument is NULL, the result will be NULL.
+    """
+
+    function = "CONCAT"
+
+    def as_sqlite(self, compiler, connection, **extra_context):
+        """ SQLite doesn't have CONCAT() but has a concatenation operator """
+        return super().as_sql(
+            compiler,
+            connection,
+            template="%(expressions)s",
+            arg_joiner=" || ",
+            **extra_context,
+        )
+
+
+class StringAgg(Aggregate, Combinable):
+    """
+    Aggregate function that collects the values of some column across all rows,
+    and creates a string by concatenating those values, with a specified separator.
+
+    This version supports PostgreSQL (STRING_AGG), MySQL (GROUP_CONCAT), and SQLite.
+    """
+    # Default function is for MySQL (GROUP_CONCAT)
+    function = 'GROUP_CONCAT'
+    template = '%(function)s(%(distinct)s%(expressions)s)'
+
+    def __init__(self, expression, distinct=False, delimiter=',', **extra):
+        self.delimiter = delimiter
+        # Handle the distinct option and output type
+        distinct_str = 'DISTINCT ' if distinct else ''
+
+        extra.update({
+            'distinct': distinct_str,
+            'output_field': CharField(),
+        })
+
+        # Check the database backend (PostgreSQL, MySQL, or SQLite)
+        if 'postgresql' in db_connection.vendor.lower():
+            self.function = 'STRING_AGG'
+            self.template = '%(function)s(%(distinct)s%(expressions)s, %(delimiter)s)'
+            extra.update({
+                "delimiter": self.delimiter,
+                "output_field": TextField(),
+            })
+
+        # Initialize the parent class with the necessary parameters
+        super().__init__(
+            expression,
+            **extra,
+        )
+
+    # Implementing abstract methods from Combinable
+    def __rand__(self, other):
+        return self._combine(other, 'AND', False)
+
+    def __ror__(self, other):
+        return self._combine(other, 'OR', False)
+
+    def __rxor__(self, other):
+        return self._combine(other, 'XOR', False)

openedx_learning-0.27.1/openedx_learning/apps/authoring/backup_restore/zipper.py

@@ -1,53 +0,0 @@
-"""
-This module provides functionality to create a zip file containing the learning package data,
-including a TOML representation of the learning package and its entities.
-"""
-import zipfile
-from pathlib import Path
-
-from openedx_learning.apps.authoring.backup_restore.toml import toml_learning_package, toml_publishable_entity
-from openedx_learning.apps.authoring.publishing import api as publishing_api
-from openedx_learning.apps.authoring.publishing.models.learning_package import LearningPackage
-
-TOML_PACKAGE_NAME = "package.toml"
-
-
-class LearningPackageZipper:
-    """
-    A class to handle the zipping of learning content for backup and restore.
-    """
-
-    def __init__(self, learning_package: LearningPackage):
-        self.learning_package = learning_package
-
-    def create_zip(self, path: str) -> None:
-        """
-        Creates a zip file containing the learning package data.
-        Args:
-            path (str): The path where the zip file will be created.
-        Raises:
-            Exception: If the learning package cannot be found or if the zip creation fails.
-        """
-        package_toml_content: str = toml_learning_package(self.learning_package)
-
-        with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
-            # Add the package.toml string
-            zipf.writestr(TOML_PACKAGE_NAME, package_toml_content)
-
-            # Add the entities directory
-            entities_folder = Path("entities")
-            zip_info = zipfile.ZipInfo(str(entities_folder) + "/")  # Ensure trailing slash
-            zipf.writestr(zip_info, "")  # Add explicit empty directory entry
-
-            # Add the collections directory
-            collections_folder = Path("collections")
-            collections_info = zipfile.ZipInfo(str(collections_folder) + "/")  # Ensure trailing slash
-            zipf.writestr(collections_info, "")  # Add explicit empty directory
-
-            # Add each entity's TOML file
-            for entity in publishing_api.get_entities(self.learning_package.pk):
-                # Create a TOML representation of the entity
-                entity_toml_content: str = toml_publishable_entity(entity)
-                entity_toml_filename = f"{entity.key}.toml"
-                entity_toml_path = entities_folder / entity_toml_filename
-                zipf.writestr(str(entity_toml_path), entity_toml_content)

openedx_learning-0.27.1/openedx_tagging/core/tagging/models/utils.py

@@ -1,54 +0,0 @@
-"""
-Utilities for tagging and taxonomy models
-"""
-from django.db.models import Aggregate, CharField
-from django.db.models.expressions import Func
-
-RESERVED_TAG_CHARS = [
-    '\t',  # Used in the database to separate tag levels in the "lineage" field
-           # e.g. lineage="Earth\tNorth America\tMexico\tMexico City"
-    ' > ',  # Used in the search index and Instantsearch frontend to separate tag levels
-            # e.g. tags_level3="Earth > North America > Mexico > Mexico City"
-    ';',   # Used in CSV exports to separate multiple tags from the same taxonomy
-           # e.g. languages-v1: en;es;fr
-]
-TAGS_CSV_SEPARATOR = RESERVED_TAG_CHARS[2]
-
-
-class ConcatNull(Func):  # pylint: disable=abstract-method
-    """
-    Concatenate two arguments together. Like normal SQL but unlike Django's
-    "Concat", if either argument is NULL, the result will be NULL.
-    """
-
-    function = "CONCAT"
-
-    def as_sqlite(self, compiler, connection, **extra_context):
-        """ SQLite doesn't have CONCAT() but has a concatenation operator """
-        return super().as_sql(
-            compiler,
-            connection,
-            template="%(expressions)s",
-            arg_joiner=" || ",
-            **extra_context,
-        )
-
-
-class StringAgg(Aggregate):  # pylint: disable=abstract-method
-    """
-    Aggregate function that collects the values of some column across all rows,
-    and creates a string by concatenating those values, with "," as a separator.
-
-    This is the same as Django's django.contrib.postgres.aggregates.StringAgg,
-    but this version works with MySQL and SQLite.
-    """
-    function = 'GROUP_CONCAT'
-    template = '%(function)s(%(distinct)s%(expressions)s)'
-
-    def __init__(self, expression, distinct=False, **extra):
-        super().__init__(
-            expression,
-            distinct='DISTINCT ' if distinct else '',
-            output_field=CharField(),
-            **extra,
-        )