ol-openedx-git-auto-export 0.8.3__tar.gz → 0.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ol_openedx_git_auto_export-0.9.1/CHANGELOG.rst +67 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/PKG-INFO +10 -4
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/README.rst +6 -0
- ol_openedx_git_auto_export-0.9.1/ol_openedx_git_auto_export/constants.py +44 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/tasks.py +64 -12
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/utils.py +174 -48
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/pyproject.toml +3 -3
- ol_openedx_git_auto_export-0.8.3/CHANGELOG.rst +0 -12
- ol_openedx_git_auto_export-0.8.3/ol_openedx_git_auto_export/constants.py +0 -39
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/.gitignore +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/LICENSE +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/admin.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/app.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/exceptions.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/management/commands/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/migrations/0001_initial.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/migrations/0002_contentgitrepository.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/models.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/settings/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/settings/common.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/settings/production.py +0 -0
- {ol_openedx_git_auto_export-0.8.3 → ol_openedx_git_auto_export-0.9.1}/ol_openedx_git_auto_export/signals.py +0 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Change Log
|
|
2
|
+
==========
|
|
3
|
+
|
|
4
|
+
Version 0.9.1 (2026-09-22)
|
|
5
|
+
---------------------------
|
|
6
|
+
|
|
7
|
+
* Fixed ``export_library_to_git`` queuing a git export for every
|
|
8
|
+
``LIBRARY_BLOCK_PUBLISHED``/``LIBRARY_CONTAINER_PUBLISHED`` signal, flooding
|
|
9
|
+
Celery when a course is imported into a v2 library. A burst now queues one
|
|
10
|
+
task, which exports once the signals stop. The course path, which exported a
|
|
11
|
+
mid-burst snapshot 5 seconds after the first signal, uses the same mechanism.
|
|
12
|
+
* Export tasks are queued from a ``transaction.on_commit`` hook with
|
|
13
|
+
``robust=True``. Studio publishes under ``ATOMIC_REQUESTS``, so a task queued
|
|
14
|
+
mid-transaction could wake before the content was visible and give up
|
|
15
|
+
without retrying.
|
|
16
|
+
* ``async_create_github_repo``'s post-creation export now goes through the
|
|
17
|
+
debounce instead of running inline, so it coalesces with an import burst
|
|
18
|
+
rather than racing it on the same clone directory. As a result
|
|
19
|
+
``migrate_giturl`` reports a repository as done once created, not once
|
|
20
|
+
exported.
|
|
21
|
+
* Fixed ``get_or_create_git_export_repo_dir`` racing itself when
|
|
22
|
+
``GIT_REPO_EXPORT_DIR`` does not exist: two processes both passed the
|
|
23
|
+
existence check and the loser's ``FileExistsError`` dropped its export.
|
|
24
|
+
* The debounce token's cache key is versioned (``git_export_debounce_v2``) so a
|
|
25
|
+
rollback doesn't see its own short-lived key as claimed by the new token. The
|
|
26
|
+
token is bounded at 7 days rather than permanent.
|
|
27
|
+
* The debounce fails open: a dead cache backend queues the export undebounced
|
|
28
|
+
rather than raising into the publish request, and a task that fails to
|
|
29
|
+
enqueue releases its marker so the next signal can retry.
|
|
30
|
+
* The publisher lookup runs once per burst instead of once per signal, so a
|
|
31
|
+
large library import no longer issues a ``get_library()`` query per block
|
|
32
|
+
just to discard it.
|
|
33
|
+
* Fixed a ``ContentLibraryNotFound`` propagating into the publish request when
|
|
34
|
+
the library row is not visible yet; the export is queued without an author.
|
|
35
|
+
* Corrected operator-facing log messages that misreported what happened: a
|
|
36
|
+
parse-failure label on a lookup failure, a retry promise where nothing
|
|
37
|
+
re-queues, a repository-creation claim emitted before the check that skips
|
|
38
|
+
it, and per-signal "starting export" lines during a burst.
|
|
39
|
+
* Documented that the debounce needs a cache backend shared between the CMS
|
|
40
|
+
processes and the Celery workers; under ``LocMemCache`` it is inert.
|
|
41
|
+
* Added tests for the export debounce logic.
|
|
42
|
+
|
|
43
|
+
Version 0.9.0 (2026-09-09)
|
|
44
|
+
---------------------------
|
|
45
|
+
|
|
46
|
+
* Raised the Django floor from ``>=4.0`` to ``>=5.2``, making this the minimum
|
|
47
|
+
version for the Ulmo and Verawood releases. See the Open edX Release
|
|
48
|
+
Compatibility table in ``docs/README.rst``.
|
|
49
|
+
* The test suite now runs under ``cms.envs.test``; the plugin registers only a
|
|
50
|
+
``cms.djangoapp`` entry point, so under LMS settings its app is not installed
|
|
51
|
+
and its models and tasks fail to import.
|
|
52
|
+
|
|
53
|
+
Version 0.8.3 (2026-08-03)
|
|
54
|
+
---------------------------
|
|
55
|
+
|
|
56
|
+
* Fixed a race where an export task could run before the library was committed
|
|
57
|
+
to the store, reporting a spurious "library not found".
|
|
58
|
+
|
|
59
|
+
Version 0.8.2 (2026-06-10)
|
|
60
|
+
---------------------------
|
|
61
|
+
|
|
62
|
+
* Fixed ``migrate_giturl`` management command passing a stale ``export_course``
|
|
63
|
+
keyword to ``async_create_github_repo``, which raised ``TypeError`` during
|
|
64
|
+
parallel repository creation.
|
|
65
|
+
* Fixed ``migrate_giturl`` reporting a possibly-unbound loop variable in its
|
|
66
|
+
repository-creation progress message; it now reports the count of courses.
|
|
67
|
+
* Added tests for the ``migrate_giturl`` command.
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: ol-openedx-git-auto-export
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: A plugin that auto saves the course OLX to git when an author publishes it
|
|
5
5
|
Author: MIT Office of Digital Learning
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
7
7
|
License-File: LICENSE
|
|
8
8
|
Requires-Python: >=3.11
|
|
9
9
|
Requires-Dist: celery>=4.4.7
|
|
10
|
-
Requires-Dist: django>=
|
|
11
|
-
Requires-Dist: edx-django-utils
|
|
10
|
+
Requires-Dist: django>=5.2
|
|
11
|
+
Requires-Dist: edx-django-utils>=8.0.0
|
|
12
12
|
Requires-Dist: edx-opaque-keys
|
|
13
13
|
Requires-Dist: gitpython>=3.1.37
|
|
14
14
|
Requires-Dist: openedx-events
|
|
@@ -17,6 +17,12 @@ Description-Content-Type: text/x-rst
|
|
|
17
17
|
Git Auto Export Plugin
|
|
18
18
|
######################
|
|
19
19
|
|
|
20
|
+
Version Compatibility
|
|
21
|
+
======================
|
|
22
|
+
|
|
23
|
+
For this plugin's compatibility with Open edX, see the
|
|
24
|
+
`Open edX Release Compatibility table <../../docs#open-edx-release-compatibility>`_.
|
|
25
|
+
|
|
20
26
|
Installation
|
|
21
27
|
============
|
|
22
28
|
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
Git Auto Export Plugin
|
|
2
2
|
######################
|
|
3
3
|
|
|
4
|
+
Version Compatibility
|
|
5
|
+
======================
|
|
6
|
+
|
|
7
|
+
For this plugin's compatibility with Open edX, see the
|
|
8
|
+
`Open edX Release Compatibility table <../../docs#open-edx-release-compatibility>`_.
|
|
9
|
+
|
|
4
10
|
Installation
|
|
5
11
|
============
|
|
6
12
|
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ContentType(StrEnum):
|
|
5
|
+
"""Enumeration for content types (Course or Library)."""
|
|
6
|
+
|
|
7
|
+
COURSE = "course"
|
|
8
|
+
LIBRARY = "library"
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def display_name(self):
|
|
12
|
+
"""Return the human-readable display name."""
|
|
13
|
+
return self.value.capitalize()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# Library key prefixes for different versions
|
|
17
|
+
LIBRARY_V1_PREFIX = "library-v1:"
|
|
18
|
+
LIBRARY_V2_PREFIX = "lib:"
|
|
19
|
+
|
|
20
|
+
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
21
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
22
|
+
GITHUB_ORG = "GITHUB_ORG"
|
|
23
|
+
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
24
|
+
|
|
25
|
+
# Library-specific feature flags
|
|
26
|
+
ENABLE_GIT_AUTO_LIBRARY_EXPORT = "ENABLE_GIT_AUTO_LIBRARY_EXPORT"
|
|
27
|
+
ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION = "ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION"
|
|
28
|
+
|
|
29
|
+
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
30
|
+
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
|
31
|
+
|
|
32
|
+
# A publish burst fires one signal per block. The pending key keeps one export
|
|
33
|
+
# task queued per burst; the debounce key holds a token every signal overwrites.
|
|
34
|
+
# A task exports only while its token is current, else it re-queues -- so the
|
|
35
|
+
# export reflects the end of the burst. A missing token counts as current.
|
|
36
|
+
EXPORT_DEBOUNCE_DELAY = 5 # seconds of quiet before the export happens
|
|
37
|
+
# v2: reusing the unversioned key would make a rollback see it as already
|
|
38
|
+
# claimed by the new long-lived token, and silently stop debouncing.
|
|
39
|
+
EXPORT_DEBOUNCE_CACHE_KEY = "git_export_debounce_v2:{content_key}"
|
|
40
|
+
EXPORT_DEBOUNCE_PENDING_CACHE_KEY = "git_export_pending:{content_key}"
|
|
41
|
+
# Must outlive the countdown plus broker/worker pickup, or a duplicate queues.
|
|
42
|
+
EXPORT_DEBOUNCE_PENDING_TTL = EXPORT_DEBOUNCE_DELAY + 55 # seconds
|
|
43
|
+
# Bounded so dormant content doesn't sit in the cache forever.
|
|
44
|
+
EXPORT_DEBOUNCE_TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
|
|
@@ -17,36 +17,78 @@ from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
|
|
|
17
17
|
from ol_openedx_git_auto_export.models import ContentGitRepository
|
|
18
18
|
from ol_openedx_git_auto_export.utils import (
|
|
19
19
|
clear_stale_git_lock,
|
|
20
|
+
current_export_token,
|
|
20
21
|
get_content_info,
|
|
21
22
|
github_repo_name_format,
|
|
22
23
|
is_auto_repo_creation_enabled,
|
|
24
|
+
queue_export_task,
|
|
25
|
+
release_export_slot,
|
|
26
|
+
schedule_export_with_debounce,
|
|
23
27
|
)
|
|
24
28
|
|
|
25
29
|
LOGGER = get_task_logger(__name__)
|
|
26
30
|
|
|
27
31
|
|
|
32
|
+
def _superseded(context_key_string, user, token):
|
|
33
|
+
"""
|
|
34
|
+
Report whether a newer signal replaced this task, re-queuing if so.
|
|
35
|
+
"""
|
|
36
|
+
current_token = current_export_token(context_key_string, token)
|
|
37
|
+
if current_token == token:
|
|
38
|
+
# Release before exporting, so signals landing during the export
|
|
39
|
+
# queue a fresh task instead of being dropped.
|
|
40
|
+
release_export_slot(context_key_string)
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
LOGGER.info(
|
|
44
|
+
"Newer signals arrived for %s; re-queuing rather than exporting "
|
|
45
|
+
"a mid-burst snapshot",
|
|
46
|
+
context_key_string,
|
|
47
|
+
)
|
|
48
|
+
try:
|
|
49
|
+
# Still holding the slot, so no signal can slip a second task in. A
|
|
50
|
+
# burst extended by another publisher keeps the first as author.
|
|
51
|
+
queue_export_task(context_key_string, user, current_token)
|
|
52
|
+
except Exception:
|
|
53
|
+
# queue_export_task released the slot on its way out.
|
|
54
|
+
LOGGER.exception(
|
|
55
|
+
"Failed to re-queue %s; exporting current state instead",
|
|
56
|
+
context_key_string,
|
|
57
|
+
)
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
|
|
28
63
|
@shared_task
|
|
29
|
-
def async_export_to_git(context_key_string, user=None):
|
|
64
|
+
def async_export_to_git(context_key_string, user=None, token=None):
|
|
30
65
|
"""Export a course or library to Git.
|
|
31
66
|
|
|
32
67
|
Args:
|
|
33
68
|
context_key_string (str): String representation of LearningContextKey
|
|
34
69
|
user: Optional user for git export
|
|
70
|
+
token: Debounce token from utils.queue_export_task; see
|
|
71
|
+
EXPORT_DEBOUNCE_CACHE_KEY in constants.py for the mechanism.
|
|
72
|
+
None skips the staleness check and the debounce state, and
|
|
73
|
+
exists only for messages queued by releases predating the token.
|
|
35
74
|
"""
|
|
75
|
+
if token and _superseded(context_key_string, user, token):
|
|
76
|
+
return
|
|
77
|
+
|
|
36
78
|
try:
|
|
37
79
|
context_key = LearningContextKey.from_string(context_key_string)
|
|
38
80
|
content_info = get_content_info(context_key)
|
|
39
81
|
except ContentNotFoundError:
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
# commit).
|
|
82
|
+
# Queued from on_commit, so not the old pre-commit race: the content
|
|
83
|
+
# was most likely deleted during the countdown. Nothing re-queues.
|
|
43
84
|
LOGGER.warning(
|
|
44
|
-
"Content %s not found
|
|
85
|
+
"Content %s not found; abandoning this export (no retry)",
|
|
45
86
|
context_key_string,
|
|
46
87
|
)
|
|
47
88
|
return
|
|
48
89
|
except Exception:
|
|
49
|
-
|
|
90
|
+
# Also covers get_content_info's lookups, so not necessarily a bad key.
|
|
91
|
+
LOGGER.exception("Failed to resolve content %s", context_key_string)
|
|
50
92
|
return
|
|
51
93
|
|
|
52
94
|
try:
|
|
@@ -82,14 +124,22 @@ def async_export_to_git(context_key_string, user=None):
|
|
|
82
124
|
else context_key,
|
|
83
125
|
)
|
|
84
126
|
except ContentGitRepository.DoesNotExist:
|
|
85
|
-
LOGGER.
|
|
86
|
-
"
|
|
87
|
-
"
|
|
127
|
+
LOGGER.info(
|
|
128
|
+
"No git repository registered for %s %s; "
|
|
129
|
+
"creating one if auto-creation is enabled.",
|
|
88
130
|
content_info["content_type"],
|
|
89
131
|
context_key_string,
|
|
90
132
|
)
|
|
91
|
-
|
|
92
|
-
|
|
133
|
+
try:
|
|
134
|
+
if is_auto_repo_creation_enabled(is_library=content_info["is_library"]):
|
|
135
|
+
async_create_github_repo.delay(str(context_key), export_content=True)
|
|
136
|
+
except Exception:
|
|
137
|
+
# Sibling except clauses don't catch this one.
|
|
138
|
+
LOGGER.exception(
|
|
139
|
+
"Failed to check/trigger repo creation for %s %s",
|
|
140
|
+
content_info["content_type"],
|
|
141
|
+
context_key_string,
|
|
142
|
+
)
|
|
93
143
|
except Exception:
|
|
94
144
|
LOGGER.exception(
|
|
95
145
|
"Unknown error occurred during async %s content export to git (%s id: %s)",
|
|
@@ -213,6 +263,8 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
213
263
|
LOGGER.error(response_msg)
|
|
214
264
|
|
|
215
265
|
if ssh_url and export_content:
|
|
216
|
-
|
|
266
|
+
# Debounced like every other export, so it coalesces with an import
|
|
267
|
+
# burst instead of racing it on the same clone directory. No author.
|
|
268
|
+
schedule_export_with_debounce(context_key, lambda: None)
|
|
217
269
|
|
|
218
270
|
return True, response_msg or ssh_url
|
|
@@ -3,14 +3,15 @@ Utility functions for the ol_openedx_git_auto_export app.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
|
-
import os
|
|
7
6
|
import re
|
|
7
|
+
import uuid
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
|
|
10
10
|
from django.conf import settings
|
|
11
11
|
from django.contrib.auth.models import User
|
|
12
12
|
from django.core.cache import cache
|
|
13
13
|
from django.core.exceptions import ImproperlyConfigured
|
|
14
|
+
from django.db import transaction
|
|
14
15
|
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
|
|
15
16
|
from openedx.core.djangoapps.content_libraries.api import (
|
|
16
17
|
ContentLibraryNotFound,
|
|
@@ -25,6 +26,9 @@ from ol_openedx_git_auto_export.constants import (
|
|
|
25
26
|
ENABLE_GIT_AUTO_LIBRARY_EXPORT,
|
|
26
27
|
EXPORT_DEBOUNCE_CACHE_KEY,
|
|
27
28
|
EXPORT_DEBOUNCE_DELAY,
|
|
29
|
+
EXPORT_DEBOUNCE_PENDING_CACHE_KEY,
|
|
30
|
+
EXPORT_DEBOUNCE_PENDING_TTL,
|
|
31
|
+
EXPORT_DEBOUNCE_TOKEN_TTL,
|
|
28
32
|
REPOSITORY_NAME_MAX_LENGTH,
|
|
29
33
|
ContentType,
|
|
30
34
|
)
|
|
@@ -107,17 +111,17 @@ def get_or_create_git_export_repo_dir():
|
|
|
107
111
|
"""
|
|
108
112
|
Ensure the git export directory exists and return its path.
|
|
109
113
|
"""
|
|
110
|
-
git_repo_export_dir =
|
|
111
|
-
settings, "GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
114
|
+
git_repo_export_dir = Path(
|
|
115
|
+
getattr(settings, "GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos")
|
|
112
116
|
)
|
|
113
|
-
if not
|
|
114
|
-
# for development/docker/vagrant if GIT_REPO_EXPORT_DIR folder does not exist then create it # noqa: E501
|
|
117
|
+
if not git_repo_export_dir.exists():
|
|
115
118
|
log.error(
|
|
116
|
-
"GIT_REPO_EXPORT_DIR
|
|
119
|
+
"GIT_REPO_EXPORT_DIR %s does not exist; creating it", git_repo_export_dir
|
|
117
120
|
)
|
|
118
|
-
|
|
121
|
+
# exist_ok: on a shared volume two processes both pass the check above.
|
|
122
|
+
git_repo_export_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
119
123
|
|
|
120
|
-
return git_repo_export_dir
|
|
124
|
+
return str(git_repo_export_dir)
|
|
121
125
|
|
|
122
126
|
|
|
123
127
|
def github_repo_name_format(course_key_str):
|
|
@@ -147,6 +151,146 @@ def github_repo_name_format(course_key_str):
|
|
|
147
151
|
return repo_name.replace("course-v1-", "")
|
|
148
152
|
|
|
149
153
|
|
|
154
|
+
def debounce_cache_key(content_key):
|
|
155
|
+
return EXPORT_DEBOUNCE_CACHE_KEY.format(content_key=str(content_key))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def pending_cache_key(content_key):
|
|
159
|
+
return EXPORT_DEBOUNCE_PENDING_CACHE_KEY.format(content_key=str(content_key))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def cache_op(op, *args, on_error, **kwargs):
|
|
163
|
+
"""
|
|
164
|
+
Run a debounce cache operation, returning on_error if the backend is down.
|
|
165
|
+
|
|
166
|
+
on_error is required because the safe value is the one you would forget:
|
|
167
|
+
exporting twice is recoverable, never exporting is not.
|
|
168
|
+
"""
|
|
169
|
+
try:
|
|
170
|
+
return op(*args, **kwargs)
|
|
171
|
+
except Exception:
|
|
172
|
+
log.exception(
|
|
173
|
+
"Git export debounce cache unavailable (%s %s); failing open",
|
|
174
|
+
# getattr: op is a bound cache method in production but a Mock
|
|
175
|
+
# in tests, and Mock.__name__ raises.
|
|
176
|
+
getattr(op, "__name__", op),
|
|
177
|
+
args[0],
|
|
178
|
+
)
|
|
179
|
+
return on_error
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def claim_export_slot(content_key):
|
|
183
|
+
"""Claim the right to queue the next export task, if nothing holds it."""
|
|
184
|
+
return cache_op(
|
|
185
|
+
cache.add,
|
|
186
|
+
pending_cache_key(content_key),
|
|
187
|
+
"1",
|
|
188
|
+
timeout=EXPORT_DEBOUNCE_PENDING_TTL,
|
|
189
|
+
on_error=True,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def release_export_slot(content_key):
|
|
194
|
+
"""Give up the queue slot; the marker outlives whatever failed holding it."""
|
|
195
|
+
cache_op(cache.delete, pending_cache_key(content_key), on_error=None)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def current_export_token(content_key, default):
|
|
199
|
+
"""
|
|
200
|
+
Return the newest token recorded for this content, else default.
|
|
201
|
+
|
|
202
|
+
Callers pass their own token, so a missing entry reads as current.
|
|
203
|
+
"""
|
|
204
|
+
return cache_op(
|
|
205
|
+
cache.get, debounce_cache_key(content_key), default, on_error=default
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def queue_export_task(content_key, user, token):
|
|
210
|
+
"""
|
|
211
|
+
Queue an export task; the caller must hold the slot from claim_export_slot.
|
|
212
|
+
"""
|
|
213
|
+
from ol_openedx_git_auto_export.tasks import async_export_to_git # noqa: PLC0415
|
|
214
|
+
|
|
215
|
+
# Renew the slot for the task about to be queued. The hand-off in
|
|
216
|
+
# _superseded keeps holding the slot rather than re-claiming it, so without
|
|
217
|
+
# this the marker expires EXPORT_DEBOUNCE_PENDING_TTL after the first claim
|
|
218
|
+
# -- and an import that keeps publishing past that lets a second chain
|
|
219
|
+
# start, putting two exports on one clone directory.
|
|
220
|
+
cache_op(
|
|
221
|
+
cache.set,
|
|
222
|
+
pending_cache_key(content_key),
|
|
223
|
+
"1",
|
|
224
|
+
timeout=EXPORT_DEBOUNCE_PENDING_TTL,
|
|
225
|
+
on_error=None,
|
|
226
|
+
)
|
|
227
|
+
log.info("Queuing git export for %s in %ds", content_key, EXPORT_DEBOUNCE_DELAY)
|
|
228
|
+
try:
|
|
229
|
+
async_export_to_git.apply_async(
|
|
230
|
+
args=[str(content_key), user],
|
|
231
|
+
kwargs={"token": token},
|
|
232
|
+
countdown=EXPORT_DEBOUNCE_DELAY,
|
|
233
|
+
)
|
|
234
|
+
except Exception:
|
|
235
|
+
log.exception("Failed to queue git export for %s", content_key)
|
|
236
|
+
release_export_slot(content_key)
|
|
237
|
+
raise
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def schedule_export_with_debounce(content_key, resolve_user):
|
|
241
|
+
"""
|
|
242
|
+
Schedule a git export task, debouncing bursts of signals for the same content.
|
|
243
|
+
|
|
244
|
+
See EXPORT_DEBOUNCE_CACHE_KEY in constants.py for the mechanism.
|
|
245
|
+
|
|
246
|
+
resolve_user is a callable so it runs once per burst, not once per signal.
|
|
247
|
+
"""
|
|
248
|
+
# on_commit: Studio publishes under ATOMIC_REQUESTS, so a task queued
|
|
249
|
+
# pre-commit can wake before the content is visible. robust=True: Django
|
|
250
|
+
# stops its hook loop at the first raiser, cancelling later signals' hooks.
|
|
251
|
+
# Lambda not partial -- Django's robust error handler logs __qualname__.
|
|
252
|
+
transaction.on_commit(
|
|
253
|
+
lambda: _queue_debounced_export(content_key, resolve_user), robust=True
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _queue_debounced_export(content_key, resolve_user):
|
|
258
|
+
"""Record this signal's token and queue a task if none is queued already."""
|
|
259
|
+
# An unrecorded token costs the queued task one extra hop, not an export.
|
|
260
|
+
token = uuid.uuid4().hex
|
|
261
|
+
cache_op(
|
|
262
|
+
cache.set,
|
|
263
|
+
debounce_cache_key(content_key),
|
|
264
|
+
token,
|
|
265
|
+
timeout=EXPORT_DEBOUNCE_TOKEN_TTL,
|
|
266
|
+
on_error=None,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
if not claim_export_slot(content_key):
|
|
270
|
+
log.debug(
|
|
271
|
+
"Git export already queued for %s, only updating the debounce token",
|
|
272
|
+
content_key,
|
|
273
|
+
)
|
|
274
|
+
return
|
|
275
|
+
|
|
276
|
+
try:
|
|
277
|
+
get_or_create_git_export_repo_dir()
|
|
278
|
+
except Exception:
|
|
279
|
+
log.exception("Failed to prepare git export directory for %s", content_key)
|
|
280
|
+
release_export_slot(content_key)
|
|
281
|
+
return
|
|
282
|
+
|
|
283
|
+
try:
|
|
284
|
+
user = resolve_user()
|
|
285
|
+
except Exception:
|
|
286
|
+
# Signals that lost the slot queued nothing, so giving up here would
|
|
287
|
+
# strand their changes. Export unattributed instead.
|
|
288
|
+
log.exception("Exporting %s without a publisher", content_key)
|
|
289
|
+
user = None
|
|
290
|
+
|
|
291
|
+
queue_export_task(content_key, user, token)
|
|
292
|
+
|
|
293
|
+
|
|
150
294
|
def export_course_to_git(course_key):
|
|
151
295
|
"""
|
|
152
296
|
Export the course to a Git repository.
|
|
@@ -154,34 +298,15 @@ def export_course_to_git(course_key):
|
|
|
154
298
|
Args:
|
|
155
299
|
course_key (CourseKey): The course key of the course to export.
|
|
156
300
|
"""
|
|
157
|
-
from ol_openedx_git_auto_export.tasks import async_export_to_git # noqa: PLC0415
|
|
158
|
-
|
|
159
301
|
if is_auto_export_enabled():
|
|
160
|
-
get_or_create_git_export_repo_dir()
|
|
161
|
-
course_module = modulestore().get_course(course_key)
|
|
162
302
|
log.info(
|
|
163
303
|
"Course published with auto-export enabled. Starting export... (course id: %s)", # noqa: E501
|
|
164
304
|
course_key,
|
|
165
305
|
)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if cache.add(debounce_key, "1", timeout=EXPORT_DEBOUNCE_DELAY):
|
|
171
|
-
log.info(
|
|
172
|
-
"Scheduling git export for course %s with %ds debounce delay",
|
|
173
|
-
course_key,
|
|
174
|
-
EXPORT_DEBOUNCE_DELAY,
|
|
175
|
-
)
|
|
176
|
-
async_export_to_git.apply_async(
|
|
177
|
-
args=[str(course_key), user],
|
|
178
|
-
countdown=EXPORT_DEBOUNCE_DELAY,
|
|
179
|
-
)
|
|
180
|
-
else:
|
|
181
|
-
log.info(
|
|
182
|
-
"Git export already scheduled for course %s, skipping duplicate signal",
|
|
183
|
-
course_key,
|
|
184
|
-
)
|
|
306
|
+
schedule_export_with_debounce(
|
|
307
|
+
course_key,
|
|
308
|
+
lambda: get_publisher_username(modulestore().get_course(course_key)),
|
|
309
|
+
)
|
|
185
310
|
|
|
186
311
|
|
|
187
312
|
def clear_stale_git_lock(git_url):
|
|
@@ -202,6 +327,21 @@ def clear_stale_git_lock(git_url):
|
|
|
202
327
|
index_lock.unlink()
|
|
203
328
|
|
|
204
329
|
|
|
330
|
+
def _get_library_publisher(library_key):
|
|
331
|
+
"""
|
|
332
|
+
Return the username that published the library, or None for v1 libraries.
|
|
333
|
+
"""
|
|
334
|
+
if not isinstance(library_key, LibraryLocatorV2):
|
|
335
|
+
return None
|
|
336
|
+
try:
|
|
337
|
+
return get_library(library_key).published_by or None
|
|
338
|
+
except ContentLibraryNotFound:
|
|
339
|
+
# Only the author is lost: the burst still exports, with user=None.
|
|
340
|
+
# Later signals lose the slot, so they never re-resolve it either.
|
|
341
|
+
log.warning("Library %s not found; exporting without a publisher", library_key)
|
|
342
|
+
return None
|
|
343
|
+
|
|
344
|
+
|
|
205
345
|
def export_library_to_git(library_key):
|
|
206
346
|
"""
|
|
207
347
|
Export the library to a Git repository.
|
|
@@ -209,28 +349,14 @@ def export_library_to_git(library_key):
|
|
|
209
349
|
Args:
|
|
210
350
|
library_key (LibraryLocator | LibraryLocatorV2): The library key to export.
|
|
211
351
|
"""
|
|
212
|
-
from ol_openedx_git_auto_export.tasks import async_export_to_git # noqa: PLC0415
|
|
213
|
-
|
|
214
352
|
if is_auto_export_enabled(is_library=True):
|
|
215
|
-
get_or_create_git_export_repo_dir()
|
|
216
353
|
log.info(
|
|
217
354
|
"Library updated with auto-export enabled. Starting export... (library id: %s)", # noqa: E501
|
|
218
355
|
library_key,
|
|
219
356
|
)
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if isinstance(library_key, LibraryLocatorV2):
|
|
224
|
-
# V2 libraries have published_by in their metadata
|
|
225
|
-
library_metadata = get_library(library_key)
|
|
226
|
-
user = (
|
|
227
|
-
library_metadata.published_by if library_metadata.published_by else None
|
|
228
|
-
)
|
|
229
|
-
else:
|
|
230
|
-
# V1 libraries don't have published_by field
|
|
231
|
-
pass
|
|
232
|
-
|
|
233
|
-
async_export_to_git.delay(str(library_key), user=user)
|
|
357
|
+
schedule_export_with_debounce(
|
|
358
|
+
library_key, lambda: _get_library_publisher(library_key)
|
|
359
|
+
)
|
|
234
360
|
else:
|
|
235
361
|
log.info(
|
|
236
362
|
"Library auto-export is disabled. Skipping export for library: %s",
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ol-openedx-git-auto-export"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.9.1"
|
|
4
4
|
description = "A plugin that auto saves the course OLX to git when an author publishes it"
|
|
5
5
|
authors = [{ name = "MIT Office of Digital Learning" }]
|
|
6
6
|
license = "BSD-3-Clause"
|
|
7
7
|
readme = "README.rst"
|
|
8
8
|
requires-python = ">=3.11"
|
|
9
9
|
dependencies = [
|
|
10
|
-
"Django>=
|
|
10
|
+
"Django>=5.2",
|
|
11
11
|
"celery>=4.4.7",
|
|
12
|
-
"edx-django-utils
|
|
12
|
+
"edx-django-utils>=8.0.0",
|
|
13
13
|
"edx-opaque-keys",
|
|
14
14
|
"gitpython>=3.1.37",
|
|
15
15
|
"openedx-events",
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
Change Log
|
|
2
|
-
==========
|
|
3
|
-
|
|
4
|
-
Version 0.8.2 (2026-06-10)
|
|
5
|
-
---------------------------
|
|
6
|
-
|
|
7
|
-
* Fixed ``migrate_giturl`` management command passing a stale ``export_course``
|
|
8
|
-
keyword to ``async_create_github_repo``, which raised ``TypeError`` during
|
|
9
|
-
parallel repository creation.
|
|
10
|
-
* Fixed ``migrate_giturl`` reporting a possibly-unbound loop variable in its
|
|
11
|
-
repository-creation progress message; it now reports the count of courses.
|
|
12
|
-
* Added tests for the ``migrate_giturl`` command.
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
from enum import StrEnum
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class ContentType(StrEnum):
|
|
5
|
-
"""Enumeration for content types (Course or Library)."""
|
|
6
|
-
|
|
7
|
-
COURSE = "course"
|
|
8
|
-
LIBRARY = "library"
|
|
9
|
-
|
|
10
|
-
@property
|
|
11
|
-
def display_name(self):
|
|
12
|
-
"""Return the human-readable display name."""
|
|
13
|
-
return self.value.capitalize()
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
# Library key prefixes for different versions
|
|
17
|
-
LIBRARY_V1_PREFIX = "library-v1:"
|
|
18
|
-
LIBRARY_V2_PREFIX = "lib:"
|
|
19
|
-
|
|
20
|
-
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
21
|
-
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
22
|
-
GITHUB_ORG = "GITHUB_ORG"
|
|
23
|
-
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
24
|
-
|
|
25
|
-
# Library-specific feature flags
|
|
26
|
-
ENABLE_GIT_AUTO_LIBRARY_EXPORT = "ENABLE_GIT_AUTO_LIBRARY_EXPORT"
|
|
27
|
-
ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION = "ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION"
|
|
28
|
-
|
|
29
|
-
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
30
|
-
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
|
31
|
-
|
|
32
|
-
# Debounce settings for the signal handler.
|
|
33
|
-
# A single course save triggers 10-30 COURSE_PUBLISHED signals in one request.
|
|
34
|
-
# cache.add() on this key ensures only the first signal schedules a task; all
|
|
35
|
-
# subsequent signals within the window are silently dropped before hitting the broker.
|
|
36
|
-
# The task is scheduled with countdown=EXPORT_DEBOUNCE_DELAY so it runs after
|
|
37
|
-
# the burst window has closed and the course state is fully settled.
|
|
38
|
-
EXPORT_DEBOUNCE_DELAY = 5 # seconds — must exceed the publish burst window
|
|
39
|
-
EXPORT_DEBOUNCE_CACHE_KEY = "git_export_debounce:{course_key}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|