ol-openedx-git-auto-export 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ol_openedx_git_auto_export-0.10.0/CHANGELOG.rst +77 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/PKG-INFO +31 -1
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/README.rst +30 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/admin.py +24 -1
- ol_openedx_git_auto_export-0.10.0/ol_openedx_git_auto_export/constants.py +44 -0
- ol_openedx_git_auto_export-0.10.0/ol_openedx_git_auto_export/management/commands/export_to_git.py +68 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/tasks.py +65 -12
- ol_openedx_git_auto_export-0.10.0/ol_openedx_git_auto_export/utils.py +507 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/pyproject.toml +1 -1
- ol_openedx_git_auto_export-0.9.0/CHANGELOG.rst +0 -12
- ol_openedx_git_auto_export-0.9.0/ol_openedx_git_auto_export/constants.py +0 -39
- ol_openedx_git_auto_export-0.9.0/ol_openedx_git_auto_export/utils.py +0 -294
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/.gitignore +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/LICENSE +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/app.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/exceptions.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/management/commands/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/migrations/0001_initial.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/migrations/0002_contentgitrepository.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/models.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/settings/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/settings/common.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/settings/production.py +0 -0
- {ol_openedx_git_auto_export-0.9.0 → ol_openedx_git_auto_export-0.10.0}/ol_openedx_git_auto_export/signals.py +0 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Change Log
|
|
2
|
+
==========
|
|
3
|
+
|
|
4
|
+
Version 0.10.0 (2026-10-12)
|
|
5
|
+
----------------------------
|
|
6
|
+
|
|
7
|
+
* Added the ``export_to_git`` management command and an ``Export
|
|
8
|
+
selected content to git`` admin action on ``ContentGitRepository``, to queue
|
|
9
|
+
exports on demand when a publish-triggered export failed. The command takes
|
|
10
|
+
``--content-keys`` or ``--all``, supports ``--dry-run``, reports keys it
|
|
11
|
+
queued, skipped, or failed to queue, and exits non-zero if any key errored or
|
|
12
|
+
a key named with ``--content-keys`` was skipped.
|
|
13
|
+
|
|
14
|
+
Version 0.9.1 (2026-09-22)
|
|
15
|
+
---------------------------
|
|
16
|
+
|
|
17
|
+
* Fixed ``export_library_to_git`` queuing a git export for every
|
|
18
|
+
``LIBRARY_BLOCK_PUBLISHED``/``LIBRARY_CONTAINER_PUBLISHED`` signal, flooding
|
|
19
|
+
Celery when a course is imported into a v2 library. A burst now queues one
|
|
20
|
+
task, which exports once the signals stop. The course path, which exported a
|
|
21
|
+
mid-burst snapshot 5 seconds after the first signal, uses the same mechanism.
|
|
22
|
+
* Export tasks are queued from a ``transaction.on_commit`` hook with
|
|
23
|
+
``robust=True``. Studio publishes under ``ATOMIC_REQUESTS``, so a task queued
|
|
24
|
+
mid-transaction could wake before the content was visible and give up
|
|
25
|
+
without retrying.
|
|
26
|
+
* ``async_create_github_repo``'s post-creation export now goes through the
|
|
27
|
+
debounce instead of running inline, so it coalesces with an import burst
|
|
28
|
+
rather than racing it on the same clone directory. As a result
|
|
29
|
+
``migrate_giturl`` reports a repository as done once created, not once
|
|
30
|
+
exported.
|
|
31
|
+
* Fixed ``get_or_create_git_export_repo_dir`` racing itself when
|
|
32
|
+
``GIT_REPO_EXPORT_DIR`` does not exist: two processes both passed the
|
|
33
|
+
existence check and the loser's ``FileExistsError`` dropped its export.
|
|
34
|
+
* The debounce token's cache key is versioned (``git_export_debounce_v2``) so a
|
|
35
|
+
rollback doesn't see its own short-lived key as claimed by the new token. The
|
|
36
|
+
token is bounded at 7 days rather than permanent.
|
|
37
|
+
* The debounce fails open: a dead cache backend queues the export undebounced
|
|
38
|
+
rather than raising into the publish request, and a task that fails to
|
|
39
|
+
enqueue releases its marker so the next signal can retry.
|
|
40
|
+
* The publisher lookup runs once per burst instead of once per signal, so a
|
|
41
|
+
large library import no longer issues a ``get_library()`` query per block
|
|
42
|
+
just to discard it.
|
|
43
|
+
* Fixed a ``ContentLibraryNotFound`` propagating into the publish request when
|
|
44
|
+
the library row is not visible yet; the export is queued without an author.
|
|
45
|
+
* Corrected operator-facing log messages that misreported what happened: a
|
|
46
|
+
parse-failure label on a lookup failure, a retry promise where nothing
|
|
47
|
+
re-queues, a repository-creation claim emitted before the check that skips
|
|
48
|
+
it, and per-signal "starting export" lines during a burst.
|
|
49
|
+
* Documented that the debounce needs a cache backend shared between the CMS
|
|
50
|
+
processes and the Celery workers; under ``LocMemCache`` it is inert.
|
|
51
|
+
* Added tests for the export debounce logic.
|
|
52
|
+
|
|
53
|
+
Version 0.9.0 (2026-09-09)
|
|
54
|
+
---------------------------
|
|
55
|
+
|
|
56
|
+
* Raised the Django floor from ``>=4.0`` to ``>=5.2``, making this the minimum
|
|
57
|
+
version for the Ulmo and Verawood releases. See the Open edX Release
|
|
58
|
+
Compatibility table in ``docs/README.rst``.
|
|
59
|
+
* The test suite now runs under ``cms.envs.test``; the plugin registers only a
|
|
60
|
+
``cms.djangoapp`` entry point, so under LMS settings its app is not installed
|
|
61
|
+
and its models and tasks fail to import.
|
|
62
|
+
|
|
63
|
+
Version 0.8.3 (2026-08-03)
|
|
64
|
+
---------------------------
|
|
65
|
+
|
|
66
|
+
* Fixed a race where an export task could run before the library was committed
|
|
67
|
+
to the store, reporting a spurious "library not found".
|
|
68
|
+
|
|
69
|
+
Version 0.8.2 (2026-06-10)
|
|
70
|
+
---------------------------
|
|
71
|
+
|
|
72
|
+
* Fixed ``migrate_giturl`` management command passing a stale ``export_course``
|
|
73
|
+
keyword to ``async_create_github_repo``, which raised ``TypeError`` during
|
|
74
|
+
parallel repository creation.
|
|
75
|
+
* Fixed ``migrate_giturl`` reporting a possibly-unbound loop variable in its
|
|
76
|
+
repository-creation progress message; it now reports the count of courses.
|
|
77
|
+
* Added tests for the ``migrate_giturl`` command.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ol-openedx-git-auto-export
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: A plugin that auto saves the course OLX to git when an author publishes it
|
|
5
5
|
Author: MIT Office of Digital Learning
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -135,3 +135,33 @@ Library v2 (Content Libraries)
|
|
|
135
135
|
- Library v1 does not emit a creation signal, so the GitHub repository is automatically created on the first library update
|
|
136
136
|
- Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
|
|
137
137
|
- Both library types support automatic git export on updates once configured
|
|
138
|
+
|
|
139
|
+
Re-exporting Without a Publish
|
|
140
|
+
==============================
|
|
141
|
+
|
|
142
|
+
Courses are exported only when published and libraries only when updated, so
|
|
143
|
+
content whose export failed stays stale in git until its next change. To queue
|
|
144
|
+
an export on demand, fix whatever made the export fail, then use either:
|
|
145
|
+
|
|
146
|
+
- The ``Export selected content to git`` action on the Content Git Repositories
|
|
147
|
+
page in Django admin.
|
|
148
|
+
- The ``export_to_git`` management command:
|
|
149
|
+
|
|
150
|
+
.. code-block::
|
|
151
|
+
|
|
152
|
+
# Specific courses or libraries
|
|
153
|
+
./manage.py cms export_to_git --content-keys course-v1:MITx+6.00x+2T2024 lib:MITx:mylib
|
|
154
|
+
|
|
155
|
+
# Every repository with export enabled; --dry-run lists without queueing
|
|
156
|
+
./manage.py cms export_to_git --all --dry-run
|
|
157
|
+
|
|
158
|
+
Both skip, and report, content that no longer exists and content whose
|
|
159
|
+
auto-export settings (``ENABLE_EXPORT_GIT`` plus the course or library flag)
|
|
160
|
+
are off, and report content whose export could not be queued. The admin action
|
|
161
|
+
and ``--content-keys`` also skip rows with ``is_export_enabled`` off, while
|
|
162
|
+
``--all`` only considers rows with it on. The command also reports keys with
|
|
163
|
+
no repository, and exits non-zero if any
|
|
164
|
+
key errored or, with ``--content-keys``, if any named key was skipped. Exports
|
|
165
|
+
go through the same debounce as publish-triggered ones: a request made during
|
|
166
|
+
a pending export's 5-second countdown is merged into it, but one made after
|
|
167
|
+
that export has started queues another.
|
|
@@ -119,3 +119,33 @@ Library v2 (Content Libraries)
|
|
|
119
119
|
- Library v1 does not emit a creation signal, so the GitHub repository is automatically created on the first library update
|
|
120
120
|
- Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
|
|
121
121
|
- Both library types support automatic git export on updates once configured
|
|
122
|
+
|
|
123
|
+
Re-exporting Without a Publish
|
|
124
|
+
==============================
|
|
125
|
+
|
|
126
|
+
Courses are exported only when published and libraries only when updated, so
|
|
127
|
+
content whose export failed stays stale in git until its next change. To queue
|
|
128
|
+
an export on demand, fix whatever made the export fail, then use either:
|
|
129
|
+
|
|
130
|
+
- The ``Export selected content to git`` action on the Content Git Repositories
|
|
131
|
+
page in Django admin.
|
|
132
|
+
- The ``export_to_git`` management command:
|
|
133
|
+
|
|
134
|
+
.. code-block::
|
|
135
|
+
|
|
136
|
+
# Specific courses or libraries
|
|
137
|
+
./manage.py cms export_to_git --content-keys course-v1:MITx+6.00x+2T2024 lib:MITx:mylib
|
|
138
|
+
|
|
139
|
+
# Every repository with export enabled; --dry-run lists without queueing
|
|
140
|
+
./manage.py cms export_to_git --all --dry-run
|
|
141
|
+
|
|
142
|
+
Both skip, and report, content that no longer exists and content whose
|
|
143
|
+
auto-export settings (``ENABLE_EXPORT_GIT`` plus the course or library flag)
|
|
144
|
+
are off, and report content whose export could not be queued. The admin action
|
|
145
|
+
and ``--content-keys`` also skip rows with ``is_export_enabled`` off, while
|
|
146
|
+
``--all`` only considers rows with it on. The command also reports keys with
|
|
147
|
+
no repository, and exits non-zero if any
|
|
148
|
+
key errored or, with ``--content-keys``, if any named key was skipped. Exports
|
|
149
|
+
go through the same debounce as publish-triggered ones: a request made during
|
|
150
|
+
a pending export's 5-second countdown is merged into it, but one made after
|
|
151
|
+
that export has started queues another.
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
Django admin pages for git-auto-export plugin
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from collections import Counter
|
|
6
|
+
|
|
7
|
+
from django.contrib import admin, messages
|
|
6
8
|
from django.contrib.admin import SimpleListFilter
|
|
7
9
|
from django.db.models import Q
|
|
8
10
|
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
|
|
@@ -13,6 +15,7 @@ from ol_openedx_git_auto_export.constants import (
|
|
|
13
15
|
ContentType,
|
|
14
16
|
)
|
|
15
17
|
from ol_openedx_git_auto_export.models import ContentGitRepository
|
|
18
|
+
from ol_openedx_git_auto_export.utils import queue_git_exports
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class ContentTypeFilter(SimpleListFilter):
|
|
@@ -61,6 +64,7 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
|
|
|
61
64
|
search_fields = ("content_key", "git_url")
|
|
62
65
|
list_filter = ("is_export_enabled", ContentTypeFilter)
|
|
63
66
|
list_per_page = 50
|
|
67
|
+
actions = ("export_to_git",)
|
|
64
68
|
|
|
65
69
|
@admin.display(description="Content Type")
|
|
66
70
|
def content_type_display(self, obj):
|
|
@@ -69,6 +73,25 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
|
|
|
69
73
|
return ContentType.LIBRARY.display_name
|
|
70
74
|
return ContentType.COURSE.display_name
|
|
71
75
|
|
|
76
|
+
# Without permissions, view-only staff would get the action too.
|
|
77
|
+
@admin.action(description="Export selected content to git", permissions=["change"])
|
|
78
|
+
def export_to_git(self, request, queryset):
|
|
79
|
+
"""Queue a git export for each selected row with export enabled."""
|
|
80
|
+
keys = queryset.values_list("content_key", flat=True)
|
|
81
|
+
queued, skipped, errored = queue_git_exports(map(str, keys))
|
|
82
|
+
self.message_user(request, f"Queued {len(queued)} git export(s).")
|
|
83
|
+
if skipped:
|
|
84
|
+
reasons = Counter(reason for _, reason in skipped)
|
|
85
|
+
summary = ", ".join(f"{n} {reason}" for reason, n in reasons.items())
|
|
86
|
+
self.message_user(
|
|
87
|
+
request, f"Skipped {len(skipped)}: {summary}.", messages.WARNING
|
|
88
|
+
)
|
|
89
|
+
if errored:
|
|
90
|
+
details = "; ".join(f"{key}: {reason}" for key, reason in errored)
|
|
91
|
+
self.message_user(
|
|
92
|
+
request, f"Failed to queue {len(errored)}: {details}", messages.ERROR
|
|
93
|
+
)
|
|
94
|
+
|
|
72
95
|
def has_delete_permission(self, request, obj=None): # noqa: ARG002
|
|
73
96
|
"""
|
|
74
97
|
Disable delete permission for ContentGitRepository objects.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ContentType(StrEnum):
|
|
5
|
+
"""Enumeration for content types (Course or Library)."""
|
|
6
|
+
|
|
7
|
+
COURSE = "course"
|
|
8
|
+
LIBRARY = "library"
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def display_name(self):
|
|
12
|
+
"""Return the human-readable display name."""
|
|
13
|
+
return self.value.capitalize()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# Library key prefixes for different versions
|
|
17
|
+
LIBRARY_V1_PREFIX = "library-v1:"
|
|
18
|
+
LIBRARY_V2_PREFIX = "lib:"
|
|
19
|
+
|
|
20
|
+
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
21
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
22
|
+
GITHUB_ORG = "GITHUB_ORG"
|
|
23
|
+
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
24
|
+
|
|
25
|
+
# Library-specific feature flags
|
|
26
|
+
ENABLE_GIT_AUTO_LIBRARY_EXPORT = "ENABLE_GIT_AUTO_LIBRARY_EXPORT"
|
|
27
|
+
ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION = "ENABLE_AUTO_GITHUB_LIBRARY_REPO_CREATION"
|
|
28
|
+
|
|
29
|
+
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
30
|
+
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
|
31
|
+
|
|
32
|
+
# A publish burst fires one signal per block. The pending key keeps one export
|
|
33
|
+
# task queued per burst; the debounce key holds a token every signal overwrites.
|
|
34
|
+
# A task exports only while its token is current, else it re-queues -- so the
|
|
35
|
+
# export reflects the end of the burst. A missing token counts as current.
|
|
36
|
+
EXPORT_DEBOUNCE_DELAY = 5 # seconds of quiet before the export happens
|
|
37
|
+
# v2: reusing the unversioned key would make a rollback see it as already
|
|
38
|
+
# claimed by the new long-lived token, and silently stop debouncing.
|
|
39
|
+
EXPORT_DEBOUNCE_CACHE_KEY = "git_export_debounce_v2:{content_key}"
|
|
40
|
+
EXPORT_DEBOUNCE_PENDING_CACHE_KEY = "git_export_pending:{content_key}"
|
|
41
|
+
# Must outlive the countdown plus broker/worker pickup, or a duplicate queues.
|
|
42
|
+
EXPORT_DEBOUNCE_PENDING_TTL = EXPORT_DEBOUNCE_DELAY + 55 # seconds
|
|
43
|
+
# Bounded so dormant content doesn't sit in the cache forever.
|
|
44
|
+
EXPORT_DEBOUNCE_TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
|
ol_openedx_git_auto_export-0.10.0/ol_openedx_git_auto_export/management/commands/export_to_git.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Queue git exports for courses and libraries without waiting for a publish event.
|
|
3
|
+
|
|
4
|
+
Use it to re-push content whose export failed. Fix the cause of the failure
|
|
5
|
+
first, or the re-queued exports will fail the same way.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
python manage.py cms export_to_git --content-keys <key> [<key> ...]
|
|
9
|
+
python manage.py cms export_to_git --all [--dry-run]
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
13
|
+
from ol_openedx_git_auto_export.models import ContentGitRepository
|
|
14
|
+
from ol_openedx_git_auto_export.utils import queue_git_exports
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Command(BaseCommand):
|
|
18
|
+
"""Queue git exports for courses and libraries on demand."""
|
|
19
|
+
|
|
20
|
+
help = "Queue git exports for courses and libraries on demand"
|
|
21
|
+
|
|
22
|
+
def add_arguments(self, parser):
|
|
23
|
+
target = parser.add_mutually_exclusive_group(required=True)
|
|
24
|
+
target.add_argument(
|
|
25
|
+
"--content-keys",
|
|
26
|
+
nargs="+",
|
|
27
|
+
metavar="KEY",
|
|
28
|
+
help="Space-separated course or library keys to export",
|
|
29
|
+
)
|
|
30
|
+
target.add_argument(
|
|
31
|
+
"--all",
|
|
32
|
+
action="store_true",
|
|
33
|
+
help="Export every repository with export enabled",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--dry-run",
|
|
37
|
+
action="store_true",
|
|
38
|
+
help="List what would be queued without queueing anything",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def handle(self, *args, **options): # noqa: ARG002
|
|
42
|
+
if options["all"]:
|
|
43
|
+
enabled = ContentGitRepository.objects.filter(is_export_enabled=True)
|
|
44
|
+
key_strings = map(str, enabled.values_list("content_key", flat=True))
|
|
45
|
+
else:
|
|
46
|
+
key_strings = options["content_keys"]
|
|
47
|
+
queued, skipped, errored = queue_git_exports(
|
|
48
|
+
key_strings, dry_run=options["dry_run"]
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
verb = "Would queue" if options["dry_run"] else "Queued"
|
|
52
|
+
self.stdout.write(self.style.SUCCESS(f"{verb} {len(queued)} export(s):"))
|
|
53
|
+
for key in queued:
|
|
54
|
+
self.stdout.write(f" • {key}")
|
|
55
|
+
self._report(f"Skipped {len(skipped)}:", skipped, self.style.WARNING)
|
|
56
|
+
self._report(f"Errored {len(errored)}:", errored, self.style.ERROR)
|
|
57
|
+
# Named keys are the operator's intent, so skipping one is a failure;
|
|
58
|
+
# under --all, rows left behind by deleted content are expected.
|
|
59
|
+
if errored or (options["content_keys"] and skipped):
|
|
60
|
+
msg = f"{len(errored)} key(s) errored, {len(skipped)} skipped"
|
|
61
|
+
raise CommandError(msg)
|
|
62
|
+
|
|
63
|
+
def _report(self, title, entries, style):
|
|
64
|
+
if not entries:
|
|
65
|
+
return
|
|
66
|
+
self.stdout.write(style(title))
|
|
67
|
+
for key, reason in entries:
|
|
68
|
+
self.stdout.write(style(f" • {key}: {reason}"))
|
|
@@ -17,36 +17,79 @@ from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
|
|
|
17
17
|
from ol_openedx_git_auto_export.models import ContentGitRepository
|
|
18
18
|
from ol_openedx_git_auto_export.utils import (
|
|
19
19
|
clear_stale_git_lock,
|
|
20
|
+
current_export_token,
|
|
20
21
|
get_content_info,
|
|
21
22
|
github_repo_name_format,
|
|
22
23
|
is_auto_repo_creation_enabled,
|
|
24
|
+
queue_export_task,
|
|
25
|
+
release_export_slot,
|
|
26
|
+
schedule_export_with_debounce,
|
|
23
27
|
)
|
|
24
28
|
|
|
25
29
|
LOGGER = get_task_logger(__name__)
|
|
26
30
|
|
|
27
31
|
|
|
32
|
+
def _superseded(context_key_string, user, token):
|
|
33
|
+
"""
|
|
34
|
+
Report whether a newer signal replaced this task, re-queuing if so.
|
|
35
|
+
"""
|
|
36
|
+
current_token = current_export_token(context_key_string, token)
|
|
37
|
+
if current_token == token:
|
|
38
|
+
# Release before exporting, so signals landing during the export
|
|
39
|
+
# queue a fresh task instead of being dropped.
|
|
40
|
+
release_export_slot(context_key_string)
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
LOGGER.info(
|
|
44
|
+
"Newer signals arrived for %s; re-queuing rather than exporting "
|
|
45
|
+
"a mid-burst snapshot",
|
|
46
|
+
context_key_string,
|
|
47
|
+
)
|
|
48
|
+
try:
|
|
49
|
+
# Still holding the slot, so no signal can slip a second task in. A
|
|
50
|
+
# burst extended by another publisher keeps the first as author.
|
|
51
|
+
queue_export_task(context_key_string, user, current_token)
|
|
52
|
+
except Exception:
|
|
53
|
+
# queue_export_task released the slot on its way out.
|
|
54
|
+
LOGGER.exception(
|
|
55
|
+
"Failed to re-queue %s; exporting current state instead",
|
|
56
|
+
context_key_string,
|
|
57
|
+
)
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
|
|
28
63
|
@shared_task
|
|
29
|
-
def async_export_to_git(context_key_string, user=None):
|
|
64
|
+
def async_export_to_git(context_key_string, user=None, token=None):
|
|
30
65
|
"""Export a course or library to Git.
|
|
31
66
|
|
|
32
67
|
Args:
|
|
33
68
|
context_key_string (str): String representation of LearningContextKey
|
|
34
69
|
user: Optional user for git export
|
|
70
|
+
token: Debounce token from utils.queue_export_task; see
|
|
71
|
+
EXPORT_DEBOUNCE_CACHE_KEY in constants.py for the mechanism.
|
|
72
|
+
None skips the staleness check and the debounce state, and
|
|
73
|
+
exists only for messages queued by releases predating the token.
|
|
35
74
|
"""
|
|
75
|
+
if token and _superseded(context_key_string, user, token):
|
|
76
|
+
return
|
|
77
|
+
|
|
36
78
|
try:
|
|
37
79
|
context_key = LearningContextKey.from_string(context_key_string)
|
|
38
80
|
content_info = get_content_info(context_key)
|
|
39
81
|
except ContentNotFoundError:
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
82
|
+
# Publishes queue from on_commit and on-demand exports write nothing, so
|
|
83
|
+
# not the old pre-commit race: the content was most likely deleted
|
|
84
|
+
# during the countdown. Nothing re-queues.
|
|
43
85
|
LOGGER.warning(
|
|
44
|
-
"Content %s not found
|
|
86
|
+
"Content %s not found; abandoning this export (no retry)",
|
|
45
87
|
context_key_string,
|
|
46
88
|
)
|
|
47
89
|
return
|
|
48
90
|
except Exception:
|
|
49
|
-
|
|
91
|
+
# Also covers get_content_info's lookups, so not necessarily a bad key.
|
|
92
|
+
LOGGER.exception("Failed to resolve content %s", context_key_string)
|
|
50
93
|
return
|
|
51
94
|
|
|
52
95
|
try:
|
|
@@ -82,14 +125,22 @@ def async_export_to_git(context_key_string, user=None):
|
|
|
82
125
|
else context_key,
|
|
83
126
|
)
|
|
84
127
|
except ContentGitRepository.DoesNotExist:
|
|
85
|
-
LOGGER.
|
|
86
|
-
"
|
|
87
|
-
"
|
|
128
|
+
LOGGER.info(
|
|
129
|
+
"No git repository registered for %s %s; "
|
|
130
|
+
"creating one if auto-creation is enabled.",
|
|
88
131
|
content_info["content_type"],
|
|
89
132
|
context_key_string,
|
|
90
133
|
)
|
|
91
|
-
|
|
92
|
-
|
|
134
|
+
try:
|
|
135
|
+
if is_auto_repo_creation_enabled(is_library=content_info["is_library"]):
|
|
136
|
+
async_create_github_repo.delay(str(context_key), export_content=True)
|
|
137
|
+
except Exception:
|
|
138
|
+
# Sibling except clauses don't catch this one.
|
|
139
|
+
LOGGER.exception(
|
|
140
|
+
"Failed to check/trigger repo creation for %s %s",
|
|
141
|
+
content_info["content_type"],
|
|
142
|
+
context_key_string,
|
|
143
|
+
)
|
|
93
144
|
except Exception:
|
|
94
145
|
LOGGER.exception(
|
|
95
146
|
"Unknown error occurred during async %s content export to git (%s id: %s)",
|
|
@@ -213,6 +264,8 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
213
264
|
LOGGER.error(response_msg)
|
|
214
265
|
|
|
215
266
|
if ssh_url and export_content:
|
|
216
|
-
|
|
267
|
+
# Debounced like every other export, so it coalesces with an import
|
|
268
|
+
# burst instead of racing it on the same clone directory. No author.
|
|
269
|
+
schedule_export_with_debounce(context_key, lambda: None)
|
|
217
270
|
|
|
218
271
|
return True, response_msg or ssh_url
|