ol-openedx-git-auto-export 0.12.0__tar.gz → 0.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/PKG-INFO +26 -1
  2. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/README.rst +25 -0
  3. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/admin.py +56 -2
  4. ol_openedx_git_auto_export-0.13.0/ol_openedx_git_auto_export/migrations/0003_contentgitexportlog.py +88 -0
  5. ol_openedx_git_auto_export-0.13.0/ol_openedx_git_auto_export/models.py +73 -0
  6. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/tasks.py +208 -35
  7. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/utils.py +191 -1
  8. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/pyproject.toml +1 -1
  9. ol_openedx_git_auto_export-0.12.0/ol_openedx_git_auto_export/models.py +0 -28
  10. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/.gitignore +0 -0
  11. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/CHANGELOG.rst +0 -0
  12. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/LICENSE +0 -0
  13. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/__init__.py +0 -0
  14. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/app.py +0 -0
  15. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/constants.py +0 -0
  16. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/exceptions.py +0 -0
  17. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/__init__.py +0 -0
  18. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/export_to_git.py +0 -0
  19. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +0 -0
  20. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/0001_initial.py +0 -0
  21. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/0002_contentgitrepository.py +0 -0
  22. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
  23. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/__init__.py +0 -0
  24. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/common.py +0 -0
  25. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/production.py +0 -0
  26. {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/signals.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ol-openedx-git-auto-export
3
- Version: 0.12.0
3
+ Version: 0.13.0
4
4
  Summary: A plugin that auto saves the course OLX to git when an author publishes it
5
5
  Author: MIT Office of Digital Learning
6
6
  License-Expression: BSD-3-Clause
@@ -146,6 +146,31 @@ Library v2 (Content Libraries)
146
146
  - Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
147
147
  - Both library types support automatic git export on updates once configured
148
148
 
149
+ Export History
150
+ ==============
151
+
152
+ Every export that runs adds a row to **Content Git Export Logs** in Django
153
+ admin (``/admin/ol_openedx_git_auto_export/contentgitexportlog/``): its
154
+ status, author, Celery task id, a log of its steps with UTC times (each git
155
+ command and the content export, as they start), and, on failure, the error
156
+ with its traceback and git's own output. Failed GitHub repository creations
157
+ are recorded there too. The Content Git Repositories list shows each row's
158
+ latest status and time. The user and password in an ``http(s)`` git URL are
159
+ stored as ``***``.
160
+
161
+ The row is created as In progress when the export starts and updated as it
162
+ goes. A worker killed mid-export (out of memory, the time limit, a restart)
163
+ leaves it In progress until the next export of the same content marks it
164
+ Failed. Until then, a "modified" time more than 10 minutes old (the export time
165
+ limit) means the run is dead. The last line of its log is the last step it
166
+ started; an empty log means it died before the export itself began, while
167
+ loading the content.
168
+
169
+ Runs that are merged into a later one, or wait for another export of the same
170
+ content, add no row; the run that finally exports does. A wait that cannot
171
+ re-queue itself is recorded as Failed. Rows are kept indefinitely; only
172
+ superusers can delete them from the admin.
173
+
149
174
  Re-exporting Without a Publish
150
175
  ==============================
151
176
 
@@ -130,6 +130,31 @@ Library v2 (Content Libraries)
130
130
  - Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
131
131
  - Both library types support automatic git export on updates once configured
132
132
 
133
+ Export History
134
+ ==============
135
+
136
+ Every export that runs adds a row to **Content Git Export Logs** in Django
137
+ admin (``/admin/ol_openedx_git_auto_export/contentgitexportlog/``): its
138
+ status, author, Celery task id, a log of its steps with UTC times (each git
139
+ command and the content export, as they start), and, on failure, the error
140
+ with its traceback and git's own output. Failed GitHub repository creations
141
+ are recorded there too. The Content Git Repositories list shows each row's
142
+ latest status and time. The user and password in an ``http(s)`` git URL are
143
+ stored as ``***``.
144
+
145
+ The row is created as In progress when the export starts and updated as it
146
+ goes. A worker killed mid-export (out of memory, the time limit, a restart)
147
+ leaves it In progress until the next export of the same content marks it
148
+ Failed. Until then, a "modified" time more than 10 minutes old (the export time
149
+ limit) means the run is dead. The last line of its log is the last step it
150
+ started; an empty log means it died before the export itself began, while
151
+ loading the content.
152
+
153
+ Runs that are merged into a later one, or wait for another export of the same
154
+ content, add no row; the run that finally exports does. A wait that cannot
155
+ re-queue itself is recorded as Failed. Rows are kept indefinitely; only
156
+ superusers can delete them from the admin.
157
+
133
158
  Re-exporting Without a Publish
134
159
  ==============================
135
160
 
@@ -6,7 +6,7 @@ from collections import Counter
6
6
 
7
7
  from django.contrib import admin, messages
8
8
  from django.contrib.admin import SimpleListFilter
9
- from django.db.models import Q
9
+ from django.db.models import OuterRef, Q, Subquery
10
10
  from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
11
11
 
12
12
  from ol_openedx_git_auto_export.constants import (
@@ -14,7 +14,11 @@ from ol_openedx_git_auto_export.constants import (
14
14
  LIBRARY_V2_PREFIX,
15
15
  ContentType,
16
16
  )
17
- from ol_openedx_git_auto_export.models import ContentGitRepository
17
+ from ol_openedx_git_auto_export.models import (
18
+ ContentGitExportLog,
19
+ ContentGitRepository,
20
+ ExportStatus,
21
+ )
18
22
  from ol_openedx_git_auto_export.utils import queue_git_exports
19
23
 
20
24
 
@@ -60,12 +64,39 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
60
64
  "git_url",
61
65
  "is_export_enabled",
62
66
  "content_type_display",
67
+ "last_export_status",
68
+ "last_export_at",
63
69
  )
64
70
  search_fields = ("content_key", "git_url")
65
71
  list_filter = ("is_export_enabled", ContentTypeFilter)
66
72
  list_per_page = 50
67
73
  actions = ("export_to_git",)
68
74
 
75
+ def get_queryset(self, request):
76
+ """Annotate each row with its newest export log entry."""
77
+ newest = ContentGitExportLog.objects.filter(
78
+ content_key=OuterRef("content_key")
79
+ ).order_by("-created", "-id")
80
+ return (
81
+ super()
82
+ .get_queryset(request)
83
+ .annotate(
84
+ last_status=Subquery(newest.values("status")[:1]),
85
+ last_at=Subquery(newest.values("created")[:1]),
86
+ )
87
+ )
88
+
89
+ @admin.display(description="Last export", ordering="last_status")
90
+ def last_export_status(self, obj):
91
+ """Status of the newest export log row, if any."""
92
+ # .get: a rolled-back release may not know a newer status.
93
+ return dict(ExportStatus.choices).get(obj.last_status, obj.last_status or "-")
94
+
95
+ @admin.display(description="Last export at", ordering="last_at")
96
+ def last_export_at(self, obj):
97
+ """Time of the newest export log row, if any."""
98
+ return obj.last_at
99
+
69
100
  @admin.display(description="Content Type")
70
101
  def content_type_display(self, obj):
71
102
  """Display whether the content is a course or library."""
@@ -102,3 +133,26 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
102
133
  To stop exporting content, set `is_export_enabled` to False
103
134
  """
104
135
  return False
136
+
137
+
138
+ @admin.register(ContentGitExportLog)
139
+ class ContentGitExportLogAdmin(admin.ModelAdmin):
140
+ """Read-only history of git export runs."""
141
+
142
+ # In progress with no activity past EXPORT_TIME_LIMIT: the run was killed.
143
+ list_display = ("content_key", "status", "user", "created", "modified", "git_url")
144
+ list_filter = ("status", ContentTypeFilter)
145
+ search_fields = ("content_key",)
146
+ list_per_page = 50
147
+
148
+ def has_add_permission(self, request): # noqa: ARG002
149
+ """Rows are written only by the export tasks."""
150
+ return False
151
+
152
+ def has_change_permission(self, request, obj=None): # noqa: ARG002
153
+ """Rows are an audit trail; staff can view but not edit them."""
154
+ return False
155
+
156
+ def has_delete_permission(self, request, obj=None): # noqa: ARG002
157
+ """Superusers only, e.g. to remove a row holding a leaked secret."""
158
+ return request.user.is_superuser
@@ -0,0 +1,88 @@
1
+ # Generated by Django 5.2.18 on 2026-11-04 12:42
2
+
3
+ import django.utils.timezone
4
+ import model_utils.fields
5
+ import opaque_keys.edx.django.models
6
+ from django.db import migrations, models
7
+
8
+
9
+ class Migration(migrations.Migration):
10
+ dependencies = [
11
+ ("ol_openedx_git_auto_export", "0002_contentgitrepository"),
12
+ ]
13
+
14
+ operations = [
15
+ migrations.CreateModel(
16
+ name="ContentGitExportLog",
17
+ fields=[
18
+ (
19
+ "id",
20
+ models.AutoField(
21
+ auto_created=True,
22
+ primary_key=True,
23
+ serialize=False,
24
+ verbose_name="ID",
25
+ ),
26
+ ),
27
+ (
28
+ "created",
29
+ model_utils.fields.AutoCreatedField(
30
+ default=django.utils.timezone.now,
31
+ editable=False,
32
+ verbose_name="created",
33
+ ),
34
+ ),
35
+ (
36
+ "modified",
37
+ model_utils.fields.AutoLastModifiedField(
38
+ default=django.utils.timezone.now,
39
+ editable=False,
40
+ verbose_name="modified",
41
+ ),
42
+ ),
43
+ (
44
+ "content_key",
45
+ opaque_keys.edx.django.models.LearningContextKeyField(
46
+ max_length=255
47
+ ),
48
+ ),
49
+ (
50
+ "status",
51
+ models.CharField(
52
+ choices=[
53
+ ("in_progress", "In progress"),
54
+ ("success", "Success"),
55
+ ("no_changes", "No changes"),
56
+ ("skipped_disabled", "Skipped: export disabled"),
57
+ ("not_found", "Content not found"),
58
+ ("no_repo", "No repository"),
59
+ ("repo_creation_failed", "Repository creation failed"),
60
+ ("failed", "Failed"),
61
+ ],
62
+ max_length=32,
63
+ ),
64
+ ),
65
+ ("user", models.CharField(blank=True, max_length=150)),
66
+ ("celery_task_id", models.CharField(blank=True, max_length=255)),
67
+ ("git_url", models.CharField(blank=True, max_length=255)),
68
+ ("error_message", models.TextField(blank=True)),
69
+ ("git_output", models.TextField(blank=True)),
70
+ ("log", models.TextField(blank=True)),
71
+ ],
72
+ options={
73
+ "verbose_name": "Content Git Export Log",
74
+ "verbose_name_plural": "Content Git Export Logs",
75
+ "ordering": ["-created"],
76
+ "indexes": [
77
+ models.Index(
78
+ fields=["content_key", "-created"],
79
+ name="ol_openedx__content_bcf928_idx",
80
+ ),
81
+ models.Index(
82
+ fields=["-created", "-id"],
83
+ name="ol_openedx__created_1d4ac7_idx",
84
+ ),
85
+ ],
86
+ },
87
+ ),
88
+ ]
@@ -0,0 +1,73 @@
1
+ """
2
+ Django models for the git auto-export plugin.
3
+ """
4
+
5
+ from django.db import models
6
+ from model_utils.models import TimeStampedModel
7
+ from opaque_keys.edx.django.models import LearningContextKeyField
8
+
9
+
10
+ class ContentGitRepository(TimeStampedModel):
11
+ """
12
+ Model to store Git repository information for courses and libraries.
13
+
14
+ This model uses LearningContextKeyField which supports both CourseKey
15
+ and LibraryLocator, making it suitable for both courses and libraries.
16
+ """
17
+
18
+ content_key = LearningContextKeyField(max_length=255, primary_key=True)
19
+ git_url = models.CharField(max_length=255)
20
+ is_export_enabled = models.BooleanField(default=True)
21
+
22
+ class Meta:
23
+ verbose_name = "Content Git Repository"
24
+ verbose_name_plural = "Content Git Repositories"
25
+ ordering = ["-created"]
26
+
27
+ def __str__(self):
28
+ return f"ContentGitRepository (content_key={self.content_key})"
29
+
30
+
31
+ class ExportStatus(models.TextChoices):
32
+ """Outcome of one export run; in_progress until it finishes."""
33
+
34
+ # Left as is by a run killed mid-export (OOM, time limit, pod restart).
35
+ IN_PROGRESS = "in_progress", "In progress"
36
+ SUCCESS = "success", "Success"
37
+ NO_CHANGES = "no_changes", "No changes"
38
+ SKIPPED_DISABLED = "skipped_disabled", "Skipped: export disabled"
39
+ NOT_FOUND = "not_found", "Content not found"
40
+ NO_REPO = "no_repo", "No repository"
41
+ REPO_CREATION_FAILED = "repo_creation_failed", "Repository creation failed"
42
+ FAILED = "failed", "Failed"
43
+
44
+
45
+ class ContentGitExportLog(TimeStampedModel):
46
+ """
47
+ One row per export run, kept so intermittent failures show as a pattern.
48
+ The newest row per content_key is its current status.
49
+ """
50
+
51
+ # Not a FK: not_found and no_repo rows have no ContentGitRepository.
52
+ content_key = LearningContextKeyField(max_length=255)
53
+ status = models.CharField(max_length=32, choices=ExportStatus.choices)
54
+ user = models.CharField(max_length=150, blank=True)
55
+ celery_task_id = models.CharField(max_length=255, blank=True)
56
+ git_url = models.CharField(max_length=255, blank=True)
57
+ error_message = models.TextField(blank=True)
58
+ git_output = models.TextField(blank=True)
59
+ # Timestamped log lines, written as the run goes.
60
+ log = models.TextField(blank=True)
61
+
62
+ class Meta:
63
+ verbose_name = "Content Git Export Log"
64
+ verbose_name_plural = "Content Git Export Logs"
65
+ ordering = ["-created"]
66
+ indexes = [
67
+ models.Index(fields=["content_key", "-created"]),
68
+ # The admin's unfiltered, newest-first list.
69
+ models.Index(fields=["-created", "-id"]),
70
+ ]
71
+
72
+ def __str__(self):
73
+ return f"ContentGitExportLog ({self.content_key}: {self.status})"
@@ -5,9 +5,12 @@ This module defines background tasks that handle the actual export of course
5
5
  content to Git repositories using Celery for asynchronous processing.
6
6
  """
7
7
 
8
+ from contextlib import contextmanager
9
+
8
10
  import requests
9
11
  from celery import shared_task # pylint: disable=import-error
10
12
  from celery.utils.log import get_task_logger
13
+ from cms.djangoapps.contentstore import git_export_utils
11
14
  from cms.djangoapps.contentstore.git_export_utils import GitExportError, export_to_git
12
15
  from django.conf import settings
13
16
  from opaque_keys import InvalidKeyError
@@ -16,21 +19,24 @@ from rest_framework import status
16
19
 
17
20
  from ol_openedx_git_auto_export.constants import EXPORT_TIME_LIMIT
18
21
  from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
19
- from ol_openedx_git_auto_export.models import ContentGitRepository
22
+ from ol_openedx_git_auto_export.models import ContentGitRepository, ExportStatus
20
23
  from ol_openedx_git_auto_export.utils import (
21
24
  acquire_export_lock,
22
25
  burst_exceeded_max_wait,
23
26
  claim_export_slot,
24
27
  clear_stale_git_lock,
28
+ close_dead_runs,
25
29
  current_export_token,
26
30
  get_content_info,
27
31
  github_repo_name_format,
28
32
  is_auto_repo_creation_enabled,
29
33
  queue_debounced_export,
30
34
  queue_export_task,
35
+ record_export_result,
31
36
  release_export_lock,
32
37
  release_export_slot,
33
38
  schedule_export_with_debounce,
39
+ track_export,
34
40
  )
35
41
 
36
42
  LOGGER = get_task_logger(__name__)
@@ -79,23 +85,82 @@ def _superseded(context_key_string, user, token):
79
85
  return True
80
86
 
81
87
 
88
+ # export_to_git's steps, by the module-level name it calls them through.
89
+ # Releases before Willow have only cmd_log and export_course_to_xml.
90
+ _EXPORT_STEPS = {
91
+ # The args may hold the URL, so only "git <command>".
92
+ "cmd_log": lambda cmd, *_: f"Running {' '.join(cmd[:2])}",
93
+ **dict.fromkeys(
94
+ (
95
+ "export_course_to_xml",
96
+ "export_library_to_xml",
97
+ "extract_library_v2_zip_to_dir",
98
+ ),
99
+ lambda *_: "Exporting the content",
100
+ ),
101
+ }
102
+
103
+
104
+ def _logged(func, describe):
105
+ def step(*args, **kwargs):
106
+ LOGGER.info(describe(*args))
107
+ return func(*args, **kwargs)
108
+
109
+ return step
110
+
111
+
112
+ @contextmanager
113
+ def _log_export_steps():
114
+ """
115
+ Log each step export_to_git takes as it starts.
116
+
117
+ Upstream logs git commands only after they end, at DEBUG, and the content
118
+ export only if it fails, so a run killed mid-push or mid-export would
119
+ show an earlier step as its last.
120
+ """
121
+ originals = {
122
+ name: getattr(git_export_utils, name)
123
+ for name in _EXPORT_STEPS
124
+ if hasattr(git_export_utils, name)
125
+ }
126
+ # shortcut: patches the module for the whole process, fine for prefork
127
+ # workers (one task per process); a pool running tasks concurrently in
128
+ # one process would need a lock.
129
+ for name, original in originals.items():
130
+ setattr(git_export_utils, name, _logged(original, _EXPORT_STEPS[name]))
131
+ try:
132
+ yield
133
+ finally:
134
+ for name, original in originals.items():
135
+ setattr(git_export_utils, name, original)
136
+
137
+
138
+ def _git_output(exc):
139
+ """Return git's own output from a GitExportError, or ''."""
140
+ output = getattr(exc.__cause__, "output", None) or ""
141
+ # bytes today (check_output without text=True); str if that changes.
142
+ if isinstance(output, bytes):
143
+ output = output.decode(errors="replace")
144
+ return output
145
+
146
+
82
147
  def _export_unless_unchanged(context_key, git_url, user):
83
148
  """
84
149
  Run export_to_git, treating unchanged content as success.
85
150
 
86
151
  export_to_git raises CANNOT_COMMIT both for this and for real failures
87
152
  such as a held index.lock; only git's own output tells them apart.
153
+
154
+ Returns False if there was nothing to commit, else True.
88
155
  """
89
156
  try:
90
157
  export_to_git(context_key, git_url, user=user)
91
158
  except GitExportError as exc:
92
- output = getattr(exc.__cause__, "output", None) or ""
93
- # bytes today (check_output without text=True); str if that changes.
94
- if isinstance(output, bytes):
95
- output = output.decode(errors="replace")
96
- if "nothing to commit" not in output:
159
+ if "nothing to commit" not in _git_output(exc):
97
160
  raise
98
161
  LOGGER.info("No changes to export for %s", context_key)
162
+ return False
163
+ return True
99
164
 
100
165
 
101
166
  def _wait_to_export(context_key_string, user, token):
@@ -105,13 +170,26 @@ def _wait_to_export(context_key_string, user, token):
105
170
  Whatever holds it may predate this task's changes, so this one runs
106
171
  after it instead of racing it.
107
172
  """
108
- if token:
109
- # Keeps the slot and the burst's start, as a superseded task does.
110
- queue_export_task(
111
- context_key_string, user, current_export_token(context_key_string, token)
173
+ try:
174
+ if token:
175
+ # Keeps the slot and the burst's start, as a superseded task does.
176
+ queue_export_task(
177
+ context_key_string,
178
+ user,
179
+ current_export_token(context_key_string, token),
180
+ )
181
+ else:
182
+ queue_debounced_export(context_key_string, lambda: user)
183
+ except Exception as exc:
184
+ # The export is lost: nothing else is queued to run it.
185
+ record_export_result(
186
+ context_key_string,
187
+ ExportStatus.FAILED,
188
+ user=user,
189
+ error_message=f"Failed to re-queue the export: {exc!r}",
190
+ exc=exc,
112
191
  )
113
- else:
114
- queue_debounced_export(context_key_string, lambda: user)
192
+ raise
115
193
 
116
194
 
117
195
  def _get_content_repo(context_key_string):
@@ -155,11 +233,17 @@ def async_export_to_git(context_key_string, user=None, token=None):
155
233
  try:
156
234
  content_repo = _get_content_repo(context_key_string)
157
235
  unlocked = _index_unlocked(content_repo)
158
- except Exception:
236
+ except Exception as exc:
159
237
  # Still holding the slot: later signals would see an export
160
238
  # queued that will never run.
161
239
  if token:
162
240
  release_export_slot(context_key_string)
241
+ record_export_result(
242
+ context_key_string,
243
+ ExportStatus.FAILED,
244
+ user=user,
245
+ exc=exc,
246
+ )
163
247
  raise
164
248
  if not unlocked:
165
249
  # A killed export's lock, or Studio's own export running.
@@ -173,14 +257,26 @@ def async_export_to_git(context_key_string, user=None, token=None):
173
257
  # queue a fresh task instead of being dropped. Only now, holding
174
258
  # the lock: this also ends the burst, which a wait must not do.
175
259
  release_export_slot(context_key_string)
176
- _export(context_key_string, user, content_repo)
260
+ if owned: # None: the cache is down, so another run may be live
261
+ close_dead_runs(context_key_string)
262
+ with (
263
+ track_export(
264
+ context_key_string,
265
+ user,
266
+ LOGGER,
267
+ git_export_utils.log,
268
+ git_url=content_repo.git_url if content_repo else "",
269
+ ),
270
+ _log_export_steps(),
271
+ ):
272
+ _export(context_key_string, user, content_repo)
177
273
  finally:
178
274
  # None means the cache was down: this export never held the lock.
179
275
  if owned:
180
276
  release_export_lock(context_key_string)
181
277
 
182
278
 
183
- def _export(context_key_string, user, content_repo):
279
+ def _export(context_key_string, user, content_repo): # noqa: C901
184
280
  """Resolve the content, then export it to content_repo."""
185
281
  try:
186
282
  context_key = LearningContextKey.from_string(context_key_string)
@@ -193,10 +289,18 @@ def _export(context_key_string, user, content_repo):
193
289
  "Content %s not found; abandoning this export (no retry)",
194
290
  context_key_string,
195
291
  )
292
+ record_export_result(context_key_string, ExportStatus.NOT_FOUND, user=user)
196
293
  return
197
- except Exception:
294
+ except Exception as exc:
198
295
  # Also covers get_content_info's lookups, so not necessarily a bad key.
199
296
  LOGGER.exception("Failed to resolve content %s", context_key_string)
297
+ if not isinstance(exc, InvalidKeyError): # unstorable key: log only
298
+ record_export_result(
299
+ context_key_string,
300
+ ExportStatus.FAILED,
301
+ user=user,
302
+ exc=exc,
303
+ )
200
304
  raise
201
305
 
202
306
  if content_repo is None:
@@ -206,59 +310,94 @@ def _export(context_key_string, user, content_repo):
206
310
  content_info["content_type"],
207
311
  context_key_string,
208
312
  )
313
+ note = "Auto-creation is disabled."
314
+ queue_error = None
209
315
  try:
210
316
  if is_auto_repo_creation_enabled(is_library=content_info["is_library"]):
211
317
  async_create_github_repo.delay(str(context_key), export_content=True)
212
- except Exception:
318
+ note = "Repository creation queued."
319
+ except Exception as exc:
213
320
  # Nothing else catches this one, and it must not fail the task.
214
321
  LOGGER.exception(
215
322
  "Failed to check/trigger repo creation for %s %s",
216
323
  content_info["content_type"],
217
324
  context_key_string,
218
325
  )
326
+ note = f"Failed to queue repository creation: {exc!r}"
327
+ queue_error = exc
328
+ record_export_result(
329
+ context_key,
330
+ ExportStatus.NO_REPO,
331
+ user=user,
332
+ error_message=note,
333
+ exc=queue_error,
334
+ )
219
335
  return
220
336
 
337
+ git_url = content_repo.git_url
338
+ content_id = (
339
+ content_info["content_module"].id
340
+ if hasattr(content_info["content_module"], "id")
341
+ else context_key
342
+ )
221
343
  try:
222
344
  if content_repo.is_export_enabled:
223
345
  LOGGER.info(
224
346
  "Starting async %s content export to git (%s id: %s)",
225
347
  content_info["content_type"],
226
348
  content_info["content_type"],
227
- content_info["content_module"].id
228
- if hasattr(content_info["content_module"], "id")
229
- else context_key,
349
+ content_id,
230
350
  )
231
351
  # _index_unlocked removed any stale index.lock; export_to_git's
232
352
  # `git reset --hard` + `git clean` handle files a crash left dirty.
233
- _export_unless_unchanged(context_key, content_repo.git_url, user)
353
+ outcome = (
354
+ ExportStatus.SUCCESS
355
+ if _export_unless_unchanged(context_key, git_url, user)
356
+ else ExportStatus.NO_CHANGES
357
+ )
234
358
  else:
235
359
  LOGGER.info(
236
360
  "Git export is disabled for %s %s. Skipping export.",
237
361
  content_info["content_type"],
238
362
  context_key_string,
239
363
  )
240
- except GitExportError:
364
+ outcome = ExportStatus.SKIPPED_DISABLED
365
+ except GitExportError as exc:
241
366
  LOGGER.exception(
242
367
  "Failed async %s content export to git (%s id: %s)",
243
368
  content_info["content_type"],
244
369
  content_info["content_type"],
245
- content_info["content_module"].id
246
- if hasattr(content_info["content_module"], "id")
247
- else context_key,
370
+ content_id,
371
+ )
372
+ record_export_result(
373
+ context_key,
374
+ ExportStatus.FAILED,
375
+ user=user,
376
+ git_url=git_url,
377
+ error_message=str(exc),
378
+ git_output=_git_output(exc),
379
+ exc=exc,
248
380
  )
249
381
  # Re-raised, not retried, so Celery records the failure; most push
250
382
  # and commit errors recur until someone fixes their cause.
251
383
  raise
252
- except Exception:
384
+ except Exception as exc:
253
385
  LOGGER.exception(
254
386
  "Unknown error occurred during async %s content export to git (%s id: %s)",
255
387
  content_info["content_type"],
256
388
  content_info["content_type"],
257
- content_info["content_module"].id
258
- if hasattr(content_info["content_module"], "id")
259
- else context_key,
389
+ content_id,
390
+ )
391
+ record_export_result(
392
+ context_key,
393
+ ExportStatus.FAILED,
394
+ user=user,
395
+ git_url=git_url,
396
+ exc=exc,
260
397
  )
261
398
  raise
399
+ else:
400
+ record_export_result(context_key, outcome, user=user, git_url=git_url)
262
401
 
263
402
 
264
403
  @shared_task(
@@ -266,7 +405,11 @@ def _export(context_key_string, user, content_repo):
266
405
  autoretry_for=(requests.exceptions.RequestException,),
267
406
  retry_kwargs={"max_retries": 3, "countdown": 10},
268
407
  )
269
- def async_create_github_repo(self, context_key_str, export_content=False): # noqa: FBT002
408
+ def async_create_github_repo( # noqa: C901, PLR0912, PLR0915
409
+ self,
410
+ context_key_str,
411
+ export_content=False, # noqa: FBT002
412
+ ):
270
413
  """
271
414
  Create a GitHub repository for the given course or library key.
272
415
 
@@ -287,9 +430,18 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
287
430
  except ContentNotFoundError as exc:
288
431
  response_msg = f"Content {context_key_str} not found: {exc}"
289
432
  LOGGER.warning(response_msg)
433
+ record_export_result(
434
+ context_key_str, ExportStatus.NOT_FOUND, error_message=response_msg
435
+ )
290
436
  return False, response_msg
291
- except Exception:
437
+ except Exception as exc:
292
438
  LOGGER.exception("Failed to parse context key: %s", context_key_str)
439
+ if not isinstance(exc, InvalidKeyError): # unstorable key: log only
440
+ record_export_result(
441
+ context_key_str,
442
+ ExportStatus.REPO_CREATION_FAILED,
443
+ exc=exc,
444
+ )
293
445
  return False, f"Invalid context key: {context_key_str}"
294
446
 
295
447
  content_id_slugified = github_repo_name_format(str(context_key))
@@ -330,13 +482,29 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
330
482
  "has_wiki": False,
331
483
  "auto_init": True,
332
484
  }
333
- response = requests.post(url, headers=headers, json=payload, timeout=30)
485
+ max_retries = self.retry_kwargs.get("max_retries", 3)
486
+ try:
487
+ response = requests.post(url, headers=headers, json=payload, timeout=30)
488
+ # A malformed 201 body raises JSONDecodeError, also a RequestException.
489
+ repo_data = (
490
+ response.json() if response.status_code == status.HTTP_201_CREATED else None
491
+ )
492
+ except requests.exceptions.RequestException as exc:
493
+ # autoretry_for retries these; record only the last attempt.
494
+ if self.request.retries >= max_retries:
495
+ record_export_result(
496
+ context_key, ExportStatus.REPO_CREATION_FAILED, exc=exc
497
+ )
498
+ raise
334
499
  if response.status_code != status.HTTP_201_CREATED:
335
- response_msg = f"Failed to create GitHub repository for {content_info['content_type']} {context_key}: {response.json()}" # noqa: E501
500
+ try:
501
+ error_body = response.json()
502
+ except ValueError: # e.g. a proxy's HTML error page
503
+ error_body = response.text[:500]
504
+ response_msg = f"Failed to create GitHub repository for {content_info['content_type']} {context_key} (HTTP {response.status_code}): {error_body}" # noqa: E501
336
505
  LOGGER.error(response_msg)
337
506
 
338
507
  # Retry the task if we haven't exceeded max retries
339
- max_retries = self.retry_kwargs.get("max_retries", 3)
340
508
  if self.request.retries < max_retries:
341
509
  LOGGER.info(
342
510
  "Retrying GitHub repository creation for %s %s (attempt %d/%d)",
@@ -348,9 +516,11 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
348
516
  countdown = self.retry_kwargs.get("countdown", 10)
349
517
  raise self.retry(countdown=countdown, exc=Exception(response_msg))
350
518
 
519
+ record_export_result(
520
+ context_key, ExportStatus.REPO_CREATION_FAILED, error_message=response_msg
521
+ )
351
522
  return False, response_msg
352
523
 
353
- repo_data = response.json()
354
524
  ssh_url = repo_data.get("ssh_url")
355
525
  if ssh_url:
356
526
  # Use the new ContentGitRepository model
@@ -371,6 +541,9 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
371
541
  Response data: {repo_data}
372
542
  """
373
543
  LOGGER.error(response_msg)
544
+ record_export_result(
545
+ context_key, ExportStatus.REPO_CREATION_FAILED, error_message=response_msg
546
+ )
374
547
 
375
548
  if ssh_url and export_content:
376
549
  # Debounced like every other export, so it coalesces with an import
@@ -5,14 +5,22 @@ Utility functions for the ol_openedx_git_auto_export app.
5
5
  import logging
6
6
  import re
7
7
  import time
8
+ import traceback
8
9
  import uuid
10
+ from contextlib import contextmanager
11
+ from contextvars import ContextVar
12
+ from datetime import UTC, datetime
9
13
  from pathlib import Path
10
14
 
15
+ from celery import current_task
11
16
  from django.conf import settings
12
17
  from django.contrib.auth.models import User
13
18
  from django.core.cache import cache
14
19
  from django.core.exceptions import ImproperlyConfigured
15
20
  from django.db import transaction
21
+ from django.db.models import TextField, Value
22
+ from django.db.models.functions import Concat
23
+ from django.utils import timezone
16
24
  from opaque_keys import InvalidKeyError
17
25
  from opaque_keys.edx.keys import LearningContextKey
18
26
  from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
@@ -42,7 +50,11 @@ from ol_openedx_git_auto_export.constants import (
42
50
  ContentType,
43
51
  )
44
52
  from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
45
- from ol_openedx_git_auto_export.models import ContentGitRepository
53
+ from ol_openedx_git_auto_export.models import (
54
+ ContentGitExportLog,
55
+ ContentGitRepository,
56
+ ExportStatus,
57
+ )
46
58
 
47
59
  log = logging.getLogger(__name__)
48
60
 
@@ -600,3 +612,181 @@ def is_auto_repo_creation_enabled(is_library=False): # noqa: FBT002
600
612
  raise ImproperlyConfigured(error_msg)
601
613
 
602
614
  return True
615
+
616
+
617
+ def record_export_result( # noqa: PLR0913
618
+ content_key,
619
+ status,
620
+ *,
621
+ user=None,
622
+ git_url="",
623
+ error_message="",
624
+ git_output="",
625
+ exc=None,
626
+ ):
627
+ """
628
+ Store the result of an export run: on the row track_export started for
629
+ this task, else on a new ContentGitExportLog row.
630
+
631
+ exc, if given, appends its traceback; an empty error_message becomes
632
+ repr(exc).
633
+
634
+ Never raises: a failed write must not change the export's own outcome.
635
+ """
636
+ try:
637
+ if exc is not None:
638
+ trace = "".join(traceback.format_exception(exc))
639
+ error_message = f"{error_message or repr(exc)}\n\n{trace}"
640
+ task_id = _task_id()
641
+ fields = {
642
+ "status": status,
643
+ "user": user or "",
644
+ "celery_task_id": task_id,
645
+ "git_url": redact_credentials(git_url or ""),
646
+ "error_message": redact_credentials(error_message),
647
+ # The tail holds git's actual error.
648
+ "git_output": redact_credentials(git_output)[-20000:],
649
+ }
650
+ run = _tracked_run.get()
651
+ # An eagerly run nested task (repo creation) has its own id and row.
652
+ if run is not None and run.celery_task_id == task_id:
653
+ _tracked_run.set(None)
654
+ if not git_url:
655
+ del fields["git_url"] # keep the one track_export stored
656
+ ContentGitExportLog.objects.filter(pk=run.pk).update(
657
+ **fields, modified=timezone.now()
658
+ )
659
+ else:
660
+ ContentGitExportLog.objects.create(
661
+ content_key=LearningContextKey.from_string(str(content_key)), **fields
662
+ )
663
+ except Exception:
664
+ log.exception("Failed to record %s export result for %s", status, content_key)
665
+
666
+
667
+ # export_to_git requires user:password in an http(s) URL; git's errors and
668
+ # CalledProcessError's message repeat the URL.
669
+ # Greedy, so a password containing "@" is covered up to the host.
670
+ _URL_CREDENTIALS = re.compile(r"(?<=://)[^/\s]+@")
671
+
672
+
673
+ def redact_credentials(text):
674
+ """Replace the user:password of any URL in text with ***."""
675
+ return _URL_CREDENTIALS.sub("***@", text)
676
+
677
+
678
+ def close_dead_runs(content_key):
679
+ """
680
+ Mark the content's in_progress rows failed. Call only while holding its
681
+ export lock: the lock outlives EXPORT_TIME_LIMIT, so their runs are dead.
682
+ """
683
+ try:
684
+ ContentGitExportLog.objects.filter(
685
+ content_key=LearningContextKey.from_string(str(content_key)),
686
+ status=ExportStatus.IN_PROGRESS,
687
+ ).update(
688
+ status=ExportStatus.FAILED,
689
+ error_message=(
690
+ "The worker stopped mid-run; its last log line is the step it was in."
691
+ ),
692
+ modified=timezone.now(),
693
+ )
694
+ except Exception:
695
+ log.exception("Failed to close dead export runs for %s", content_key)
696
+
697
+
698
+ def _task_id():
699
+ # A direct (non-Celery) call has a request with no id.
700
+ return getattr(getattr(current_task, "request", None), "id", None) or ""
701
+
702
+
703
+ # The in_progress row of the export running in this context, until its result.
704
+ _tracked_run = ContextVar("tracked_export_run", default=None)
705
+
706
+
707
+ class _RunLogHandler(logging.Handler):
708
+ """Append each log line to a run's row as it is logged."""
709
+
710
+ def __init__(self, pk):
711
+ super().__init__()
712
+ self.pk = pk
713
+
714
+ def emit(self, record):
715
+ try:
716
+ when = datetime.fromtimestamp(record.created, tz=UTC)
717
+ message = redact_credentials(record.getMessage())[:2000]
718
+ line = f"{when:%H:%M:%SZ} {record.levelname} {message}\n"
719
+ ContentGitExportLog.objects.filter(pk=self.pk).update(
720
+ log=Concat("log", Value(line), output_field=TextField()),
721
+ modified=timezone.now(),
722
+ )
723
+ except Exception: # noqa: BLE001
724
+ self.handleError(record)
725
+
726
+
727
+ @contextmanager
728
+ def track_export(content_key, user, *loggers, git_url=""):
729
+ """
730
+ Record an export as in_progress before it runs, with what loggers (and
731
+ this module) log during it, until record_export_result stores its result.
732
+
733
+ A run killed mid-export keeps in_progress, and its last log line is the
734
+ last step it started.
735
+ """
736
+ run = None
737
+ try:
738
+ run = ContentGitExportLog.objects.create(
739
+ content_key=LearningContextKey.from_string(str(content_key)),
740
+ status=ExportStatus.IN_PROGRESS,
741
+ user=user or "",
742
+ celery_task_id=_task_id(),
743
+ git_url=redact_credentials(git_url or ""),
744
+ )
745
+ except InvalidKeyError:
746
+ pass # unstorable; _export logs it
747
+ except Exception:
748
+ log.exception("Failed to record the start of the %s export", content_key)
749
+ if run is None:
750
+ yield
751
+ return
752
+
753
+ handler = _RunLogHandler(run.pk)
754
+ handler.setLevel(logging.INFO) # DEBUG lines hold full commands and output
755
+ loggers = (*loggers, log)
756
+ # shortcut: handlers are process-wide, fine for prefork workers (one task
757
+ # per process); a pool running tasks concurrently in one process would
758
+ # need a per-run filter.
759
+ levels = {logger: logger.level for logger in loggers}
760
+ for logger in loggers:
761
+ logger.addHandler(handler)
762
+ # The trail needs INFO lines, whatever level the worker logs at.
763
+ if logger.getEffectiveLevel() > logging.INFO:
764
+ logger.setLevel(logging.INFO)
765
+ token = _tracked_run.set(run)
766
+ try:
767
+ yield
768
+ except BaseException as exc:
769
+ # e.g. SystemExit, which export code doesn't catch or record.
770
+ if _tracked_run.get() is run:
771
+ record_export_result(content_key, ExportStatus.FAILED, user=user, exc=exc)
772
+ raise
773
+ finally:
774
+ _tracked_run.reset(token)
775
+ for logger, level in levels.items():
776
+ logger.removeHandler(handler)
777
+ logger.setLevel(level)
778
+ try:
779
+ # Still in_progress: the result's write failed (see the log) or
780
+ # a path skipped record_export_result. Either way, not killed.
781
+ ContentGitExportLog.objects.filter(
782
+ pk=run.pk, status=ExportStatus.IN_PROGRESS
783
+ ).update(
784
+ status=ExportStatus.FAILED,
785
+ error_message=(
786
+ "No result was recorded for this run; see its log or the "
787
+ "worker log."
788
+ ),
789
+ modified=timezone.now(),
790
+ )
791
+ except Exception:
792
+ log.exception("Failed to close the %s export record", content_key)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ol-openedx-git-auto-export"
3
- version = "0.12.0"
3
+ version = "0.13.0"
4
4
  description = "A plugin that auto saves the course OLX to git when an author publishes it"
5
5
  authors = [{ name = "MIT Office of Digital Learning" }]
6
6
  license = "BSD-3-Clause"
@@ -1,28 +0,0 @@
1
- """
2
- Django models for the git auto-export plugin.
3
- """
4
-
5
- from django.db import models
6
- from model_utils.models import TimeStampedModel
7
- from opaque_keys.edx.django.models import LearningContextKeyField
8
-
9
-
10
- class ContentGitRepository(TimeStampedModel):
11
- """
12
- Model to store Git repository information for courses and libraries.
13
-
14
- This model uses LearningContextKeyField which supports both CourseKey
15
- and LibraryLocator, making it suitable for both courses and libraries.
16
- """
17
-
18
- content_key = LearningContextKeyField(max_length=255, primary_key=True)
19
- git_url = models.CharField(max_length=255)
20
- is_export_enabled = models.BooleanField(default=True)
21
-
22
- class Meta:
23
- verbose_name = "Content Git Repository"
24
- verbose_name_plural = "Content Git Repositories"
25
- ordering = ["-created"]
26
-
27
- def __str__(self):
28
- return f"ContentGitRepository (content_key={self.content_key})"