ol-openedx-git-auto-export 0.12.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/PKG-INFO +26 -1
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/README.rst +25 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/admin.py +56 -2
- ol_openedx_git_auto_export-0.13.0/ol_openedx_git_auto_export/migrations/0003_contentgitexportlog.py +88 -0
- ol_openedx_git_auto_export-0.13.0/ol_openedx_git_auto_export/models.py +73 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/tasks.py +208 -35
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/utils.py +191 -1
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/pyproject.toml +1 -1
- ol_openedx_git_auto_export-0.12.0/ol_openedx_git_auto_export/models.py +0 -28
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/.gitignore +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/CHANGELOG.rst +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/LICENSE +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/app.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/constants.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/exceptions.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/export_to_git.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/0001_initial.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/0002_contentgitrepository.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/common.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/settings/production.py +0 -0
- {ol_openedx_git_auto_export-0.12.0 → ol_openedx_git_auto_export-0.13.0}/ol_openedx_git_auto_export/signals.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ol-openedx-git-auto-export
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: A plugin that auto saves the course OLX to git when an author publishes it
|
|
5
5
|
Author: MIT Office of Digital Learning
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -146,6 +146,31 @@ Library v2 (Content Libraries)
|
|
|
146
146
|
- Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
|
|
147
147
|
- Both library types support automatic git export on updates once configured
|
|
148
148
|
|
|
149
|
+
Export History
|
|
150
|
+
==============
|
|
151
|
+
|
|
152
|
+
Every export that runs adds a row to **Content Git Export Logs** in Django
|
|
153
|
+
admin (``/admin/ol_openedx_git_auto_export/contentgitexportlog/``): its
|
|
154
|
+
status, author, Celery task id, a log of its steps with UTC times (each git
|
|
155
|
+
command and the content export, as they start), and, on failure, the error
|
|
156
|
+
with its traceback and git's own output. Failed GitHub repository creations
|
|
157
|
+
are recorded there too. The Content Git Repositories list shows each row's
|
|
158
|
+
latest status and time. The user and password in an ``http(s)`` git URL are
|
|
159
|
+
stored as ``***``.
|
|
160
|
+
|
|
161
|
+
The row is created as In progress when the export starts and updated as it
|
|
162
|
+
goes. A worker killed mid-export (out of memory, the time limit, a restart)
|
|
163
|
+
leaves it In progress until the next export of the same content marks it
|
|
164
|
+
Failed. Until then, a "modified" time more than 10 minutes old (the export time
|
|
165
|
+
limit) means the run is dead. The last line of its log is the last step it
|
|
166
|
+
started; an empty log means it died before the export itself began, while
|
|
167
|
+
loading the content.
|
|
168
|
+
|
|
169
|
+
Runs that are merged into a later one, or wait for another export of the same
|
|
170
|
+
content, add no row; the run that finally exports does. A wait that cannot
|
|
171
|
+
re-queue itself is recorded as Failed. Rows are kept indefinitely; only
|
|
172
|
+
superusers can delete them from the admin.
|
|
173
|
+
|
|
149
174
|
Re-exporting Without a Publish
|
|
150
175
|
==============================
|
|
151
176
|
|
|
@@ -130,6 +130,31 @@ Library v2 (Content Libraries)
|
|
|
130
130
|
- Library v2 uses the new event architecture and supports both creation and update signals for immediate repository creation
|
|
131
131
|
- Both library types support automatic git export on updates once configured
|
|
132
132
|
|
|
133
|
+
Export History
|
|
134
|
+
==============
|
|
135
|
+
|
|
136
|
+
Every export that runs adds a row to **Content Git Export Logs** in Django
|
|
137
|
+
admin (``/admin/ol_openedx_git_auto_export/contentgitexportlog/``): its
|
|
138
|
+
status, author, Celery task id, a log of its steps with UTC times (each git
|
|
139
|
+
command and the content export, as they start), and, on failure, the error
|
|
140
|
+
with its traceback and git's own output. Failed GitHub repository creations
|
|
141
|
+
are recorded there too. The Content Git Repositories list shows each row's
|
|
142
|
+
latest status and time. The user and password in an ``http(s)`` git URL are
|
|
143
|
+
stored as ``***``.
|
|
144
|
+
|
|
145
|
+
The row is created as In progress when the export starts and updated as it
|
|
146
|
+
goes. A worker killed mid-export (out of memory, the time limit, a restart)
|
|
147
|
+
leaves it In progress until the next export of the same content marks it
|
|
148
|
+
Failed. Until then, a "modified" time more than 10 minutes old (the export time
|
|
149
|
+
limit) means the run is dead. The last line of its log is the last step it
|
|
150
|
+
started; an empty log means it died before the export itself began, while
|
|
151
|
+
loading the content.
|
|
152
|
+
|
|
153
|
+
Runs that are merged into a later one, or wait for another export of the same
|
|
154
|
+
content, add no row; the run that finally exports does. A wait that cannot
|
|
155
|
+
re-queue itself is recorded as Failed. Rows are kept indefinitely; only
|
|
156
|
+
superusers can delete them from the admin.
|
|
157
|
+
|
|
133
158
|
Re-exporting Without a Publish
|
|
134
159
|
==============================
|
|
135
160
|
|
|
@@ -6,7 +6,7 @@ from collections import Counter
|
|
|
6
6
|
|
|
7
7
|
from django.contrib import admin, messages
|
|
8
8
|
from django.contrib.admin import SimpleListFilter
|
|
9
|
-
from django.db.models import Q
|
|
9
|
+
from django.db.models import OuterRef, Q, Subquery
|
|
10
10
|
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
|
|
11
11
|
|
|
12
12
|
from ol_openedx_git_auto_export.constants import (
|
|
@@ -14,7 +14,11 @@ from ol_openedx_git_auto_export.constants import (
|
|
|
14
14
|
LIBRARY_V2_PREFIX,
|
|
15
15
|
ContentType,
|
|
16
16
|
)
|
|
17
|
-
from ol_openedx_git_auto_export.models import
|
|
17
|
+
from ol_openedx_git_auto_export.models import (
|
|
18
|
+
ContentGitExportLog,
|
|
19
|
+
ContentGitRepository,
|
|
20
|
+
ExportStatus,
|
|
21
|
+
)
|
|
18
22
|
from ol_openedx_git_auto_export.utils import queue_git_exports
|
|
19
23
|
|
|
20
24
|
|
|
@@ -60,12 +64,39 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
|
|
|
60
64
|
"git_url",
|
|
61
65
|
"is_export_enabled",
|
|
62
66
|
"content_type_display",
|
|
67
|
+
"last_export_status",
|
|
68
|
+
"last_export_at",
|
|
63
69
|
)
|
|
64
70
|
search_fields = ("content_key", "git_url")
|
|
65
71
|
list_filter = ("is_export_enabled", ContentTypeFilter)
|
|
66
72
|
list_per_page = 50
|
|
67
73
|
actions = ("export_to_git",)
|
|
68
74
|
|
|
75
|
+
def get_queryset(self, request):
|
|
76
|
+
"""Annotate each row with its newest export log entry."""
|
|
77
|
+
newest = ContentGitExportLog.objects.filter(
|
|
78
|
+
content_key=OuterRef("content_key")
|
|
79
|
+
).order_by("-created", "-id")
|
|
80
|
+
return (
|
|
81
|
+
super()
|
|
82
|
+
.get_queryset(request)
|
|
83
|
+
.annotate(
|
|
84
|
+
last_status=Subquery(newest.values("status")[:1]),
|
|
85
|
+
last_at=Subquery(newest.values("created")[:1]),
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
@admin.display(description="Last export", ordering="last_status")
|
|
90
|
+
def last_export_status(self, obj):
|
|
91
|
+
"""Status of the newest export log row, if any."""
|
|
92
|
+
# .get: a rolled-back release may not know a newer status.
|
|
93
|
+
return dict(ExportStatus.choices).get(obj.last_status, obj.last_status or "-")
|
|
94
|
+
|
|
95
|
+
@admin.display(description="Last export at", ordering="last_at")
|
|
96
|
+
def last_export_at(self, obj):
|
|
97
|
+
"""Time of the newest export log row, if any."""
|
|
98
|
+
return obj.last_at
|
|
99
|
+
|
|
69
100
|
@admin.display(description="Content Type")
|
|
70
101
|
def content_type_display(self, obj):
|
|
71
102
|
"""Display whether the content is a course or library."""
|
|
@@ -102,3 +133,26 @@ class ContentGitRepositoryAdmin(admin.ModelAdmin):
|
|
|
102
133
|
To stop exporting content, set `is_export_enabled` to False
|
|
103
134
|
"""
|
|
104
135
|
return False
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@admin.register(ContentGitExportLog)
|
|
139
|
+
class ContentGitExportLogAdmin(admin.ModelAdmin):
|
|
140
|
+
"""Read-only history of git export runs."""
|
|
141
|
+
|
|
142
|
+
# In progress with no activity past EXPORT_TIME_LIMIT: the run was killed.
|
|
143
|
+
list_display = ("content_key", "status", "user", "created", "modified", "git_url")
|
|
144
|
+
list_filter = ("status", ContentTypeFilter)
|
|
145
|
+
search_fields = ("content_key",)
|
|
146
|
+
list_per_page = 50
|
|
147
|
+
|
|
148
|
+
def has_add_permission(self, request): # noqa: ARG002
|
|
149
|
+
"""Rows are written only by the export tasks."""
|
|
150
|
+
return False
|
|
151
|
+
|
|
152
|
+
def has_change_permission(self, request, obj=None): # noqa: ARG002
|
|
153
|
+
"""Rows are an audit trail; staff can view but not edit them."""
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
def has_delete_permission(self, request, obj=None): # noqa: ARG002
|
|
157
|
+
"""Superusers only, e.g. to remove a row holding a leaked secret."""
|
|
158
|
+
return request.user.is_superuser
|
ol_openedx_git_auto_export-0.13.0/ol_openedx_git_auto_export/migrations/0003_contentgitexportlog.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Generated by Django 5.2.18 on 2026-11-04 12:42
|
|
2
|
+
|
|
3
|
+
import django.utils.timezone
|
|
4
|
+
import model_utils.fields
|
|
5
|
+
import opaque_keys.edx.django.models
|
|
6
|
+
from django.db import migrations, models
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Migration(migrations.Migration):
|
|
10
|
+
dependencies = [
|
|
11
|
+
("ol_openedx_git_auto_export", "0002_contentgitrepository"),
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
operations = [
|
|
15
|
+
migrations.CreateModel(
|
|
16
|
+
name="ContentGitExportLog",
|
|
17
|
+
fields=[
|
|
18
|
+
(
|
|
19
|
+
"id",
|
|
20
|
+
models.AutoField(
|
|
21
|
+
auto_created=True,
|
|
22
|
+
primary_key=True,
|
|
23
|
+
serialize=False,
|
|
24
|
+
verbose_name="ID",
|
|
25
|
+
),
|
|
26
|
+
),
|
|
27
|
+
(
|
|
28
|
+
"created",
|
|
29
|
+
model_utils.fields.AutoCreatedField(
|
|
30
|
+
default=django.utils.timezone.now,
|
|
31
|
+
editable=False,
|
|
32
|
+
verbose_name="created",
|
|
33
|
+
),
|
|
34
|
+
),
|
|
35
|
+
(
|
|
36
|
+
"modified",
|
|
37
|
+
model_utils.fields.AutoLastModifiedField(
|
|
38
|
+
default=django.utils.timezone.now,
|
|
39
|
+
editable=False,
|
|
40
|
+
verbose_name="modified",
|
|
41
|
+
),
|
|
42
|
+
),
|
|
43
|
+
(
|
|
44
|
+
"content_key",
|
|
45
|
+
opaque_keys.edx.django.models.LearningContextKeyField(
|
|
46
|
+
max_length=255
|
|
47
|
+
),
|
|
48
|
+
),
|
|
49
|
+
(
|
|
50
|
+
"status",
|
|
51
|
+
models.CharField(
|
|
52
|
+
choices=[
|
|
53
|
+
("in_progress", "In progress"),
|
|
54
|
+
("success", "Success"),
|
|
55
|
+
("no_changes", "No changes"),
|
|
56
|
+
("skipped_disabled", "Skipped: export disabled"),
|
|
57
|
+
("not_found", "Content not found"),
|
|
58
|
+
("no_repo", "No repository"),
|
|
59
|
+
("repo_creation_failed", "Repository creation failed"),
|
|
60
|
+
("failed", "Failed"),
|
|
61
|
+
],
|
|
62
|
+
max_length=32,
|
|
63
|
+
),
|
|
64
|
+
),
|
|
65
|
+
("user", models.CharField(blank=True, max_length=150)),
|
|
66
|
+
("celery_task_id", models.CharField(blank=True, max_length=255)),
|
|
67
|
+
("git_url", models.CharField(blank=True, max_length=255)),
|
|
68
|
+
("error_message", models.TextField(blank=True)),
|
|
69
|
+
("git_output", models.TextField(blank=True)),
|
|
70
|
+
("log", models.TextField(blank=True)),
|
|
71
|
+
],
|
|
72
|
+
options={
|
|
73
|
+
"verbose_name": "Content Git Export Log",
|
|
74
|
+
"verbose_name_plural": "Content Git Export Logs",
|
|
75
|
+
"ordering": ["-created"],
|
|
76
|
+
"indexes": [
|
|
77
|
+
models.Index(
|
|
78
|
+
fields=["content_key", "-created"],
|
|
79
|
+
name="ol_openedx__content_bcf928_idx",
|
|
80
|
+
),
|
|
81
|
+
models.Index(
|
|
82
|
+
fields=["-created", "-id"],
|
|
83
|
+
name="ol_openedx__created_1d4ac7_idx",
|
|
84
|
+
),
|
|
85
|
+
],
|
|
86
|
+
},
|
|
87
|
+
),
|
|
88
|
+
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Django models for the git auto-export plugin.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.db import models
|
|
6
|
+
from model_utils.models import TimeStampedModel
|
|
7
|
+
from opaque_keys.edx.django.models import LearningContextKeyField
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ContentGitRepository(TimeStampedModel):
|
|
11
|
+
"""
|
|
12
|
+
Model to store Git repository information for courses and libraries.
|
|
13
|
+
|
|
14
|
+
This model uses LearningContextKeyField which supports both CourseKey
|
|
15
|
+
and LibraryLocator, making it suitable for both courses and libraries.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
content_key = LearningContextKeyField(max_length=255, primary_key=True)
|
|
19
|
+
git_url = models.CharField(max_length=255)
|
|
20
|
+
is_export_enabled = models.BooleanField(default=True)
|
|
21
|
+
|
|
22
|
+
class Meta:
|
|
23
|
+
verbose_name = "Content Git Repository"
|
|
24
|
+
verbose_name_plural = "Content Git Repositories"
|
|
25
|
+
ordering = ["-created"]
|
|
26
|
+
|
|
27
|
+
def __str__(self):
|
|
28
|
+
return f"ContentGitRepository (content_key={self.content_key})"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ExportStatus(models.TextChoices):
|
|
32
|
+
"""Outcome of one export run; in_progress until it finishes."""
|
|
33
|
+
|
|
34
|
+
# Left as is by a run killed mid-export (OOM, time limit, pod restart).
|
|
35
|
+
IN_PROGRESS = "in_progress", "In progress"
|
|
36
|
+
SUCCESS = "success", "Success"
|
|
37
|
+
NO_CHANGES = "no_changes", "No changes"
|
|
38
|
+
SKIPPED_DISABLED = "skipped_disabled", "Skipped: export disabled"
|
|
39
|
+
NOT_FOUND = "not_found", "Content not found"
|
|
40
|
+
NO_REPO = "no_repo", "No repository"
|
|
41
|
+
REPO_CREATION_FAILED = "repo_creation_failed", "Repository creation failed"
|
|
42
|
+
FAILED = "failed", "Failed"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ContentGitExportLog(TimeStampedModel):
|
|
46
|
+
"""
|
|
47
|
+
One row per export run, kept so intermittent failures show as a pattern.
|
|
48
|
+
The newest row per content_key is its current status.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
# Not a FK: not_found and no_repo rows have no ContentGitRepository.
|
|
52
|
+
content_key = LearningContextKeyField(max_length=255)
|
|
53
|
+
status = models.CharField(max_length=32, choices=ExportStatus.choices)
|
|
54
|
+
user = models.CharField(max_length=150, blank=True)
|
|
55
|
+
celery_task_id = models.CharField(max_length=255, blank=True)
|
|
56
|
+
git_url = models.CharField(max_length=255, blank=True)
|
|
57
|
+
error_message = models.TextField(blank=True)
|
|
58
|
+
git_output = models.TextField(blank=True)
|
|
59
|
+
# Timestamped log lines, written as the run goes.
|
|
60
|
+
log = models.TextField(blank=True)
|
|
61
|
+
|
|
62
|
+
class Meta:
|
|
63
|
+
verbose_name = "Content Git Export Log"
|
|
64
|
+
verbose_name_plural = "Content Git Export Logs"
|
|
65
|
+
ordering = ["-created"]
|
|
66
|
+
indexes = [
|
|
67
|
+
models.Index(fields=["content_key", "-created"]),
|
|
68
|
+
# The admin's unfiltered, newest-first list.
|
|
69
|
+
models.Index(fields=["-created", "-id"]),
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
def __str__(self):
|
|
73
|
+
return f"ContentGitExportLog ({self.content_key}: {self.status})"
|
|
@@ -5,9 +5,12 @@ This module defines background tasks that handle the actual export of course
|
|
|
5
5
|
content to Git repositories using Celery for asynchronous processing.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
|
|
8
10
|
import requests
|
|
9
11
|
from celery import shared_task # pylint: disable=import-error
|
|
10
12
|
from celery.utils.log import get_task_logger
|
|
13
|
+
from cms.djangoapps.contentstore import git_export_utils
|
|
11
14
|
from cms.djangoapps.contentstore.git_export_utils import GitExportError, export_to_git
|
|
12
15
|
from django.conf import settings
|
|
13
16
|
from opaque_keys import InvalidKeyError
|
|
@@ -16,21 +19,24 @@ from rest_framework import status
|
|
|
16
19
|
|
|
17
20
|
from ol_openedx_git_auto_export.constants import EXPORT_TIME_LIMIT
|
|
18
21
|
from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
|
|
19
|
-
from ol_openedx_git_auto_export.models import ContentGitRepository
|
|
22
|
+
from ol_openedx_git_auto_export.models import ContentGitRepository, ExportStatus
|
|
20
23
|
from ol_openedx_git_auto_export.utils import (
|
|
21
24
|
acquire_export_lock,
|
|
22
25
|
burst_exceeded_max_wait,
|
|
23
26
|
claim_export_slot,
|
|
24
27
|
clear_stale_git_lock,
|
|
28
|
+
close_dead_runs,
|
|
25
29
|
current_export_token,
|
|
26
30
|
get_content_info,
|
|
27
31
|
github_repo_name_format,
|
|
28
32
|
is_auto_repo_creation_enabled,
|
|
29
33
|
queue_debounced_export,
|
|
30
34
|
queue_export_task,
|
|
35
|
+
record_export_result,
|
|
31
36
|
release_export_lock,
|
|
32
37
|
release_export_slot,
|
|
33
38
|
schedule_export_with_debounce,
|
|
39
|
+
track_export,
|
|
34
40
|
)
|
|
35
41
|
|
|
36
42
|
LOGGER = get_task_logger(__name__)
|
|
@@ -79,23 +85,82 @@ def _superseded(context_key_string, user, token):
|
|
|
79
85
|
return True
|
|
80
86
|
|
|
81
87
|
|
|
88
|
+
# export_to_git's steps, by the module-level name it calls them through.
|
|
89
|
+
# Releases before Willow have only cmd_log and export_course_to_xml.
|
|
90
|
+
_EXPORT_STEPS = {
|
|
91
|
+
# The args may hold the URL, so only "git <command>".
|
|
92
|
+
"cmd_log": lambda cmd, *_: f"Running {' '.join(cmd[:2])}",
|
|
93
|
+
**dict.fromkeys(
|
|
94
|
+
(
|
|
95
|
+
"export_course_to_xml",
|
|
96
|
+
"export_library_to_xml",
|
|
97
|
+
"extract_library_v2_zip_to_dir",
|
|
98
|
+
),
|
|
99
|
+
lambda *_: "Exporting the content",
|
|
100
|
+
),
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _logged(func, describe):
|
|
105
|
+
def step(*args, **kwargs):
|
|
106
|
+
LOGGER.info(describe(*args))
|
|
107
|
+
return func(*args, **kwargs)
|
|
108
|
+
|
|
109
|
+
return step
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@contextmanager
|
|
113
|
+
def _log_export_steps():
|
|
114
|
+
"""
|
|
115
|
+
Log each step export_to_git takes as it starts.
|
|
116
|
+
|
|
117
|
+
Upstream logs git commands only after they end, at DEBUG, and the content
|
|
118
|
+
export only if it fails, so a run killed mid-push or mid-export would
|
|
119
|
+
show an earlier step as its last.
|
|
120
|
+
"""
|
|
121
|
+
originals = {
|
|
122
|
+
name: getattr(git_export_utils, name)
|
|
123
|
+
for name in _EXPORT_STEPS
|
|
124
|
+
if hasattr(git_export_utils, name)
|
|
125
|
+
}
|
|
126
|
+
# shortcut: patches the module for the whole process, fine for prefork
|
|
127
|
+
# workers (one task per process); a pool running tasks concurrently in
|
|
128
|
+
# one process would need a lock.
|
|
129
|
+
for name, original in originals.items():
|
|
130
|
+
setattr(git_export_utils, name, _logged(original, _EXPORT_STEPS[name]))
|
|
131
|
+
try:
|
|
132
|
+
yield
|
|
133
|
+
finally:
|
|
134
|
+
for name, original in originals.items():
|
|
135
|
+
setattr(git_export_utils, name, original)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _git_output(exc):
|
|
139
|
+
"""Return git's own output from a GitExportError, or ''."""
|
|
140
|
+
output = getattr(exc.__cause__, "output", None) or ""
|
|
141
|
+
# bytes today (check_output without text=True); str if that changes.
|
|
142
|
+
if isinstance(output, bytes):
|
|
143
|
+
output = output.decode(errors="replace")
|
|
144
|
+
return output
|
|
145
|
+
|
|
146
|
+
|
|
82
147
|
def _export_unless_unchanged(context_key, git_url, user):
|
|
83
148
|
"""
|
|
84
149
|
Run export_to_git, treating unchanged content as success.
|
|
85
150
|
|
|
86
151
|
export_to_git raises CANNOT_COMMIT both for this and for real failures
|
|
87
152
|
such as a held index.lock; only git's own output tells them apart.
|
|
153
|
+
|
|
154
|
+
Returns False if there was nothing to commit, else True.
|
|
88
155
|
"""
|
|
89
156
|
try:
|
|
90
157
|
export_to_git(context_key, git_url, user=user)
|
|
91
158
|
except GitExportError as exc:
|
|
92
|
-
|
|
93
|
-
# bytes today (check_output without text=True); str if that changes.
|
|
94
|
-
if isinstance(output, bytes):
|
|
95
|
-
output = output.decode(errors="replace")
|
|
96
|
-
if "nothing to commit" not in output:
|
|
159
|
+
if "nothing to commit" not in _git_output(exc):
|
|
97
160
|
raise
|
|
98
161
|
LOGGER.info("No changes to export for %s", context_key)
|
|
162
|
+
return False
|
|
163
|
+
return True
|
|
99
164
|
|
|
100
165
|
|
|
101
166
|
def _wait_to_export(context_key_string, user, token):
|
|
@@ -105,13 +170,26 @@ def _wait_to_export(context_key_string, user, token):
|
|
|
105
170
|
Whatever holds it may predate this task's changes, so this one runs
|
|
106
171
|
after it instead of racing it.
|
|
107
172
|
"""
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
173
|
+
try:
|
|
174
|
+
if token:
|
|
175
|
+
# Keeps the slot and the burst's start, as a superseded task does.
|
|
176
|
+
queue_export_task(
|
|
177
|
+
context_key_string,
|
|
178
|
+
user,
|
|
179
|
+
current_export_token(context_key_string, token),
|
|
180
|
+
)
|
|
181
|
+
else:
|
|
182
|
+
queue_debounced_export(context_key_string, lambda: user)
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
# The export is lost: nothing else is queued to run it.
|
|
185
|
+
record_export_result(
|
|
186
|
+
context_key_string,
|
|
187
|
+
ExportStatus.FAILED,
|
|
188
|
+
user=user,
|
|
189
|
+
error_message=f"Failed to re-queue the export: {exc!r}",
|
|
190
|
+
exc=exc,
|
|
112
191
|
)
|
|
113
|
-
|
|
114
|
-
queue_debounced_export(context_key_string, lambda: user)
|
|
192
|
+
raise
|
|
115
193
|
|
|
116
194
|
|
|
117
195
|
def _get_content_repo(context_key_string):
|
|
@@ -155,11 +233,17 @@ def async_export_to_git(context_key_string, user=None, token=None):
|
|
|
155
233
|
try:
|
|
156
234
|
content_repo = _get_content_repo(context_key_string)
|
|
157
235
|
unlocked = _index_unlocked(content_repo)
|
|
158
|
-
except Exception:
|
|
236
|
+
except Exception as exc:
|
|
159
237
|
# Still holding the slot: later signals would see an export
|
|
160
238
|
# queued that will never run.
|
|
161
239
|
if token:
|
|
162
240
|
release_export_slot(context_key_string)
|
|
241
|
+
record_export_result(
|
|
242
|
+
context_key_string,
|
|
243
|
+
ExportStatus.FAILED,
|
|
244
|
+
user=user,
|
|
245
|
+
exc=exc,
|
|
246
|
+
)
|
|
163
247
|
raise
|
|
164
248
|
if not unlocked:
|
|
165
249
|
# A killed export's lock, or Studio's own export running.
|
|
@@ -173,14 +257,26 @@ def async_export_to_git(context_key_string, user=None, token=None):
|
|
|
173
257
|
# queue a fresh task instead of being dropped. Only now, holding
|
|
174
258
|
# the lock: this also ends the burst, which a wait must not do.
|
|
175
259
|
release_export_slot(context_key_string)
|
|
176
|
-
|
|
260
|
+
if owned: # None: the cache is down, so another run may be live
|
|
261
|
+
close_dead_runs(context_key_string)
|
|
262
|
+
with (
|
|
263
|
+
track_export(
|
|
264
|
+
context_key_string,
|
|
265
|
+
user,
|
|
266
|
+
LOGGER,
|
|
267
|
+
git_export_utils.log,
|
|
268
|
+
git_url=content_repo.git_url if content_repo else "",
|
|
269
|
+
),
|
|
270
|
+
_log_export_steps(),
|
|
271
|
+
):
|
|
272
|
+
_export(context_key_string, user, content_repo)
|
|
177
273
|
finally:
|
|
178
274
|
# None means the cache was down: this export never held the lock.
|
|
179
275
|
if owned:
|
|
180
276
|
release_export_lock(context_key_string)
|
|
181
277
|
|
|
182
278
|
|
|
183
|
-
def _export(context_key_string, user, content_repo):
|
|
279
|
+
def _export(context_key_string, user, content_repo): # noqa: C901
|
|
184
280
|
"""Resolve the content, then export it to content_repo."""
|
|
185
281
|
try:
|
|
186
282
|
context_key = LearningContextKey.from_string(context_key_string)
|
|
@@ -193,10 +289,18 @@ def _export(context_key_string, user, content_repo):
|
|
|
193
289
|
"Content %s not found; abandoning this export (no retry)",
|
|
194
290
|
context_key_string,
|
|
195
291
|
)
|
|
292
|
+
record_export_result(context_key_string, ExportStatus.NOT_FOUND, user=user)
|
|
196
293
|
return
|
|
197
|
-
except Exception:
|
|
294
|
+
except Exception as exc:
|
|
198
295
|
# Also covers get_content_info's lookups, so not necessarily a bad key.
|
|
199
296
|
LOGGER.exception("Failed to resolve content %s", context_key_string)
|
|
297
|
+
if not isinstance(exc, InvalidKeyError): # unstorable key: log only
|
|
298
|
+
record_export_result(
|
|
299
|
+
context_key_string,
|
|
300
|
+
ExportStatus.FAILED,
|
|
301
|
+
user=user,
|
|
302
|
+
exc=exc,
|
|
303
|
+
)
|
|
200
304
|
raise
|
|
201
305
|
|
|
202
306
|
if content_repo is None:
|
|
@@ -206,59 +310,94 @@ def _export(context_key_string, user, content_repo):
|
|
|
206
310
|
content_info["content_type"],
|
|
207
311
|
context_key_string,
|
|
208
312
|
)
|
|
313
|
+
note = "Auto-creation is disabled."
|
|
314
|
+
queue_error = None
|
|
209
315
|
try:
|
|
210
316
|
if is_auto_repo_creation_enabled(is_library=content_info["is_library"]):
|
|
211
317
|
async_create_github_repo.delay(str(context_key), export_content=True)
|
|
212
|
-
|
|
318
|
+
note = "Repository creation queued."
|
|
319
|
+
except Exception as exc:
|
|
213
320
|
# Nothing else catches this one, and it must not fail the task.
|
|
214
321
|
LOGGER.exception(
|
|
215
322
|
"Failed to check/trigger repo creation for %s %s",
|
|
216
323
|
content_info["content_type"],
|
|
217
324
|
context_key_string,
|
|
218
325
|
)
|
|
326
|
+
note = f"Failed to queue repository creation: {exc!r}"
|
|
327
|
+
queue_error = exc
|
|
328
|
+
record_export_result(
|
|
329
|
+
context_key,
|
|
330
|
+
ExportStatus.NO_REPO,
|
|
331
|
+
user=user,
|
|
332
|
+
error_message=note,
|
|
333
|
+
exc=queue_error,
|
|
334
|
+
)
|
|
219
335
|
return
|
|
220
336
|
|
|
337
|
+
git_url = content_repo.git_url
|
|
338
|
+
content_id = (
|
|
339
|
+
content_info["content_module"].id
|
|
340
|
+
if hasattr(content_info["content_module"], "id")
|
|
341
|
+
else context_key
|
|
342
|
+
)
|
|
221
343
|
try:
|
|
222
344
|
if content_repo.is_export_enabled:
|
|
223
345
|
LOGGER.info(
|
|
224
346
|
"Starting async %s content export to git (%s id: %s)",
|
|
225
347
|
content_info["content_type"],
|
|
226
348
|
content_info["content_type"],
|
|
227
|
-
|
|
228
|
-
if hasattr(content_info["content_module"], "id")
|
|
229
|
-
else context_key,
|
|
349
|
+
content_id,
|
|
230
350
|
)
|
|
231
351
|
# _index_unlocked removed any stale index.lock; export_to_git's
|
|
232
352
|
# `git reset --hard` + `git clean` handle files a crash left dirty.
|
|
233
|
-
|
|
353
|
+
outcome = (
|
|
354
|
+
ExportStatus.SUCCESS
|
|
355
|
+
if _export_unless_unchanged(context_key, git_url, user)
|
|
356
|
+
else ExportStatus.NO_CHANGES
|
|
357
|
+
)
|
|
234
358
|
else:
|
|
235
359
|
LOGGER.info(
|
|
236
360
|
"Git export is disabled for %s %s. Skipping export.",
|
|
237
361
|
content_info["content_type"],
|
|
238
362
|
context_key_string,
|
|
239
363
|
)
|
|
240
|
-
|
|
364
|
+
outcome = ExportStatus.SKIPPED_DISABLED
|
|
365
|
+
except GitExportError as exc:
|
|
241
366
|
LOGGER.exception(
|
|
242
367
|
"Failed async %s content export to git (%s id: %s)",
|
|
243
368
|
content_info["content_type"],
|
|
244
369
|
content_info["content_type"],
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
370
|
+
content_id,
|
|
371
|
+
)
|
|
372
|
+
record_export_result(
|
|
373
|
+
context_key,
|
|
374
|
+
ExportStatus.FAILED,
|
|
375
|
+
user=user,
|
|
376
|
+
git_url=git_url,
|
|
377
|
+
error_message=str(exc),
|
|
378
|
+
git_output=_git_output(exc),
|
|
379
|
+
exc=exc,
|
|
248
380
|
)
|
|
249
381
|
# Re-raised, not retried, so Celery records the failure; most push
|
|
250
382
|
# and commit errors recur until someone fixes their cause.
|
|
251
383
|
raise
|
|
252
|
-
except Exception:
|
|
384
|
+
except Exception as exc:
|
|
253
385
|
LOGGER.exception(
|
|
254
386
|
"Unknown error occurred during async %s content export to git (%s id: %s)",
|
|
255
387
|
content_info["content_type"],
|
|
256
388
|
content_info["content_type"],
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
389
|
+
content_id,
|
|
390
|
+
)
|
|
391
|
+
record_export_result(
|
|
392
|
+
context_key,
|
|
393
|
+
ExportStatus.FAILED,
|
|
394
|
+
user=user,
|
|
395
|
+
git_url=git_url,
|
|
396
|
+
exc=exc,
|
|
260
397
|
)
|
|
261
398
|
raise
|
|
399
|
+
else:
|
|
400
|
+
record_export_result(context_key, outcome, user=user, git_url=git_url)
|
|
262
401
|
|
|
263
402
|
|
|
264
403
|
@shared_task(
|
|
@@ -266,7 +405,11 @@ def _export(context_key_string, user, content_repo):
|
|
|
266
405
|
autoretry_for=(requests.exceptions.RequestException,),
|
|
267
406
|
retry_kwargs={"max_retries": 3, "countdown": 10},
|
|
268
407
|
)
|
|
269
|
-
def async_create_github_repo(
|
|
408
|
+
def async_create_github_repo( # noqa: C901, PLR0912, PLR0915
|
|
409
|
+
self,
|
|
410
|
+
context_key_str,
|
|
411
|
+
export_content=False, # noqa: FBT002
|
|
412
|
+
):
|
|
270
413
|
"""
|
|
271
414
|
Create a GitHub repository for the given course or library key.
|
|
272
415
|
|
|
@@ -287,9 +430,18 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
287
430
|
except ContentNotFoundError as exc:
|
|
288
431
|
response_msg = f"Content {context_key_str} not found: {exc}"
|
|
289
432
|
LOGGER.warning(response_msg)
|
|
433
|
+
record_export_result(
|
|
434
|
+
context_key_str, ExportStatus.NOT_FOUND, error_message=response_msg
|
|
435
|
+
)
|
|
290
436
|
return False, response_msg
|
|
291
|
-
except Exception:
|
|
437
|
+
except Exception as exc:
|
|
292
438
|
LOGGER.exception("Failed to parse context key: %s", context_key_str)
|
|
439
|
+
if not isinstance(exc, InvalidKeyError): # unstorable key: log only
|
|
440
|
+
record_export_result(
|
|
441
|
+
context_key_str,
|
|
442
|
+
ExportStatus.REPO_CREATION_FAILED,
|
|
443
|
+
exc=exc,
|
|
444
|
+
)
|
|
293
445
|
return False, f"Invalid context key: {context_key_str}"
|
|
294
446
|
|
|
295
447
|
content_id_slugified = github_repo_name_format(str(context_key))
|
|
@@ -330,13 +482,29 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
330
482
|
"has_wiki": False,
|
|
331
483
|
"auto_init": True,
|
|
332
484
|
}
|
|
333
|
-
|
|
485
|
+
max_retries = self.retry_kwargs.get("max_retries", 3)
|
|
486
|
+
try:
|
|
487
|
+
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
|
488
|
+
# A malformed 201 body raises JSONDecodeError, also a RequestException.
|
|
489
|
+
repo_data = (
|
|
490
|
+
response.json() if response.status_code == status.HTTP_201_CREATED else None
|
|
491
|
+
)
|
|
492
|
+
except requests.exceptions.RequestException as exc:
|
|
493
|
+
# autoretry_for retries these; record only the last attempt.
|
|
494
|
+
if self.request.retries >= max_retries:
|
|
495
|
+
record_export_result(
|
|
496
|
+
context_key, ExportStatus.REPO_CREATION_FAILED, exc=exc
|
|
497
|
+
)
|
|
498
|
+
raise
|
|
334
499
|
if response.status_code != status.HTTP_201_CREATED:
|
|
335
|
-
|
|
500
|
+
try:
|
|
501
|
+
error_body = response.json()
|
|
502
|
+
except ValueError: # e.g. a proxy's HTML error page
|
|
503
|
+
error_body = response.text[:500]
|
|
504
|
+
response_msg = f"Failed to create GitHub repository for {content_info['content_type']} {context_key} (HTTP {response.status_code}): {error_body}" # noqa: E501
|
|
336
505
|
LOGGER.error(response_msg)
|
|
337
506
|
|
|
338
507
|
# Retry the task if we haven't exceeded max retries
|
|
339
|
-
max_retries = self.retry_kwargs.get("max_retries", 3)
|
|
340
508
|
if self.request.retries < max_retries:
|
|
341
509
|
LOGGER.info(
|
|
342
510
|
"Retrying GitHub repository creation for %s %s (attempt %d/%d)",
|
|
@@ -348,9 +516,11 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
348
516
|
countdown = self.retry_kwargs.get("countdown", 10)
|
|
349
517
|
raise self.retry(countdown=countdown, exc=Exception(response_msg))
|
|
350
518
|
|
|
519
|
+
record_export_result(
|
|
520
|
+
context_key, ExportStatus.REPO_CREATION_FAILED, error_message=response_msg
|
|
521
|
+
)
|
|
351
522
|
return False, response_msg
|
|
352
523
|
|
|
353
|
-
repo_data = response.json()
|
|
354
524
|
ssh_url = repo_data.get("ssh_url")
|
|
355
525
|
if ssh_url:
|
|
356
526
|
# Use the new ContentGitRepository model
|
|
@@ -371,6 +541,9 @@ def async_create_github_repo(self, context_key_str, export_content=False): # no
|
|
|
371
541
|
Response data: {repo_data}
|
|
372
542
|
"""
|
|
373
543
|
LOGGER.error(response_msg)
|
|
544
|
+
record_export_result(
|
|
545
|
+
context_key, ExportStatus.REPO_CREATION_FAILED, error_message=response_msg
|
|
546
|
+
)
|
|
374
547
|
|
|
375
548
|
if ssh_url and export_content:
|
|
376
549
|
# Debounced like every other export, so it coalesces with an import
|
|
@@ -5,14 +5,22 @@ Utility functions for the ol_openedx_git_auto_export app.
|
|
|
5
5
|
import logging
|
|
6
6
|
import re
|
|
7
7
|
import time
|
|
8
|
+
import traceback
|
|
8
9
|
import uuid
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from contextvars import ContextVar
|
|
12
|
+
from datetime import UTC, datetime
|
|
9
13
|
from pathlib import Path
|
|
10
14
|
|
|
15
|
+
from celery import current_task
|
|
11
16
|
from django.conf import settings
|
|
12
17
|
from django.contrib.auth.models import User
|
|
13
18
|
from django.core.cache import cache
|
|
14
19
|
from django.core.exceptions import ImproperlyConfigured
|
|
15
20
|
from django.db import transaction
|
|
21
|
+
from django.db.models import TextField, Value
|
|
22
|
+
from django.db.models.functions import Concat
|
|
23
|
+
from django.utils import timezone
|
|
16
24
|
from opaque_keys import InvalidKeyError
|
|
17
25
|
from opaque_keys.edx.keys import LearningContextKey
|
|
18
26
|
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
|
|
@@ -42,7 +50,11 @@ from ol_openedx_git_auto_export.constants import (
|
|
|
42
50
|
ContentType,
|
|
43
51
|
)
|
|
44
52
|
from ol_openedx_git_auto_export.exceptions import ContentNotFoundError
|
|
45
|
-
from ol_openedx_git_auto_export.models import
|
|
53
|
+
from ol_openedx_git_auto_export.models import (
|
|
54
|
+
ContentGitExportLog,
|
|
55
|
+
ContentGitRepository,
|
|
56
|
+
ExportStatus,
|
|
57
|
+
)
|
|
46
58
|
|
|
47
59
|
log = logging.getLogger(__name__)
|
|
48
60
|
|
|
@@ -600,3 +612,181 @@ def is_auto_repo_creation_enabled(is_library=False): # noqa: FBT002
|
|
|
600
612
|
raise ImproperlyConfigured(error_msg)
|
|
601
613
|
|
|
602
614
|
return True
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def record_export_result( # noqa: PLR0913
|
|
618
|
+
content_key,
|
|
619
|
+
status,
|
|
620
|
+
*,
|
|
621
|
+
user=None,
|
|
622
|
+
git_url="",
|
|
623
|
+
error_message="",
|
|
624
|
+
git_output="",
|
|
625
|
+
exc=None,
|
|
626
|
+
):
|
|
627
|
+
"""
|
|
628
|
+
Store the result of an export run: on the row track_export started for
|
|
629
|
+
this task, else on a new ContentGitExportLog row.
|
|
630
|
+
|
|
631
|
+
exc, if given, appends its traceback; an empty error_message becomes
|
|
632
|
+
repr(exc).
|
|
633
|
+
|
|
634
|
+
Never raises: a failed write must not change the export's own outcome.
|
|
635
|
+
"""
|
|
636
|
+
try:
|
|
637
|
+
if exc is not None:
|
|
638
|
+
trace = "".join(traceback.format_exception(exc))
|
|
639
|
+
error_message = f"{error_message or repr(exc)}\n\n{trace}"
|
|
640
|
+
task_id = _task_id()
|
|
641
|
+
fields = {
|
|
642
|
+
"status": status,
|
|
643
|
+
"user": user or "",
|
|
644
|
+
"celery_task_id": task_id,
|
|
645
|
+
"git_url": redact_credentials(git_url or ""),
|
|
646
|
+
"error_message": redact_credentials(error_message),
|
|
647
|
+
# The tail holds git's actual error.
|
|
648
|
+
"git_output": redact_credentials(git_output)[-20000:],
|
|
649
|
+
}
|
|
650
|
+
run = _tracked_run.get()
|
|
651
|
+
# An eagerly run nested task (repo creation) has its own id and row.
|
|
652
|
+
if run is not None and run.celery_task_id == task_id:
|
|
653
|
+
_tracked_run.set(None)
|
|
654
|
+
if not git_url:
|
|
655
|
+
del fields["git_url"] # keep the one track_export stored
|
|
656
|
+
ContentGitExportLog.objects.filter(pk=run.pk).update(
|
|
657
|
+
**fields, modified=timezone.now()
|
|
658
|
+
)
|
|
659
|
+
else:
|
|
660
|
+
ContentGitExportLog.objects.create(
|
|
661
|
+
content_key=LearningContextKey.from_string(str(content_key)), **fields
|
|
662
|
+
)
|
|
663
|
+
except Exception:
|
|
664
|
+
log.exception("Failed to record %s export result for %s", status, content_key)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
# export_to_git requires user:password in an http(s) URL; git's errors and
|
|
668
|
+
# CalledProcessError's message repeat the URL.
|
|
669
|
+
# Greedy, so a password containing "@" is covered up to the host.
|
|
670
|
+
_URL_CREDENTIALS = re.compile(r"(?<=://)[^/\s]+@")
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def redact_credentials(text):
|
|
674
|
+
"""Replace the user:password of any URL in text with ***."""
|
|
675
|
+
return _URL_CREDENTIALS.sub("***@", text)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def close_dead_runs(content_key):
|
|
679
|
+
"""
|
|
680
|
+
Mark the content's in_progress rows failed. Call only while holding its
|
|
681
|
+
export lock: the lock outlives EXPORT_TIME_LIMIT, so their runs are dead.
|
|
682
|
+
"""
|
|
683
|
+
try:
|
|
684
|
+
ContentGitExportLog.objects.filter(
|
|
685
|
+
content_key=LearningContextKey.from_string(str(content_key)),
|
|
686
|
+
status=ExportStatus.IN_PROGRESS,
|
|
687
|
+
).update(
|
|
688
|
+
status=ExportStatus.FAILED,
|
|
689
|
+
error_message=(
|
|
690
|
+
"The worker stopped mid-run; its last log line is the step it was in."
|
|
691
|
+
),
|
|
692
|
+
modified=timezone.now(),
|
|
693
|
+
)
|
|
694
|
+
except Exception:
|
|
695
|
+
log.exception("Failed to close dead export runs for %s", content_key)
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _task_id():
|
|
699
|
+
# A direct (non-Celery) call has a request with no id.
|
|
700
|
+
return getattr(getattr(current_task, "request", None), "id", None) or ""
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
# The in_progress row of the export running in this context, until its result.
|
|
704
|
+
_tracked_run = ContextVar("tracked_export_run", default=None)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
class _RunLogHandler(logging.Handler):
|
|
708
|
+
"""Append each log line to a run's row as it is logged."""
|
|
709
|
+
|
|
710
|
+
def __init__(self, pk):
|
|
711
|
+
super().__init__()
|
|
712
|
+
self.pk = pk
|
|
713
|
+
|
|
714
|
+
def emit(self, record):
|
|
715
|
+
try:
|
|
716
|
+
when = datetime.fromtimestamp(record.created, tz=UTC)
|
|
717
|
+
message = redact_credentials(record.getMessage())[:2000]
|
|
718
|
+
line = f"{when:%H:%M:%SZ} {record.levelname} {message}\n"
|
|
719
|
+
ContentGitExportLog.objects.filter(pk=self.pk).update(
|
|
720
|
+
log=Concat("log", Value(line), output_field=TextField()),
|
|
721
|
+
modified=timezone.now(),
|
|
722
|
+
)
|
|
723
|
+
except Exception: # noqa: BLE001
|
|
724
|
+
self.handleError(record)
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
@contextmanager
|
|
728
|
+
def track_export(content_key, user, *loggers, git_url=""):
|
|
729
|
+
"""
|
|
730
|
+
Record an export as in_progress before it runs, with what loggers (and
|
|
731
|
+
this module) log during it, until record_export_result stores its result.
|
|
732
|
+
|
|
733
|
+
A run killed mid-export keeps in_progress, and its last log line is the
|
|
734
|
+
last step it started.
|
|
735
|
+
"""
|
|
736
|
+
run = None
|
|
737
|
+
try:
|
|
738
|
+
run = ContentGitExportLog.objects.create(
|
|
739
|
+
content_key=LearningContextKey.from_string(str(content_key)),
|
|
740
|
+
status=ExportStatus.IN_PROGRESS,
|
|
741
|
+
user=user or "",
|
|
742
|
+
celery_task_id=_task_id(),
|
|
743
|
+
git_url=redact_credentials(git_url or ""),
|
|
744
|
+
)
|
|
745
|
+
except InvalidKeyError:
|
|
746
|
+
pass # unstorable; _export logs it
|
|
747
|
+
except Exception:
|
|
748
|
+
log.exception("Failed to record the start of the %s export", content_key)
|
|
749
|
+
if run is None:
|
|
750
|
+
yield
|
|
751
|
+
return
|
|
752
|
+
|
|
753
|
+
handler = _RunLogHandler(run.pk)
|
|
754
|
+
handler.setLevel(logging.INFO) # DEBUG lines hold full commands and output
|
|
755
|
+
loggers = (*loggers, log)
|
|
756
|
+
# shortcut: handlers are process-wide, fine for prefork workers (one task
|
|
757
|
+
# per process); a pool running tasks concurrently in one process would
|
|
758
|
+
# need a per-run filter.
|
|
759
|
+
levels = {logger: logger.level for logger in loggers}
|
|
760
|
+
for logger in loggers:
|
|
761
|
+
logger.addHandler(handler)
|
|
762
|
+
# The trail needs INFO lines, whatever level the worker logs at.
|
|
763
|
+
if logger.getEffectiveLevel() > logging.INFO:
|
|
764
|
+
logger.setLevel(logging.INFO)
|
|
765
|
+
token = _tracked_run.set(run)
|
|
766
|
+
try:
|
|
767
|
+
yield
|
|
768
|
+
except BaseException as exc:
|
|
769
|
+
# e.g. SystemExit, which export code doesn't catch or record.
|
|
770
|
+
if _tracked_run.get() is run:
|
|
771
|
+
record_export_result(content_key, ExportStatus.FAILED, user=user, exc=exc)
|
|
772
|
+
raise
|
|
773
|
+
finally:
|
|
774
|
+
_tracked_run.reset(token)
|
|
775
|
+
for logger, level in levels.items():
|
|
776
|
+
logger.removeHandler(handler)
|
|
777
|
+
logger.setLevel(level)
|
|
778
|
+
try:
|
|
779
|
+
# Still in_progress: the result's write failed (see the log) or
|
|
780
|
+
# a path skipped record_export_result. Either way, not killed.
|
|
781
|
+
ContentGitExportLog.objects.filter(
|
|
782
|
+
pk=run.pk, status=ExportStatus.IN_PROGRESS
|
|
783
|
+
).update(
|
|
784
|
+
status=ExportStatus.FAILED,
|
|
785
|
+
error_message=(
|
|
786
|
+
"No result was recorded for this run; see its log or the "
|
|
787
|
+
"worker log."
|
|
788
|
+
),
|
|
789
|
+
modified=timezone.now(),
|
|
790
|
+
)
|
|
791
|
+
except Exception:
|
|
792
|
+
log.exception("Failed to close the %s export record", content_key)
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Django models for the git auto-export plugin.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
from django.db import models
|
|
6
|
-
from model_utils.models import TimeStampedModel
|
|
7
|
-
from opaque_keys.edx.django.models import LearningContextKeyField
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class ContentGitRepository(TimeStampedModel):
|
|
11
|
-
"""
|
|
12
|
-
Model to store Git repository information for courses and libraries.
|
|
13
|
-
|
|
14
|
-
This model uses LearningContextKeyField which supports both CourseKey
|
|
15
|
-
and LibraryLocator, making it suitable for both courses and libraries.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
content_key = LearningContextKeyField(max_length=255, primary_key=True)
|
|
19
|
-
git_url = models.CharField(max_length=255)
|
|
20
|
-
is_export_enabled = models.BooleanField(default=True)
|
|
21
|
-
|
|
22
|
-
class Meta:
|
|
23
|
-
verbose_name = "Content Git Repository"
|
|
24
|
-
verbose_name_plural = "Content Git Repositories"
|
|
25
|
-
ordering = ["-created"]
|
|
26
|
-
|
|
27
|
-
def __str__(self):
|
|
28
|
-
return f"ContentGitRepository (content_key={self.content_key})"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|