ol-openedx-git-auto-export 0.6.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/PKG-INFO +9 -4
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/README.rst +8 -3
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/admin.py +34 -0
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/ol_openedx_git_auto_export/app.py +6 -1
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/constants.py +7 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +326 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/migrations/0001_initial.py +49 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/models.py +25 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/settings/__init__.py +0 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/settings/common.py +22 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/settings/production.py +22 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/signals.py +60 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/tasks.py +155 -0
- ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/utils.py +129 -0
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/pyproject.toml +1 -1
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/constants.py +0 -1
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/settings/common.py +0 -9
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/settings/production.py +0 -9
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/signals.py +0 -50
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/tasks.py +0 -33
- ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/utils.py +0 -21
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/.gitignore +0 -0
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/LICENSE +0 -0
- {ol_openedx_git_auto_export-0.6.0 → ol_openedx_git_auto_export-0.7.1}/ol_openedx_git_auto_export/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.6.0/ol_openedx_git_auto_export/settings → ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/management/commands}/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ol-openedx-git-auto-export
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: A plugin that auto saves the course OLX to git when an author publishes it
|
|
5
5
|
Author: MIT Office of Digital Learning
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -35,7 +35,12 @@ Configuration
|
|
|
35
35
|
|
|
36
36
|
"FEATURES": {
|
|
37
37
|
"ENABLE_EXPORT_GIT": true,
|
|
38
|
+
"ENABLE_AUTO_GITHUB_REPO_CREATION": true # Optional, to auto create github repo for new courses
|
|
38
39
|
}
|
|
40
|
+
# Set when ENABLE_AUTO_GITHUB_REPO_CREATION is true
|
|
41
|
+
GITHUB_ORG_API_URL = "https://api.github.com/orgs/<GITHUB_ORG_NAME>" # For GitHub Enterprise, change the URL accordingly
|
|
42
|
+
GITHUB_ACCESS_TOKEN = "<GITHUB_PERSONAL_ACCESS_TOKEN>" # Token must have 'repo - Full control of private repositories' permission
|
|
43
|
+
|
|
39
44
|
|
|
40
45
|
- Set your commit user in ``cms/envs/common.py``, if you don't want to use the default one
|
|
41
46
|
|
|
@@ -51,15 +56,15 @@ Configuration
|
|
|
51
56
|
|
|
52
57
|
Setup github authentication for plugin
|
|
53
58
|
---------------------------------------
|
|
54
|
-
If you're testing from a docker machine running devstack setup github
|
|
59
|
+
If you're testing from a docker machine running devstack setup github authentication for plugin, you'll need to generate SSH keys in that
|
|
55
60
|
machine and add them to your Github account
|
|
56
61
|
(https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/ -
|
|
57
62
|
https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account)
|
|
58
63
|
|
|
59
64
|
Studio/CMS UI settings
|
|
60
65
|
----------------------
|
|
61
|
-
- Open studio
|
|
62
|
-
-
|
|
66
|
+
- Open studio admin at `/admin/ol_openedx_git_auto_export/coursegitrepository/`
|
|
67
|
+
- Add your course_id and in the GIT URL, add your OLX git repo. For example ``git@github.com:<GITHUB_USERNAME>/edx4edxlite.git``.
|
|
63
68
|
- Make a change to the course content and publish.
|
|
64
69
|
- When using Tutor, attach with the CMS container using ``tutor dev/local start cms`` and enter ``yes`` to the prompt to add the GitHub to known hosts.
|
|
65
70
|
- You should see a new commit in your OLX repo.
|
|
@@ -19,7 +19,12 @@ Configuration
|
|
|
19
19
|
|
|
20
20
|
"FEATURES": {
|
|
21
21
|
"ENABLE_EXPORT_GIT": true,
|
|
22
|
+
"ENABLE_AUTO_GITHUB_REPO_CREATION": true # Optional, to auto create github repo for new courses
|
|
22
23
|
}
|
|
24
|
+
# Set when ENABLE_AUTO_GITHUB_REPO_CREATION is true
|
|
25
|
+
GITHUB_ORG_API_URL = "https://api.github.com/orgs/<GITHUB_ORG_NAME>" # For GitHub Enterprise, change the URL accordingly
|
|
26
|
+
GITHUB_ACCESS_TOKEN = "<GITHUB_PERSONAL_ACCESS_TOKEN>" # Token must have 'repo - Full control of private repositories' permission
|
|
27
|
+
|
|
23
28
|
|
|
24
29
|
- Set your commit user in ``cms/envs/common.py``, if you don't want to use the default one
|
|
25
30
|
|
|
@@ -35,15 +40,15 @@ Configuration
|
|
|
35
40
|
|
|
36
41
|
Setup github authentication for plugin
|
|
37
42
|
---------------------------------------
|
|
38
|
-
If you're testing from a docker machine running devstack setup github
|
|
43
|
+
If you're testing from a docker machine running devstack setup github authentication for plugin, you'll need to generate SSH keys in that
|
|
39
44
|
machine and add them to your Github account
|
|
40
45
|
(https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/ -
|
|
41
46
|
https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account)
|
|
42
47
|
|
|
43
48
|
Studio/CMS UI settings
|
|
44
49
|
----------------------
|
|
45
|
-
- Open studio
|
|
46
|
-
-
|
|
50
|
+
- Open studio admin at `/admin/ol_openedx_git_auto_export/coursegitrepository/`
|
|
51
|
+
- Add your course_id and in the GIT URL, add your OLX git repo. For example ``git@github.com:<GITHUB_USERNAME>/edx4edxlite.git``.
|
|
47
52
|
- Make a change to the course content and publish.
|
|
48
53
|
- When using Tutor, attach with the CMS container using ``tutor dev/local start cms`` and enter ``yes`` to the prompt to add the GitHub to known hosts.
|
|
49
54
|
- You should see a new commit in your OLX repo.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Django admin pages for git-auto-export plugin
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.contrib import admin
|
|
6
|
+
|
|
7
|
+
from ol_openedx_git_auto_export.models import CourseGitRepository
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@admin.register(CourseGitRepository)
|
|
11
|
+
class CourseGitRepositoryAdmin(admin.ModelAdmin):
|
|
12
|
+
"""
|
|
13
|
+
Admin interface for the CourseGitRepository model.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
list_display = (
|
|
17
|
+
"course_key",
|
|
18
|
+
"git_url",
|
|
19
|
+
"is_export_enabled",
|
|
20
|
+
)
|
|
21
|
+
search_fields = ("course_key", "git_url")
|
|
22
|
+
list_filter = ("is_export_enabled",)
|
|
23
|
+
list_per_page = 50
|
|
24
|
+
|
|
25
|
+
def has_delete_permission(self, request, obj=None): # noqa: ARG002
|
|
26
|
+
"""
|
|
27
|
+
Disable delete permission for CourseGitRepository objects.
|
|
28
|
+
|
|
29
|
+
Deleting a CourseGitRepository could lead to orphaned repositories
|
|
30
|
+
on GitHub and loss of course export functionality.
|
|
31
|
+
|
|
32
|
+
To stop exporting a course, set `is_export_enabled` to False
|
|
33
|
+
"""
|
|
34
|
+
return False
|
|
@@ -30,7 +30,12 @@ class GitAutoExportConfig(AppConfig):
|
|
|
30
30
|
PluginSignals.RECEIVER_FUNC_NAME: "listen_for_course_publish",
|
|
31
31
|
PluginSignals.SIGNAL_PATH: "xmodule.modulestore.django.COURSE_PUBLISHED", # noqa: E501
|
|
32
32
|
PluginSignals.DISPATCH_UID: "ol_openedx_git_auto_export.signals.listen_for_course_publish", # noqa: E501
|
|
33
|
-
}
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
PluginSignals.RECEIVER_FUNC_NAME: "listen_for_course_created",
|
|
36
|
+
PluginSignals.SIGNAL_PATH: "openedx_events.content_authoring.signals.COURSE_CREATED", # noqa: E501
|
|
37
|
+
PluginSignals.DISPATCH_UID: "ol_openedx_git_auto_export.signals.listen_for_course_created", # noqa: E501
|
|
38
|
+
},
|
|
34
39
|
],
|
|
35
40
|
},
|
|
36
41
|
},
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
2
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
3
|
+
GITHUB_ORG = "GITHUB_ORG"
|
|
4
|
+
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
5
|
+
|
|
6
|
+
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
7
|
+
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
ol_openedx_git_auto_export-0.7.1/ol_openedx_git_auto_export/management/commands/migrate_giturl.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Django management command for migrating git URLs
|
|
3
|
+
from course advance settings to CourseGitRepository model.
|
|
4
|
+
|
|
5
|
+
This command scans existing courses in the modulestore for git URLs and migrates them
|
|
6
|
+
to the CourseGitRepository model. It handles duplicate git URLs by creating new
|
|
7
|
+
GitHub repositories for courses that would otherwise share the same repository.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
python manage.py migrate_giturl [course_id1] [course_id2] ...
|
|
11
|
+
python manage.py migrate_giturl --all
|
|
12
|
+
|
|
13
|
+
Examples:
|
|
14
|
+
# Migrate all courses
|
|
15
|
+
python manage.py migrate_giturl --all
|
|
16
|
+
|
|
17
|
+
# Migrate specific courses
|
|
18
|
+
python manage.py migrate_giturl course-v1:MITx+6.00x+2T2024
|
|
19
|
+
|
|
20
|
+
The command will:
|
|
21
|
+
1. Query all courses (or specified courses) from CourseOverview
|
|
22
|
+
2. Extract git URLs from the course modulestore data
|
|
23
|
+
3. Create CourseGitRepository records for courses with git URLs
|
|
24
|
+
4. Handle duplicate git URLs by creating new GitHub repositories
|
|
25
|
+
5. Create new GitHub repositories for courses without git URLs
|
|
26
|
+
|
|
27
|
+
This is typically run as a one-time migration when setting up the git auto-export plugin
|
|
28
|
+
or when consolidating existing course git configurations.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
import time
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
from celery import group
|
|
35
|
+
from django.core.management.base import BaseCommand
|
|
36
|
+
from ol_openedx_git_auto_export.models import CourseGitRepository
|
|
37
|
+
from ol_openedx_git_auto_export.tasks import (
|
|
38
|
+
async_create_github_repo,
|
|
39
|
+
)
|
|
40
|
+
from ol_openedx_git_auto_export.utils import is_auto_repo_creation_enabled
|
|
41
|
+
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
|
|
42
|
+
from xmodule.modulestore.django import modulestore
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Command(BaseCommand):
|
|
46
|
+
help = """
|
|
47
|
+
Migrate git URL from course(s) advanced settings to CourseGitRepository model
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def add_arguments(self, parser):
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"course_keys",
|
|
53
|
+
nargs="*",
|
|
54
|
+
type=str,
|
|
55
|
+
help="List of course keys to migrate their giturl",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--all",
|
|
59
|
+
action="store_true",
|
|
60
|
+
help="Migrate all courses in the system",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def handle(self, *args, **options): # noqa: ARG002
|
|
64
|
+
if not is_auto_repo_creation_enabled():
|
|
65
|
+
return
|
|
66
|
+
|
|
67
|
+
course_keys = options.get("course_keys", [])
|
|
68
|
+
all_courses = options.get("all")
|
|
69
|
+
|
|
70
|
+
if not course_keys and not all_courses:
|
|
71
|
+
# Either course_keys or --all flag must be provided
|
|
72
|
+
command_name = Path(__file__).stem
|
|
73
|
+
self.stderr.write(
|
|
74
|
+
self.style.ERROR(
|
|
75
|
+
"Error: You must specify either course keys or use the --all flag.\n" # noqa: E501
|
|
76
|
+
"Examples:\n"
|
|
77
|
+
f" python manage.py {command_name} --all\n"
|
|
78
|
+
f" python manage.py {command_name} course-v1:MITx+6.00x+2T2024"
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
# Both course_keys and --all flag cannot be used together
|
|
84
|
+
if course_keys and all_courses:
|
|
85
|
+
self.stderr.write(
|
|
86
|
+
self.style.ERROR(
|
|
87
|
+
"Error: Cannot use both course keys and --all flag together. "
|
|
88
|
+
"Please use one or the other."
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
return
|
|
92
|
+
|
|
93
|
+
courses = CourseOverview.objects.all().order_by("created")
|
|
94
|
+
if course_keys:
|
|
95
|
+
courses = courses.filter(id__in=course_keys)
|
|
96
|
+
self.stdout.write(
|
|
97
|
+
self.style.SUCCESS(
|
|
98
|
+
f"Processing {len(course_keys)} specified course(s)..."
|
|
99
|
+
)
|
|
100
|
+
)
|
|
101
|
+
else:
|
|
102
|
+
self.stdout.write(
|
|
103
|
+
self.style.SUCCESS(
|
|
104
|
+
f"Processing all {courses.count()} course(s) in the system..."
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
migration_logs = {
|
|
109
|
+
"existing_repos": [],
|
|
110
|
+
"duplicate_repos": [],
|
|
111
|
+
"new_repos_success": [],
|
|
112
|
+
"new_repos_failed": [],
|
|
113
|
+
"no_giturl": [],
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
processed_giturls = set()
|
|
117
|
+
repos_to_create = []
|
|
118
|
+
for course in courses:
|
|
119
|
+
course_module = modulestore().get_course(course.id, depth=1)
|
|
120
|
+
giturl = course_module.giturl
|
|
121
|
+
|
|
122
|
+
if not giturl:
|
|
123
|
+
migration_logs["no_giturl"].append({"course_id": str(course.id)})
|
|
124
|
+
elif giturl not in processed_giturls:
|
|
125
|
+
processed_giturls.add(giturl)
|
|
126
|
+
migration_logs["existing_repos"].append(
|
|
127
|
+
{"course_id": str(course.id), "giturl": giturl}
|
|
128
|
+
)
|
|
129
|
+
CourseGitRepository.objects.get_or_create(
|
|
130
|
+
course_key=course.id,
|
|
131
|
+
defaults={"git_url": giturl},
|
|
132
|
+
)
|
|
133
|
+
continue
|
|
134
|
+
elif giturl in processed_giturls:
|
|
135
|
+
migration_logs["duplicate_repos"].append(
|
|
136
|
+
{"course_id": str(course.id), "duplicate_giturl": giturl}
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
repos_to_create.append(course.id)
|
|
140
|
+
|
|
141
|
+
if repos_to_create:
|
|
142
|
+
self.stdout.write(f"Processing repository creation for {course.id}...")
|
|
143
|
+
self._process_repositories_in_parallel(repos_to_create, migration_logs)
|
|
144
|
+
|
|
145
|
+
self._display_migration_summary(migration_logs)
|
|
146
|
+
|
|
147
|
+
def _process_repositories_in_parallel(self, repos_to_create, migration_logs):
|
|
148
|
+
"""
|
|
149
|
+
Process repository creation in parallel for better resource management.
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
repos_to_create (list): List of course IDs that need repositories
|
|
153
|
+
migration_logs (dict): Dictionary to store migration results
|
|
154
|
+
"""
|
|
155
|
+
total_repos = len(repos_to_create)
|
|
156
|
+
self.stdout.write(f"Processing {total_repos} repositories in parallel...")
|
|
157
|
+
|
|
158
|
+
# Create group of tasks for all repositories
|
|
159
|
+
job = group(
|
|
160
|
+
async_create_github_repo.s(str(course_id), export_course=True)
|
|
161
|
+
for course_id in repos_to_create
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Execute batch in parallel
|
|
165
|
+
group_result = job.apply_async()
|
|
166
|
+
|
|
167
|
+
# Monitor progress for this batch
|
|
168
|
+
self._monitor_group_progress(group_result, len(repos_to_create))
|
|
169
|
+
|
|
170
|
+
batch_results = group_result.get(propagate=False)
|
|
171
|
+
|
|
172
|
+
# Process results for this batch
|
|
173
|
+
self._process_batch_results(repos_to_create, batch_results, migration_logs)
|
|
174
|
+
|
|
175
|
+
def _process_batch_results(self, course_ids, results, migration_logs):
|
|
176
|
+
"""
|
|
177
|
+
Process the results from a batch of repository creation tasks.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
course_ids (list): List of course IDs in this batch
|
|
181
|
+
results (list): Results from the Celery group execution
|
|
182
|
+
migration_logs (dict): Dictionary to store migration results
|
|
183
|
+
"""
|
|
184
|
+
for i, result in enumerate(results):
|
|
185
|
+
course_id = course_ids[i]
|
|
186
|
+
course_module = modulestore().get_course(course_id, depth=1)
|
|
187
|
+
giturl = course_module.giturl if course_module else None
|
|
188
|
+
|
|
189
|
+
# Handle individual task results or exceptions
|
|
190
|
+
if isinstance(result, Exception):
|
|
191
|
+
migration_logs["new_repos_failed"].append(
|
|
192
|
+
{
|
|
193
|
+
"course_id": str(course_id),
|
|
194
|
+
"reason": f"Task failed with exception: {result!s}",
|
|
195
|
+
}
|
|
196
|
+
)
|
|
197
|
+
elif isinstance(result, tuple):
|
|
198
|
+
is_success, response_msg = result
|
|
199
|
+
if is_success:
|
|
200
|
+
migration_logs["new_repos_success"].append(
|
|
201
|
+
{
|
|
202
|
+
"course_id": str(course_id),
|
|
203
|
+
"ssh_url": response_msg,
|
|
204
|
+
"reason": "duplicate_giturl" if giturl else "no_giturl",
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
else:
|
|
208
|
+
migration_logs["new_repos_failed"].append(
|
|
209
|
+
{"course_id": str(course_id), "reason": response_msg}
|
|
210
|
+
)
|
|
211
|
+
else:
|
|
212
|
+
migration_logs["new_repos_failed"].append(
|
|
213
|
+
{
|
|
214
|
+
"course_id": str(course_id),
|
|
215
|
+
"reason": f"Unexpected result format: {result}",
|
|
216
|
+
}
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
def _monitor_group_progress(self, group_result, total_tasks):
|
|
220
|
+
"""
|
|
221
|
+
Monitor the progress of parallel task execution.
|
|
222
|
+
|
|
223
|
+
Args:
|
|
224
|
+
group_result: The GroupResult object from Celery
|
|
225
|
+
total_tasks (int): Total number of tasks in the group
|
|
226
|
+
"""
|
|
227
|
+
if total_tasks <= 1:
|
|
228
|
+
return # Skip monitoring for single tasks
|
|
229
|
+
|
|
230
|
+
self.stdout.write("Monitoring task progress...")
|
|
231
|
+
last_completed = 0
|
|
232
|
+
|
|
233
|
+
while not group_result.ready():
|
|
234
|
+
completed = sum(1 for task in group_result.children if task.ready())
|
|
235
|
+
|
|
236
|
+
if completed != last_completed:
|
|
237
|
+
progress_percent = (completed / total_tasks) * 100
|
|
238
|
+
self.stdout.write(
|
|
239
|
+
f"Progress: {completed}/{total_tasks} tasks completed ({progress_percent:.1f}%)" # noqa: E501
|
|
240
|
+
)
|
|
241
|
+
last_completed = completed
|
|
242
|
+
|
|
243
|
+
time.sleep(1) # Check progress every second
|
|
244
|
+
|
|
245
|
+
self.stdout.write(f"✅ All {total_tasks} tasks completed!")
|
|
246
|
+
|
|
247
|
+
def _display_migration_summary(self, logs):
|
|
248
|
+
"""
|
|
249
|
+
Display a comprehensive summary of the migration results.
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
logs (dict): Dictionary containing categorized migration results
|
|
253
|
+
"""
|
|
254
|
+
self.stdout.write("\n" + "=" * 80)
|
|
255
|
+
self.stdout.write(self.style.SUCCESS("GIT URL MIGRATION SUMMARY"))
|
|
256
|
+
self.stdout.write("=" * 80)
|
|
257
|
+
|
|
258
|
+
# Summary statistics
|
|
259
|
+
total_existing = len(logs["existing_repos"])
|
|
260
|
+
total_duplicates = len(logs["duplicate_repos"])
|
|
261
|
+
total_no_giturl = len(logs["no_giturl"])
|
|
262
|
+
total_new_success = len(logs["new_repos_success"])
|
|
263
|
+
total_new_failed = len(logs["new_repos_failed"])
|
|
264
|
+
total_processed = total_existing + total_duplicates + total_no_giturl
|
|
265
|
+
|
|
266
|
+
self.stdout.write("\n📊 STATISTICS:")
|
|
267
|
+
self.stdout.write(f" Total courses processed: {total_processed}")
|
|
268
|
+
self.stdout.write(f" Existing repositories: {total_existing}")
|
|
269
|
+
self.stdout.write(f" Duplicate git URLs: {total_duplicates}")
|
|
270
|
+
self.stdout.write(f" Courses without git URLs: {total_no_giturl}")
|
|
271
|
+
self.stdout.write(f" New repositories created: {total_new_success}")
|
|
272
|
+
self.stdout.write(f" Failed repository creations: {total_new_failed}")
|
|
273
|
+
|
|
274
|
+
self.display_migration_section(
|
|
275
|
+
"\n✅ EXISTING REPOSITORIES",
|
|
276
|
+
logs["existing_repos"],
|
|
277
|
+
"existing",
|
|
278
|
+
self.style.SUCCESS,
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
self.display_migration_section(
|
|
282
|
+
"\n🆕 NEW REPOSITORIES CREATED",
|
|
283
|
+
logs["new_repos_success"],
|
|
284
|
+
"success",
|
|
285
|
+
self.style.SUCCESS,
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
self.display_migration_section(
|
|
289
|
+
"\n❌ FAILED REPOSITORY CREATIONS",
|
|
290
|
+
logs["new_repos_failed"],
|
|
291
|
+
"failed",
|
|
292
|
+
self.style.ERROR,
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
self.display_migration_section(
|
|
296
|
+
"\n⚠️ DUPLICATE GIT URLs HANDLED",
|
|
297
|
+
logs["duplicate_repos"],
|
|
298
|
+
"duplicate",
|
|
299
|
+
self.style.WARNING,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
def display_migration_section(self, title, repo_stats, message_type, style_fn):
|
|
303
|
+
"""
|
|
304
|
+
Display a section of migration results with consistent formatting.
|
|
305
|
+
|
|
306
|
+
Args:
|
|
307
|
+
title (str): Title for the section (e.g., "EXISTING REPOSITORIES")
|
|
308
|
+
repo_stats (list): List of repository data dictionaries
|
|
309
|
+
message_type (str): Type of message to display (e.g., "existing", "success")
|
|
310
|
+
style_fn (callable): Function to apply styling to the output
|
|
311
|
+
"""
|
|
312
|
+
if not repo_stats:
|
|
313
|
+
return
|
|
314
|
+
|
|
315
|
+
message_formats = {
|
|
316
|
+
"existing": lambda data: f" • {data['course_id']}: {data['giturl']}",
|
|
317
|
+
"success": lambda data: f" • {data['course_id']}: {data['ssh_url']} (reason: {data['reason']})", # noqa: E501
|
|
318
|
+
"failed": lambda data: f" • {data['course_id']}: Creation failed (reason: {data['reason']})", # noqa: E501
|
|
319
|
+
"duplicate": lambda data: f" • {data['course_id']}: Had duplicate URL {data['duplicate_giturl']}", # noqa: E501
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
count = len(repo_stats)
|
|
323
|
+
self.stdout.write(f"\n{title} ({count}):")
|
|
324
|
+
for repo in repo_stats:
|
|
325
|
+
message = message_formats[message_type](repo)
|
|
326
|
+
self.stdout.write(style_fn(message))
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Generated by Django 4.2.23 on 2025-09-26 10:14
|
|
2
|
+
|
|
3
|
+
import django.utils.timezone
|
|
4
|
+
import model_utils.fields
|
|
5
|
+
import opaque_keys.edx.django.models
|
|
6
|
+
from django.db import migrations, models
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Migration(migrations.Migration):
|
|
10
|
+
initial = True
|
|
11
|
+
|
|
12
|
+
dependencies = [] # type: ignore # noqa: PGH003
|
|
13
|
+
|
|
14
|
+
operations = [
|
|
15
|
+
migrations.CreateModel(
|
|
16
|
+
name="CourseGitRepository",
|
|
17
|
+
fields=[
|
|
18
|
+
(
|
|
19
|
+
"created",
|
|
20
|
+
model_utils.fields.AutoCreatedField(
|
|
21
|
+
default=django.utils.timezone.now,
|
|
22
|
+
editable=False,
|
|
23
|
+
verbose_name="created",
|
|
24
|
+
),
|
|
25
|
+
),
|
|
26
|
+
(
|
|
27
|
+
"modified",
|
|
28
|
+
model_utils.fields.AutoLastModifiedField(
|
|
29
|
+
default=django.utils.timezone.now,
|
|
30
|
+
editable=False,
|
|
31
|
+
verbose_name="modified",
|
|
32
|
+
),
|
|
33
|
+
),
|
|
34
|
+
(
|
|
35
|
+
"course_key",
|
|
36
|
+
opaque_keys.edx.django.models.CourseKeyField(
|
|
37
|
+
max_length=255, primary_key=True, serialize=False
|
|
38
|
+
),
|
|
39
|
+
),
|
|
40
|
+
("git_url", models.CharField(max_length=255)),
|
|
41
|
+
("is_export_enabled", models.BooleanField(default=True)),
|
|
42
|
+
],
|
|
43
|
+
options={
|
|
44
|
+
"verbose_name": "Course Git Repository",
|
|
45
|
+
"verbose_name_plural": "Course Git Repositories",
|
|
46
|
+
"ordering": ["-created"],
|
|
47
|
+
},
|
|
48
|
+
),
|
|
49
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Django models for the git auto-export plugin.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from django.db import models
|
|
6
|
+
from model_utils.models import TimeStampedModel
|
|
7
|
+
from opaque_keys.edx.django.models import CourseKeyField
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CourseGitRepository(TimeStampedModel):
|
|
11
|
+
"""
|
|
12
|
+
Model to store Git repository information for courses.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
course_key = CourseKeyField(max_length=255, primary_key=True)
|
|
16
|
+
git_url = models.CharField(max_length=255)
|
|
17
|
+
is_export_enabled = models.BooleanField(default=True)
|
|
18
|
+
|
|
19
|
+
class Meta:
|
|
20
|
+
verbose_name = "Course Git Repository"
|
|
21
|
+
verbose_name_plural = "Course Git Repositories"
|
|
22
|
+
ordering = ["-created"]
|
|
23
|
+
|
|
24
|
+
def __str__(self):
|
|
25
|
+
return f"CourseGitRepository (course_key={self.course_key})"
|
|
File without changes
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Common settings unique to the git auto export plugin."""
|
|
2
|
+
|
|
3
|
+
from ol_openedx_git_auto_export.constants import (
|
|
4
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
+
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def plugin_settings(settings):
|
|
10
|
+
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
+
env_tokens = getattr(settings, "ENV_TOKENS", {})
|
|
12
|
+
settings.GIT_REPO_EXPORT_DIR = env_tokens.get(
|
|
13
|
+
"GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
14
|
+
)
|
|
15
|
+
settings.GITHUB_ORG_API_URL = env_tokens.get("GITHUB_ORG_API_URL", "")
|
|
16
|
+
settings.GITHUB_ACCESS_TOKEN = env_tokens.get("GITHUB_ACCESS_TOKEN")
|
|
17
|
+
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = env_tokens.get("FEATURES", {}).get(
|
|
18
|
+
ENABLE_GIT_AUTO_EXPORT, True
|
|
19
|
+
)
|
|
20
|
+
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = env_tokens.get(
|
|
21
|
+
"FEATURES", {}
|
|
22
|
+
).get(ENABLE_AUTO_GITHUB_REPO_CREATION, False)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Production settings unique to the git auto export plugin."""
|
|
2
|
+
|
|
3
|
+
from ol_openedx_git_auto_export.constants import (
|
|
4
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
+
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def plugin_settings(settings):
|
|
10
|
+
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
+
env_tokens = getattr(settings, "ENV_TOKENS", {})
|
|
12
|
+
settings.GIT_REPO_EXPORT_DIR = env_tokens.get(
|
|
13
|
+
"GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
14
|
+
)
|
|
15
|
+
settings.GITHUB_ORG_API_URL = env_tokens.get("GITHUB_ORG_API_URL", "")
|
|
16
|
+
settings.GITHUB_ACCESS_TOKEN = env_tokens.get("GITHUB_ACCESS_TOKEN")
|
|
17
|
+
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = env_tokens.get("FEATURES", {}).get(
|
|
18
|
+
ENABLE_GIT_AUTO_EXPORT, True
|
|
19
|
+
)
|
|
20
|
+
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = env_tokens.get(
|
|
21
|
+
"FEATURES", {}
|
|
22
|
+
).get(ENABLE_AUTO_GITHUB_REPO_CREATION, False)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Signal handlers for the git auto-export plugin.
|
|
3
|
+
|
|
4
|
+
This module contains Django signal handlers that respond to course publishing,
|
|
5
|
+
creation, and rerun events to automatically create GitHub repositories and
|
|
6
|
+
export course content to them.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from common.djangoapps.course_action_state.models import CourseRerunState
|
|
12
|
+
from django.db.models.signals import post_save
|
|
13
|
+
from django.dispatch import receiver
|
|
14
|
+
|
|
15
|
+
from ol_openedx_git_auto_export.constants import (
|
|
16
|
+
COURSE_RERUN_STATE_SUCCEEDED,
|
|
17
|
+
)
|
|
18
|
+
from ol_openedx_git_auto_export.tasks import (
|
|
19
|
+
async_create_github_repo,
|
|
20
|
+
)
|
|
21
|
+
from ol_openedx_git_auto_export.utils import (
|
|
22
|
+
export_course_to_git,
|
|
23
|
+
is_auto_repo_creation_enabled,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
log = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def listen_for_course_publish(
|
|
30
|
+
sender, # noqa: ARG001
|
|
31
|
+
course_key,
|
|
32
|
+
**kwargs, # noqa: ARG001
|
|
33
|
+
): # pylint: disable=unused-argument
|
|
34
|
+
"""
|
|
35
|
+
Receives publishing signal and performs publishing related workflows
|
|
36
|
+
"""
|
|
37
|
+
export_course_to_git(course_key)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def listen_for_course_created(**kwargs):
|
|
41
|
+
"""
|
|
42
|
+
Handle course created signal to create a GitHub repository for the course
|
|
43
|
+
"""
|
|
44
|
+
course_key = kwargs.get("course").course_key
|
|
45
|
+
|
|
46
|
+
if is_auto_repo_creation_enabled():
|
|
47
|
+
async_create_github_repo.delay(str(course_key))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@receiver(post_save, sender=CourseRerunState)
|
|
51
|
+
def listen_for_course_rerun_state_post_save(sender, instance, **kwargs): # noqa: ARG001
|
|
52
|
+
"""
|
|
53
|
+
Listen for `CourseRerunState` post_save and
|
|
54
|
+
create GitHub repository and export course content for successfully rerun courses
|
|
55
|
+
"""
|
|
56
|
+
if instance.state != COURSE_RERUN_STATE_SUCCEEDED:
|
|
57
|
+
return
|
|
58
|
+
|
|
59
|
+
if is_auto_repo_creation_enabled():
|
|
60
|
+
async_create_github_repo.delay(str(instance.course_key), export_course=True)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Celery tasks for asynchronous git export operations.
|
|
3
|
+
|
|
4
|
+
This module defines background tasks that handle the actual export of course
|
|
5
|
+
content to Git repositories using Celery for asynchronous processing.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
from celery import shared_task # pylint: disable=import-error
|
|
10
|
+
from celery.utils.log import get_task_logger
|
|
11
|
+
from cms.djangoapps.contentstore.git_export_utils import GitExportError, export_to_git
|
|
12
|
+
from django.conf import settings
|
|
13
|
+
from opaque_keys.edx.keys import CourseKey
|
|
14
|
+
from rest_framework import status
|
|
15
|
+
from xmodule.modulestore.django import modulestore
|
|
16
|
+
|
|
17
|
+
from ol_openedx_git_auto_export.models import CourseGitRepository
|
|
18
|
+
from ol_openedx_git_auto_export.utils import (
|
|
19
|
+
export_course_to_git,
|
|
20
|
+
github_repo_name_format,
|
|
21
|
+
is_auto_repo_creation_enabled,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
LOGGER = get_task_logger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@shared_task
|
|
28
|
+
def async_export_to_git(course_key_string, user=None):
|
|
29
|
+
"""
|
|
30
|
+
Exports a course to Git.
|
|
31
|
+
""" # noqa: D401
|
|
32
|
+
course_key = CourseKey.from_string(course_key_string)
|
|
33
|
+
course_module = modulestore().get_course(course_key)
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
course_repo = CourseGitRepository.objects.get(course_key=course_key)
|
|
37
|
+
if course_repo.is_export_enabled:
|
|
38
|
+
LOGGER.info(
|
|
39
|
+
"Starting async course content export to git (course id: %s)",
|
|
40
|
+
course_module.id,
|
|
41
|
+
)
|
|
42
|
+
export_to_git(course_module.id, course_repo.git_url, user=user)
|
|
43
|
+
else:
|
|
44
|
+
LOGGER.info(
|
|
45
|
+
"Git export is disabled for course %s. Skipping export.",
|
|
46
|
+
course_key_string,
|
|
47
|
+
)
|
|
48
|
+
except GitExportError:
|
|
49
|
+
LOGGER.exception(
|
|
50
|
+
"Failed async course content export to git (course id: %s)",
|
|
51
|
+
course_module.id,
|
|
52
|
+
)
|
|
53
|
+
except CourseGitRepository.DoesNotExist:
|
|
54
|
+
LOGGER.exception(
|
|
55
|
+
"CourseGitRepository does not exist for course %s."
|
|
56
|
+
"Creating repository and exporting course content.",
|
|
57
|
+
course_key_string,
|
|
58
|
+
)
|
|
59
|
+
if is_auto_repo_creation_enabled():
|
|
60
|
+
async_create_github_repo.delay(str(course_key), export_course=True)
|
|
61
|
+
except Exception:
|
|
62
|
+
LOGGER.exception(
|
|
63
|
+
"Unknown error occured during async course content export to git (course id: %s)", # noqa: E501
|
|
64
|
+
course_module.id,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@shared_task(
|
|
69
|
+
bind=True,
|
|
70
|
+
autoretry_for=(requests.exceptions.RequestException,),
|
|
71
|
+
retry_kwargs={"max_retries": 3, "countdown": 10},
|
|
72
|
+
)
|
|
73
|
+
def async_create_github_repo(self, course_key_str, export_course=False): # noqa: FBT002
|
|
74
|
+
"""
|
|
75
|
+
Create a GitHub repository for the given course key.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
course_key_str (str): The course key for which to create the repository.
|
|
79
|
+
export_course (bool): Whether to export the course content
|
|
80
|
+
after creating the repo.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
tuple(bool, str): A tuple containing a boolean indicating success,
|
|
84
|
+
and the SSH URL of the created repository or an error message.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
course_key = CourseKey.from_string(course_key_str)
|
|
88
|
+
course_id_slugified = github_repo_name_format(str(course_key))
|
|
89
|
+
|
|
90
|
+
response_msg = ""
|
|
91
|
+
if CourseGitRepository.objects.filter(course_key=course_key).exists():
|
|
92
|
+
response_msg = f"GitHub repository already exists for course {course_key}. Skipping creation." # noqa: E501
|
|
93
|
+
LOGGER.info(response_msg)
|
|
94
|
+
return False, response_msg
|
|
95
|
+
|
|
96
|
+
course_module = modulestore().get_course(course_key)
|
|
97
|
+
url = f"{settings.GITHUB_ORG_API_URL}/repos"
|
|
98
|
+
# https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28
|
|
99
|
+
headers = {
|
|
100
|
+
"Accept": "application/vnd.github+json",
|
|
101
|
+
"Authorization": f"Bearer {settings.GITHUB_ACCESS_TOKEN}",
|
|
102
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
103
|
+
}
|
|
104
|
+
repo_desc = f"{course_module.display_name}, exported from https://{settings.CMS_BASE}/course/{course_key_str}"
|
|
105
|
+
payload = {
|
|
106
|
+
"name": course_id_slugified,
|
|
107
|
+
"description": repo_desc,
|
|
108
|
+
"private": True,
|
|
109
|
+
"has_issues": False,
|
|
110
|
+
"has_projects": False,
|
|
111
|
+
"has_wiki": False,
|
|
112
|
+
"auto_init": True,
|
|
113
|
+
}
|
|
114
|
+
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
|
115
|
+
if response.status_code != status.HTTP_201_CREATED:
|
|
116
|
+
response_msg = f"Failed to create GitHub repository for course {course_key}: {response.json()}" # noqa: E501
|
|
117
|
+
LOGGER.error(response_msg)
|
|
118
|
+
|
|
119
|
+
# Retry the task if we haven't exceeded max retries
|
|
120
|
+
max_retries = self.retry_kwargs.get("max_retries", 3)
|
|
121
|
+
if self.request.retries < max_retries:
|
|
122
|
+
LOGGER.info(
|
|
123
|
+
"Retrying GitHub repository creation for course %s (attempt %d/%d)",
|
|
124
|
+
course_key,
|
|
125
|
+
self.request.retries + 1,
|
|
126
|
+
max_retries,
|
|
127
|
+
)
|
|
128
|
+
countdown = self.retry_kwargs.get("countdown", 10)
|
|
129
|
+
raise self.retry(countdown=countdown, exc=Exception(response_msg))
|
|
130
|
+
|
|
131
|
+
return False, response_msg
|
|
132
|
+
|
|
133
|
+
repo_data = response.json()
|
|
134
|
+
ssh_url = repo_data.get("ssh_url")
|
|
135
|
+
if ssh_url:
|
|
136
|
+
CourseGitRepository.objects.create(
|
|
137
|
+
course_key=course_key,
|
|
138
|
+
git_url=ssh_url,
|
|
139
|
+
)
|
|
140
|
+
LOGGER.info(
|
|
141
|
+
"GitHub repository created for course %s: %s",
|
|
142
|
+
course_key,
|
|
143
|
+
ssh_url,
|
|
144
|
+
)
|
|
145
|
+
else:
|
|
146
|
+
response_msg = f"""
|
|
147
|
+
Failed to retrieve SSH URL from GitHub response for course {course_key}.
|
|
148
|
+
Response data: {repo_data}
|
|
149
|
+
"""
|
|
150
|
+
LOGGER.error(response_msg)
|
|
151
|
+
|
|
152
|
+
if ssh_url and export_course:
|
|
153
|
+
export_course_to_git(course_key)
|
|
154
|
+
|
|
155
|
+
return True, response_msg or ssh_url
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Utility functions for the ol_openedx_git_auto_export app.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from django.conf import settings
|
|
10
|
+
from django.contrib.auth.models import User
|
|
11
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
12
|
+
from xmodule.modulestore.django import modulestore
|
|
13
|
+
|
|
14
|
+
from ol_openedx_git_auto_export.constants import (
|
|
15
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
16
|
+
ENABLE_GIT_AUTO_EXPORT,
|
|
17
|
+
REPOSITORY_NAME_MAX_LENGTH,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_publisher_username(course_module):
|
|
24
|
+
"""
|
|
25
|
+
Return the username of the user who published the course.
|
|
26
|
+
If the user cannot be found, returns None.
|
|
27
|
+
"""
|
|
28
|
+
if not course_module:
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
user_id = getattr(course_module, "published_by", None)
|
|
32
|
+
if not user_id:
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
user = User.objects.filter(id=user_id).first()
|
|
36
|
+
return user.username if user else None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_or_create_git_export_repo_dir():
|
|
40
|
+
"""
|
|
41
|
+
Ensure the git export directory exists and return its path.
|
|
42
|
+
"""
|
|
43
|
+
git_repo_export_dir = getattr(
|
|
44
|
+
settings, "GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
45
|
+
)
|
|
46
|
+
if not os.path.exists(git_repo_export_dir): # noqa: PTH110
|
|
47
|
+
# for development/docker/vagrant if GIT_REPO_EXPORT_DIR folder does not exist then create it # noqa: E501
|
|
48
|
+
log.error(
|
|
49
|
+
"GIT_REPO_EXPORT_DIR is not available in settings, please create it first"
|
|
50
|
+
)
|
|
51
|
+
os.makedirs(git_repo_export_dir, 0o755) # noqa: PTH103
|
|
52
|
+
|
|
53
|
+
return git_repo_export_dir
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def github_repo_name_format(course_key_str):
|
|
57
|
+
"""
|
|
58
|
+
Format course ID to comply with GitHub repository naming conventions using slugify.
|
|
59
|
+
|
|
60
|
+
GitHub repository names:
|
|
61
|
+
- Can only contain alphanumeric characters
|
|
62
|
+
- And hyphens (-), underscores (_), and periods (.)
|
|
63
|
+
- Cannot start or end with a hyphen
|
|
64
|
+
- Maximum length is 100 characters
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
course_key_str (str): The course key string to format
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
str: GitHub-compliant repository name
|
|
71
|
+
"""
|
|
72
|
+
# Replace all characters with - hyphen except alphanumeric, hyphen, underscore, and period # noqa: E501
|
|
73
|
+
repo_name = re.sub(r"[^A-Za-z0-9_.-]", "-", course_key_str).strip("-")
|
|
74
|
+
|
|
75
|
+
# Truncate to 100 characters if needed
|
|
76
|
+
# Take the last characters to preserve course run identifier
|
|
77
|
+
if len(repo_name) > REPOSITORY_NAME_MAX_LENGTH:
|
|
78
|
+
repo_name = repo_name[-REPOSITORY_NAME_MAX_LENGTH:].lstrip("-")
|
|
79
|
+
|
|
80
|
+
return repo_name.replace("course-v1-", "")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def export_course_to_git(course_key):
|
|
84
|
+
"""
|
|
85
|
+
Export the course to a Git repository.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
course_key (CourseKey): The course key of the course to export.
|
|
89
|
+
"""
|
|
90
|
+
from ol_openedx_git_auto_export.tasks import async_export_to_git # noqa: PLC0415
|
|
91
|
+
|
|
92
|
+
if settings.FEATURES.get("ENABLE_EXPORT_GIT") and settings.FEATURES.get(
|
|
93
|
+
ENABLE_GIT_AUTO_EXPORT
|
|
94
|
+
):
|
|
95
|
+
get_or_create_git_export_repo_dir()
|
|
96
|
+
course_module = modulestore().get_course(course_key)
|
|
97
|
+
log.info(
|
|
98
|
+
"Course published with auto-export enabled. Starting export... (course id: %s)", # noqa: E501
|
|
99
|
+
course_key,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
user = get_publisher_username(course_module)
|
|
103
|
+
async_export_to_git.delay(str(course_key), user)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def is_auto_repo_creation_enabled():
|
|
107
|
+
"""
|
|
108
|
+
Check if automatic GitHub repository creation is enabled.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
course_key (CourseKey): The course key of the course to check.
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
bool: True if automatic GitHub repository creation is enabled, False otherwise.
|
|
115
|
+
|
|
116
|
+
Raises:
|
|
117
|
+
ImproperlyConfigured: If GITHUB_ORG_API_URL or GITHUB_ACCESS_TOKEN is not set.
|
|
118
|
+
"""
|
|
119
|
+
if not settings.FEATURES.get(ENABLE_AUTO_GITHUB_REPO_CREATION):
|
|
120
|
+
log.info(
|
|
121
|
+
"GitHub repo creation is disabled. Skipping GitHub repo creation ...",
|
|
122
|
+
)
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
if not (settings.GITHUB_ORG_API_URL and settings.GITHUB_ACCESS_TOKEN):
|
|
126
|
+
error_msg = "GITHUB_ORG_API_URL or GITHUB_ACCESS_TOKEN is not set in settings. Skipping GitHub repo creation." # noqa: E501
|
|
127
|
+
raise ImproperlyConfigured(error_msg)
|
|
128
|
+
|
|
129
|
+
return True
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
"""Common settings unique to the git auto export plugin."""
|
|
2
|
-
|
|
3
|
-
from ol_openedx_git_auto_export.constants import ENABLE_GIT_AUTO_EXPORT
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def plugin_settings(settings):
|
|
7
|
-
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
8
|
-
settings.GIT_REPO_EXPORT_DIR = "/openedx/export_course_repos"
|
|
9
|
-
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = True
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
"""Production settings unique to the git auto export plugin."""
|
|
2
|
-
|
|
3
|
-
from ol_openedx_git_auto_export.constants import ENABLE_GIT_AUTO_EXPORT
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def plugin_settings(settings):
|
|
7
|
-
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
8
|
-
settings.GIT_REPO_EXPORT_DIR = "/openedx/export_course_repos"
|
|
9
|
-
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = True
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import os
|
|
3
|
-
|
|
4
|
-
from django.conf import settings
|
|
5
|
-
from xmodule.modulestore.django import modulestore
|
|
6
|
-
|
|
7
|
-
from ol_openedx_git_auto_export.constants import ENABLE_GIT_AUTO_EXPORT
|
|
8
|
-
from ol_openedx_git_auto_export.tasks import async_export_to_git
|
|
9
|
-
from ol_openedx_git_auto_export.utils import get_publisher_username
|
|
10
|
-
|
|
11
|
-
log = logging.getLogger(__name__)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def listen_for_course_publish(
|
|
15
|
-
sender, # noqa: ARG001
|
|
16
|
-
course_key,
|
|
17
|
-
**kwargs, # noqa: ARG001
|
|
18
|
-
): # pylint: disable=unused-argument
|
|
19
|
-
"""
|
|
20
|
-
Receives publishing signal and performs publishing related workflows
|
|
21
|
-
"""
|
|
22
|
-
if settings.FEATURES.get("ENABLE_EXPORT_GIT") and settings.FEATURES.get(
|
|
23
|
-
ENABLE_GIT_AUTO_EXPORT
|
|
24
|
-
):
|
|
25
|
-
course_module = modulestore().get_course(course_key)
|
|
26
|
-
if not course_module.giturl:
|
|
27
|
-
log.info(
|
|
28
|
-
"Course %s does not have a GIT URL set in course advance settings, skipping export.", # noqa: E501
|
|
29
|
-
course_module.id,
|
|
30
|
-
)
|
|
31
|
-
return
|
|
32
|
-
|
|
33
|
-
git_repo_export_dir = getattr(
|
|
34
|
-
settings, "GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
35
|
-
)
|
|
36
|
-
if not os.path.exists(git_repo_export_dir): # noqa: PTH110
|
|
37
|
-
# for development/docker/vagrant if GIT_REPO_EXPORT_DIR folder does not exist then create it # noqa: E501
|
|
38
|
-
log.error(
|
|
39
|
-
"GIT_REPO_EXPORT_DIR is not available in settings, please create it first" # noqa: E501
|
|
40
|
-
)
|
|
41
|
-
os.makedirs(git_repo_export_dir, 0o755) # noqa: PTH103
|
|
42
|
-
|
|
43
|
-
log.info(
|
|
44
|
-
"Course published with auto-export enabled. Starting export... (course id: %s)", # noqa: E501
|
|
45
|
-
course_key,
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
user = get_publisher_username(course_module)
|
|
49
|
-
# If the Git auto-export is enabled, push the course changes to Git
|
|
50
|
-
async_export_to_git.delay(str(course_key), user)
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
from celery import shared_task # pylint: disable=import-error
|
|
2
|
-
from celery.utils.log import get_task_logger
|
|
3
|
-
from cms.djangoapps.contentstore.git_export_utils import GitExportError, export_to_git
|
|
4
|
-
from opaque_keys.edx.keys import CourseKey
|
|
5
|
-
from xmodule.modulestore.django import modulestore
|
|
6
|
-
|
|
7
|
-
LOGGER = get_task_logger(__name__)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
@shared_task
|
|
11
|
-
def async_export_to_git(course_key_string, user=None):
|
|
12
|
-
"""
|
|
13
|
-
Exports a course to Git.
|
|
14
|
-
""" # noqa: D401
|
|
15
|
-
course_key = CourseKey.from_string(course_key_string)
|
|
16
|
-
course_module = modulestore().get_course(course_key)
|
|
17
|
-
|
|
18
|
-
try:
|
|
19
|
-
LOGGER.debug(
|
|
20
|
-
"Starting async course content export to git (course id: %s)",
|
|
21
|
-
course_module.id,
|
|
22
|
-
)
|
|
23
|
-
export_to_git(course_module.id, course_module.giturl, user=user)
|
|
24
|
-
except GitExportError:
|
|
25
|
-
LOGGER.exception(
|
|
26
|
-
"Failed async course content export to git (course id: %s)",
|
|
27
|
-
course_module.id,
|
|
28
|
-
)
|
|
29
|
-
except Exception:
|
|
30
|
-
LOGGER.exception(
|
|
31
|
-
"Unknown error occured during async course content export to git (course id: %s)", # noqa: E501
|
|
32
|
-
course_module.id,
|
|
33
|
-
)
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Utility functions for the ol_openedx_git_auto_export app.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
from django.contrib.auth.models import User
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def get_publisher_username(course_module):
|
|
9
|
-
"""
|
|
10
|
-
Return the username of the user who published the course.
|
|
11
|
-
If the user cannot be found, returns None.
|
|
12
|
-
"""
|
|
13
|
-
if not course_module:
|
|
14
|
-
return None
|
|
15
|
-
|
|
16
|
-
user_id = getattr(course_module, "published_by", None)
|
|
17
|
-
if not user_id:
|
|
18
|
-
return None
|
|
19
|
-
|
|
20
|
-
user = User.objects.filter(id=user_id).first()
|
|
21
|
-
return user.username if user else None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|