ol-openedx-git-auto-export 0.7.0__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/PKG-INFO +1 -1
- ol_openedx_git_auto_export-0.7.2/ol_openedx_git_auto_export/constants.py +16 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/management/commands/migrate_giturl.py +9 -3
- ol_openedx_git_auto_export-0.7.2/ol_openedx_git_auto_export/settings/common.py +22 -0
- ol_openedx_git_auto_export-0.7.2/ol_openedx_git_auto_export/settings/production.py +22 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/tasks.py +8 -1
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/utils.py +39 -1
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/pyproject.toml +1 -1
- ol_openedx_git_auto_export-0.7.0/ol_openedx_git_auto_export/constants.py +0 -7
- ol_openedx_git_auto_export-0.7.0/ol_openedx_git_auto_export/settings/common.py +0 -15
- ol_openedx_git_auto_export-0.7.0/ol_openedx_git_auto_export/settings/production.py +0 -15
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/.gitignore +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/LICENSE +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/README.rst +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/admin.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/app.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/management/commands/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/migrations/0001_initial.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/migrations/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/models.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/settings/__init__.py +0 -0
- {ol_openedx_git_auto_export-0.7.0 → ol_openedx_git_auto_export-0.7.2}/ol_openedx_git_auto_export/signals.py +0 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
2
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
3
|
+
GITHUB_ORG = "GITHUB_ORG"
|
|
4
|
+
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
5
|
+
|
|
6
|
+
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
7
|
+
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
|
8
|
+
|
|
9
|
+
# Debounce settings for the signal handler.
|
|
10
|
+
# A single course save triggers 10-30 COURSE_PUBLISHED signals in one request.
|
|
11
|
+
# cache.add() on this key ensures only the first signal schedules a task; all
|
|
12
|
+
# subsequent signals within the window are silently dropped before hitting the broker.
|
|
13
|
+
# The task is scheduled with countdown=EXPORT_DEBOUNCE_DELAY so it runs after
|
|
14
|
+
# the burst window has closed and the course state is fully settled.
|
|
15
|
+
EXPORT_DEBOUNCE_DELAY = 5 # seconds — must exceed the publish burst window
|
|
16
|
+
EXPORT_DEBOUNCE_CACHE_KEY = "git_export_debounce:{course_key}"
|
|
@@ -314,9 +314,15 @@ class Command(BaseCommand):
|
|
|
314
314
|
|
|
315
315
|
message_formats = {
|
|
316
316
|
"existing": lambda data: f" • {data['course_id']}: {data['giturl']}",
|
|
317
|
-
"success": lambda data:
|
|
318
|
-
|
|
319
|
-
|
|
317
|
+
"success": lambda data: (
|
|
318
|
+
f" • {data['course_id']}: {data['ssh_url']} (reason: {data['reason']})"
|
|
319
|
+
),
|
|
320
|
+
"failed": lambda data: (
|
|
321
|
+
f" • {data['course_id']}: Creation failed (reason: {data['reason']})"
|
|
322
|
+
),
|
|
323
|
+
"duplicate": lambda data: (
|
|
324
|
+
f" • {data['course_id']}: Had duplicate URL {data['duplicate_giturl']}"
|
|
325
|
+
),
|
|
320
326
|
}
|
|
321
327
|
|
|
322
328
|
count = len(repo_stats)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Common settings unique to the git auto export plugin."""
|
|
2
|
+
|
|
3
|
+
from ol_openedx_git_auto_export.constants import (
|
|
4
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
+
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def plugin_settings(settings):
|
|
10
|
+
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
+
env_tokens = getattr(settings, "ENV_TOKENS", {})
|
|
12
|
+
settings.GIT_REPO_EXPORT_DIR = env_tokens.get(
|
|
13
|
+
"GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
14
|
+
)
|
|
15
|
+
settings.GITHUB_ORG_API_URL = env_tokens.get("GITHUB_ORG_API_URL", "")
|
|
16
|
+
settings.GITHUB_ACCESS_TOKEN = env_tokens.get("GITHUB_ACCESS_TOKEN")
|
|
17
|
+
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = env_tokens.get("FEATURES", {}).get(
|
|
18
|
+
ENABLE_GIT_AUTO_EXPORT, True
|
|
19
|
+
)
|
|
20
|
+
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = env_tokens.get(
|
|
21
|
+
"FEATURES", {}
|
|
22
|
+
).get(ENABLE_AUTO_GITHUB_REPO_CREATION, False)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Production settings unique to the git auto export plugin."""
|
|
2
|
+
|
|
3
|
+
from ol_openedx_git_auto_export.constants import (
|
|
4
|
+
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
+
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def plugin_settings(settings):
|
|
10
|
+
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
+
env_tokens = getattr(settings, "ENV_TOKENS", {})
|
|
12
|
+
settings.GIT_REPO_EXPORT_DIR = env_tokens.get(
|
|
13
|
+
"GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
14
|
+
)
|
|
15
|
+
settings.GITHUB_ORG_API_URL = env_tokens.get("GITHUB_ORG_API_URL", "")
|
|
16
|
+
settings.GITHUB_ACCESS_TOKEN = env_tokens.get("GITHUB_ACCESS_TOKEN")
|
|
17
|
+
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = env_tokens.get("FEATURES", {}).get(
|
|
18
|
+
ENABLE_GIT_AUTO_EXPORT, True
|
|
19
|
+
)
|
|
20
|
+
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = env_tokens.get(
|
|
21
|
+
"FEATURES", {}
|
|
22
|
+
).get(ENABLE_AUTO_GITHUB_REPO_CREATION, False)
|
|
@@ -16,8 +16,10 @@ from xmodule.modulestore.django import modulestore
|
|
|
16
16
|
|
|
17
17
|
from ol_openedx_git_auto_export.models import CourseGitRepository
|
|
18
18
|
from ol_openedx_git_auto_export.utils import (
|
|
19
|
+
clear_stale_git_lock,
|
|
19
20
|
export_course_to_git,
|
|
20
21
|
github_repo_name_format,
|
|
22
|
+
is_auto_repo_creation_enabled,
|
|
21
23
|
)
|
|
22
24
|
|
|
23
25
|
LOGGER = get_task_logger(__name__)
|
|
@@ -38,6 +40,10 @@ def async_export_to_git(course_key_string, user=None):
|
|
|
38
40
|
"Starting async course content export to git (course id: %s)",
|
|
39
41
|
course_module.id,
|
|
40
42
|
)
|
|
43
|
+
# Remove any stale .git/index.lock left by a previously crashed worker.
|
|
44
|
+
# Dirty working-tree files from a prior crash are cleaned by the
|
|
45
|
+
# `git reset --hard origin/<branch>` + `git clean` inside export_to_git.
|
|
46
|
+
clear_stale_git_lock(course_repo.git_url)
|
|
41
47
|
export_to_git(course_module.id, course_repo.git_url, user=user)
|
|
42
48
|
else:
|
|
43
49
|
LOGGER.info(
|
|
@@ -55,7 +61,8 @@ def async_export_to_git(course_key_string, user=None):
|
|
|
55
61
|
"Creating repository and exporting course content.",
|
|
56
62
|
course_key_string,
|
|
57
63
|
)
|
|
58
|
-
|
|
64
|
+
if is_auto_repo_creation_enabled():
|
|
65
|
+
async_create_github_repo.delay(str(course_key), export_course=True)
|
|
59
66
|
except Exception:
|
|
60
67
|
LOGGER.exception(
|
|
61
68
|
"Unknown error occured during async course content export to git (course id: %s)", # noqa: E501
|
|
@@ -5,15 +5,19 @@ Utility functions for the ol_openedx_git_auto_export app.
|
|
|
5
5
|
import logging
|
|
6
6
|
import os
|
|
7
7
|
import re
|
|
8
|
+
from pathlib import Path
|
|
8
9
|
|
|
9
10
|
from django.conf import settings
|
|
10
11
|
from django.contrib.auth.models import User
|
|
12
|
+
from django.core.cache import cache
|
|
11
13
|
from django.core.exceptions import ImproperlyConfigured
|
|
12
14
|
from xmodule.modulestore.django import modulestore
|
|
13
15
|
|
|
14
16
|
from ol_openedx_git_auto_export.constants import (
|
|
15
17
|
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
16
18
|
ENABLE_GIT_AUTO_EXPORT,
|
|
19
|
+
EXPORT_DEBOUNCE_CACHE_KEY,
|
|
20
|
+
EXPORT_DEBOUNCE_DELAY,
|
|
17
21
|
REPOSITORY_NAME_MAX_LENGTH,
|
|
18
22
|
)
|
|
19
23
|
|
|
@@ -100,7 +104,41 @@ def export_course_to_git(course_key):
|
|
|
100
104
|
)
|
|
101
105
|
|
|
102
106
|
user = get_publisher_username(course_module)
|
|
103
|
-
|
|
107
|
+
|
|
108
|
+
debounce_key = EXPORT_DEBOUNCE_CACHE_KEY.format(course_key=str(course_key))
|
|
109
|
+
if cache.add(debounce_key, "1", timeout=EXPORT_DEBOUNCE_DELAY):
|
|
110
|
+
log.info(
|
|
111
|
+
"Scheduling git export for course %s with %ds debounce delay",
|
|
112
|
+
course_key,
|
|
113
|
+
EXPORT_DEBOUNCE_DELAY,
|
|
114
|
+
)
|
|
115
|
+
async_export_to_git.apply_async(
|
|
116
|
+
args=[str(course_key), user],
|
|
117
|
+
countdown=EXPORT_DEBOUNCE_DELAY,
|
|
118
|
+
)
|
|
119
|
+
else:
|
|
120
|
+
log.info(
|
|
121
|
+
"Git export already scheduled for course %s, skipping duplicate signal",
|
|
122
|
+
course_key,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def clear_stale_git_lock(git_url):
|
|
127
|
+
"""
|
|
128
|
+
Remove a stale .git/index.lock file for the local clone of git_url, if present.
|
|
129
|
+
|
|
130
|
+
A stale lock file can be left behind when a worker process is killed mid-operation.
|
|
131
|
+
"""
|
|
132
|
+
git_repo_export_dir = getattr(
|
|
133
|
+
settings, "GIT_REPO_EXPORT_DIR", "/openedx/export_course_repos"
|
|
134
|
+
)
|
|
135
|
+
rdir = git_url.rsplit("/", 1)[-1].rsplit(".git", 1)[0]
|
|
136
|
+
index_lock = Path(git_repo_export_dir) / rdir / ".git" / "index.lock"
|
|
137
|
+
if index_lock.exists():
|
|
138
|
+
log.warning(
|
|
139
|
+
"Removing stale .git/index.lock for repo %s at %s", git_url, index_lock
|
|
140
|
+
)
|
|
141
|
+
index_lock.unlink()
|
|
104
142
|
|
|
105
143
|
|
|
106
144
|
def is_auto_repo_creation_enabled():
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
ENABLE_GIT_AUTO_EXPORT = "ENABLE_GIT_AUTO_EXPORT"
|
|
2
|
-
ENABLE_AUTO_GITHUB_REPO_CREATION = "ENABLE_AUTO_GITHUB_REPO_CREATION"
|
|
3
|
-
GITHUB_ORG = "GITHUB_ORG"
|
|
4
|
-
GITHUB_ACCESS_TOKEN = "GITHUB_ACCESS_TOKEN" # noqa: S105
|
|
5
|
-
|
|
6
|
-
COURSE_RERUN_STATE_SUCCEEDED = "succeeded"
|
|
7
|
-
REPOSITORY_NAME_MAX_LENGTH = 100 # Max length from GitHub for repo name
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
"""Common settings unique to the git auto export plugin."""
|
|
2
|
-
|
|
3
|
-
from ol_openedx_git_auto_export.constants import (
|
|
4
|
-
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
-
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
-
)
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def plugin_settings(settings):
|
|
10
|
-
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
-
settings.GIT_REPO_EXPORT_DIR = "/openedx/export_course_repos"
|
|
12
|
-
settings.GITHUB_ORG_API_URL = "https://github.mit.edu/api/v3/orgs/mitodl"
|
|
13
|
-
settings.GITHUB_ACCESS_TOKEN = "token" # noqa: S105
|
|
14
|
-
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = True
|
|
15
|
-
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = False
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
"""Production settings unique to the git auto export plugin."""
|
|
2
|
-
|
|
3
|
-
from ol_openedx_git_auto_export.constants import (
|
|
4
|
-
ENABLE_AUTO_GITHUB_REPO_CREATION,
|
|
5
|
-
ENABLE_GIT_AUTO_EXPORT,
|
|
6
|
-
)
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def plugin_settings(settings):
|
|
10
|
-
"""Settings for the git auto export plugin.""" # noqa: D401
|
|
11
|
-
settings.GIT_REPO_EXPORT_DIR = "/openedx/export_course_repos"
|
|
12
|
-
settings.GITHUB_ORG_API_URL = "https://github.mit.edu/api/v3/orgs/mitodl"
|
|
13
|
-
settings.GITHUB_ACCESS_TOKEN = "token" # noqa: S105
|
|
14
|
-
settings.FEATURES[ENABLE_GIT_AUTO_EXPORT] = True
|
|
15
|
-
settings.FEATURES[ENABLE_AUTO_GITHUB_REPO_CREATION] = False
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|