ol-openedx-course-translations 0.3.0__py3-none-any.whl → 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ol-openedx-course-translations might be problematic. Click here for more details.
- ol_openedx_course_translations/admin.py +29 -0
- ol_openedx_course_translations/apps.py +1 -0
- ol_openedx_course_translations/filters.py +39 -0
- ol_openedx_course_translations/management/commands/translate_course.py +80 -32
- ol_openedx_course_translations/middleware.py +1 -1
- ol_openedx_course_translations/migrations/0001_add_translation_logs.py +84 -0
- ol_openedx_course_translations/migrations/__init__.py +0 -0
- ol_openedx_course_translations/models.py +57 -0
- ol_openedx_course_translations/providers/llm_providers.py +19 -3
- ol_openedx_course_translations/settings/common.py +1 -0
- ol_openedx_course_translations/settings/lms.py +23 -0
- ol_openedx_course_translations/utils/constants.py +2 -0
- ol_openedx_course_translations/utils/course_translations.py +29 -2
- {ol_openedx_course_translations-0.3.0.dist-info → ol_openedx_course_translations-0.3.5.dist-info}/METADATA +4 -2
- {ol_openedx_course_translations-0.3.0.dist-info → ol_openedx_course_translations-0.3.5.dist-info}/RECORD +18 -13
- {ol_openedx_course_translations-0.3.0.dist-info → ol_openedx_course_translations-0.3.5.dist-info}/WHEEL +0 -0
- {ol_openedx_course_translations-0.3.0.dist-info → ol_openedx_course_translations-0.3.5.dist-info}/entry_points.txt +0 -0
- {ol_openedx_course_translations-0.3.0.dist-info → ol_openedx_course_translations-0.3.5.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Django admin configuration for course translations plugin."""
|
|
2
|
+
|
|
3
|
+
from django.contrib import admin
|
|
4
|
+
|
|
5
|
+
from ol_openedx_course_translations.models import CourseTranslationLog
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@admin.register(CourseTranslationLog)
|
|
9
|
+
class CourseTranslationLogAdmin(admin.ModelAdmin):
|
|
10
|
+
"""Admin interface for CourseTranslationLog model."""
|
|
11
|
+
|
|
12
|
+
_common_fields = (
|
|
13
|
+
"source_course_language",
|
|
14
|
+
"target_course_language",
|
|
15
|
+
"srt_provider_name",
|
|
16
|
+
"srt_provider_model",
|
|
17
|
+
"content_provider_name",
|
|
18
|
+
"content_provider_model",
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
list_display = ("id", "source_course_id", *_common_fields, "created_at")
|
|
22
|
+
list_filter = _common_fields
|
|
23
|
+
readonly_fields = (
|
|
24
|
+
"source_course_id",
|
|
25
|
+
*_common_fields,
|
|
26
|
+
"created_at",
|
|
27
|
+
"command_stats",
|
|
28
|
+
)
|
|
29
|
+
search_fields = ("source_course_id",)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Filters for Open edX course translations.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from openedx_filters import PipelineStep
|
|
6
|
+
from xmodule.modulestore.django import modulestore
|
|
7
|
+
|
|
8
|
+
from ol_openedx_course_translations.utils.constants import ENGLISH_LANGUAGE_CODE
|
|
9
|
+
|
|
10
|
+
VIDEO_BLOCK_TYPE = "video"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AddDestLangForVideoBlock(PipelineStep):
|
|
14
|
+
"""
|
|
15
|
+
Pipeline step to add destination language for video transcripts
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def run_filter(self, context, student_view_context):
|
|
19
|
+
"""
|
|
20
|
+
Add the destination language to the student view context if a video block
|
|
21
|
+
with transcripts in the course language is found among the child blocks.
|
|
22
|
+
"""
|
|
23
|
+
for child in dict(context)["block"].children:
|
|
24
|
+
if child.block_type == VIDEO_BLOCK_TYPE:
|
|
25
|
+
student_view_context["dest_lang"] = (
|
|
26
|
+
ENGLISH_LANGUAGE_CODE # default to English
|
|
27
|
+
)
|
|
28
|
+
video_block = modulestore().get_item(child)
|
|
29
|
+
transcripts_info = video_block.get_transcripts_info()
|
|
30
|
+
course_lang = getattr(
|
|
31
|
+
context.get("course", None), "language", ENGLISH_LANGUAGE_CODE
|
|
32
|
+
)
|
|
33
|
+
if (
|
|
34
|
+
transcripts_info
|
|
35
|
+
and transcripts_info.get("transcripts", {})
|
|
36
|
+
and course_lang in transcripts_info["transcripts"]
|
|
37
|
+
):
|
|
38
|
+
student_view_context["dest_lang"] = course_lang
|
|
39
|
+
return {"context": context, "student_view_context": student_view_context}
|
|
@@ -11,6 +11,7 @@ from celery import group
|
|
|
11
11
|
from django.conf import settings
|
|
12
12
|
from django.core.management.base import BaseCommand, CommandError
|
|
13
13
|
|
|
14
|
+
from ol_openedx_course_translations.models import CourseTranslationLog
|
|
14
15
|
from ol_openedx_course_translations.tasks import (
|
|
15
16
|
translate_file_task,
|
|
16
17
|
translate_grading_policy_task,
|
|
@@ -21,6 +22,7 @@ from ol_openedx_course_translations.utils.course_translations import (
|
|
|
21
22
|
create_translated_archive,
|
|
22
23
|
create_translated_copy,
|
|
23
24
|
extract_course_archive,
|
|
25
|
+
generate_course_key_from_xml,
|
|
24
26
|
get_translatable_file_paths,
|
|
25
27
|
update_course_language_attribute,
|
|
26
28
|
validate_course_inputs,
|
|
@@ -193,9 +195,10 @@ class Command(BaseCommand):
|
|
|
193
195
|
def handle(self, **options) -> None:
|
|
194
196
|
"""Handle the translate_course command."""
|
|
195
197
|
try:
|
|
198
|
+
start_time = time.perf_counter()
|
|
196
199
|
course_archive_path = Path(options["course_archive_path"])
|
|
197
|
-
source_language = options["source_language"]
|
|
198
|
-
target_language = options["target_language"]
|
|
200
|
+
source_language = options["source_language"].upper()
|
|
201
|
+
target_language = options["target_language"].upper()
|
|
199
202
|
content_provider_spec = options["content_translation_provider"]
|
|
200
203
|
srt_provider_spec = options["srt_translation_provider"]
|
|
201
204
|
glossary_directory = options.get("glossary_directory")
|
|
@@ -252,18 +255,28 @@ class Command(BaseCommand):
|
|
|
252
255
|
)
|
|
253
256
|
|
|
254
257
|
# Wait for all tasks and report status
|
|
255
|
-
self._wait_and_report_tasks()
|
|
256
|
-
|
|
258
|
+
command_stats = self._wait_and_report_tasks()
|
|
259
|
+
total_time_taken_msg = (
|
|
260
|
+
f"Command finished in: {time.perf_counter() - start_time:.2f} seconds."
|
|
261
|
+
)
|
|
262
|
+
self.stdout.write(self.style.SUCCESS(total_time_taken_msg))
|
|
263
|
+
command_stats.append(total_time_taken_msg)
|
|
264
|
+
|
|
265
|
+
# Add translation log entry
|
|
266
|
+
self._add_translation_log_entry(
|
|
267
|
+
source_language=source_language,
|
|
268
|
+
target_language=target_language,
|
|
269
|
+
command_stats=command_stats,
|
|
270
|
+
)
|
|
257
271
|
# Create final archive
|
|
258
272
|
translated_archive_path = create_translated_archive(
|
|
259
273
|
translated_course_dir, target_language, course_archive_path.stem
|
|
260
274
|
)
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
f"Translation completed. Archive created: {translated_archive_path}"
|
|
265
|
-
)
|
|
275
|
+
success_msg = (
|
|
276
|
+
f"Translation completed successfully. Translated archive created: "
|
|
277
|
+
f"{translated_archive_path}"
|
|
266
278
|
)
|
|
279
|
+
self.stdout.write(self.style.SUCCESS(success_msg))
|
|
267
280
|
|
|
268
281
|
except Exception as e:
|
|
269
282
|
logger.exception("Translation failed")
|
|
@@ -417,7 +430,7 @@ class Command(BaseCommand):
|
|
|
417
430
|
self.tasks.append(("policy", str(policy_file), task))
|
|
418
431
|
logger.info("Added policy.json task for: %s", policy_file)
|
|
419
432
|
|
|
420
|
-
def _wait_and_report_tasks(self) ->
|
|
433
|
+
def _wait_and_report_tasks(self) -> list[str]: # noqa: C901, PLR0915, PLR0912
|
|
421
434
|
"""
|
|
422
435
|
Execute all tasks as a Celery group and wait for completion.
|
|
423
436
|
|
|
@@ -427,9 +440,10 @@ class Command(BaseCommand):
|
|
|
427
440
|
Raises:
|
|
428
441
|
CommandError: If any tasks fail
|
|
429
442
|
"""
|
|
443
|
+
stats = []
|
|
430
444
|
if not self.tasks:
|
|
431
445
|
self.stdout.write("No tasks to execute.")
|
|
432
|
-
return
|
|
446
|
+
return []
|
|
433
447
|
|
|
434
448
|
total_tasks = len(self.tasks)
|
|
435
449
|
self.stdout.write(
|
|
@@ -449,7 +463,6 @@ class Command(BaseCommand):
|
|
|
449
463
|
|
|
450
464
|
# Wait for all tasks to complete with progress reporting
|
|
451
465
|
completed_count = 0
|
|
452
|
-
self.stdout.write(f"Progress: 0/{total_tasks} tasks completed")
|
|
453
466
|
self.stdout.flush()
|
|
454
467
|
|
|
455
468
|
try:
|
|
@@ -491,47 +504,82 @@ class Command(BaseCommand):
|
|
|
491
504
|
|
|
492
505
|
if isinstance(task_result, dict):
|
|
493
506
|
status = task_result.get("status", "unknown")
|
|
494
|
-
|
|
495
507
|
if status == "success":
|
|
496
508
|
completed_tasks += 1
|
|
497
|
-
|
|
509
|
+
msg = f"✓ {task_type}: {file_path}"
|
|
510
|
+
stats.append(msg)
|
|
511
|
+
self.stdout.write(self.style.SUCCESS(msg))
|
|
498
512
|
elif status == "skipped":
|
|
499
513
|
skipped_tasks += 1
|
|
500
514
|
reason = task_result.get("reason", "Skipped")
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
)
|
|
515
|
+
msg = f"⊘ {task_type}: {file_path} - {reason}"
|
|
516
|
+
stats.append(msg)
|
|
517
|
+
self.stdout.write(self.style.WARNING(msg))
|
|
504
518
|
elif status == "error":
|
|
505
519
|
failed_tasks += 1
|
|
506
520
|
error = task_result.get("error", "Unknown error")
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
)
|
|
521
|
+
msg = f"✗ {task_type}: {file_path} - {error}"
|
|
522
|
+
stats.append(msg)
|
|
523
|
+
self.stdout.write(self.style.ERROR(msg))
|
|
510
524
|
else:
|
|
511
525
|
failed_tasks += 1
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
)
|
|
516
|
-
)
|
|
526
|
+
msg = f"✗ {task_type}: {file_path} - Unknown status: {status}"
|
|
527
|
+
stats.append(msg)
|
|
528
|
+
self.stdout.write(self.style.ERROR(msg))
|
|
517
529
|
else:
|
|
518
530
|
# Task raised an exception
|
|
519
531
|
failed_tasks += 1
|
|
520
532
|
error_msg = str(task_result) if task_result else "Task failed"
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
)
|
|
533
|
+
msg = f"✗ {task_type}: {file_path} - {error_msg}"
|
|
534
|
+
stats.append(msg)
|
|
535
|
+
self.stdout.write(self.style.ERROR(msg))
|
|
524
536
|
|
|
525
537
|
# Print summary
|
|
526
538
|
self.stdout.write("\n" + "=" * 60)
|
|
527
|
-
|
|
528
|
-
|
|
539
|
+
successful_tasks_stats = (
|
|
540
|
+
f"Total tasks: {total_tasks}\nCompleted: {completed_tasks}"
|
|
541
|
+
)
|
|
542
|
+
stats.append(successful_tasks_stats)
|
|
543
|
+
self.stdout.write(self.style.SUCCESS(successful_tasks_stats))
|
|
529
544
|
if skipped_tasks > 0:
|
|
530
|
-
|
|
545
|
+
skipped_tasks_stats = f"Skipped: {skipped_tasks}"
|
|
546
|
+
stats.append(skipped_tasks_stats)
|
|
547
|
+
self.stdout.write(self.style.WARNING(skipped_tasks_stats))
|
|
531
548
|
if failed_tasks > 0:
|
|
532
|
-
|
|
549
|
+
failed_tasks_stats = f"Failed: {failed_tasks}"
|
|
550
|
+
stats.append(failed_tasks_stats)
|
|
551
|
+
self.stdout.write(self.style.ERROR(failed_tasks_stats))
|
|
533
552
|
self.stdout.write("=" * 60 + "\n")
|
|
534
553
|
|
|
535
554
|
if failed_tasks > 0:
|
|
536
555
|
error_msg = f"{failed_tasks} translation tasks failed"
|
|
537
556
|
raise CommandError(error_msg)
|
|
557
|
+
|
|
558
|
+
return stats
|
|
559
|
+
|
|
560
|
+
def _add_translation_log_entry(
|
|
561
|
+
self, source_language, target_language, command_stats=None
|
|
562
|
+
) -> None:
|
|
563
|
+
"""
|
|
564
|
+
Add a log entry for the course translation operation.
|
|
565
|
+
|
|
566
|
+
Args:
|
|
567
|
+
source_language: Source language code
|
|
568
|
+
target_language: Target language code
|
|
569
|
+
command_stats: List of command statistics/logs
|
|
570
|
+
"""
|
|
571
|
+
source_course_id = generate_course_key_from_xml(
|
|
572
|
+
course_dir_path=self.translated_course_dir
|
|
573
|
+
)
|
|
574
|
+
command_stats_str = "\n".join(command_stats) if command_stats else ""
|
|
575
|
+
|
|
576
|
+
CourseTranslationLog.objects.create(
|
|
577
|
+
source_course_id=source_course_id,
|
|
578
|
+
source_course_language=source_language,
|
|
579
|
+
target_course_language=target_language,
|
|
580
|
+
srt_provider_name=self.srt_provider_name,
|
|
581
|
+
srt_provider_model=self.srt_model or "",
|
|
582
|
+
content_provider_name=self.content_provider_name,
|
|
583
|
+
content_provider_model=self.content_model or "",
|
|
584
|
+
command_stats=command_stats_str,
|
|
585
|
+
)
|
|
@@ -14,7 +14,7 @@ from openedx.core.djangoapps.lang_pref import LANGUAGE_KEY
|
|
|
14
14
|
from openedx.core.djangoapps.lang_pref import helpers as lang_pref_helpers
|
|
15
15
|
from openedx.core.djangoapps.user_api.preferences.api import set_user_preference
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
from ol_openedx_course_translations.utils.constants import ENGLISH_LANGUAGE_CODE
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def should_process_request(request):
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Generated by Django 5.2.9 on 2026-01-15 11:04
|
|
2
|
+
|
|
3
|
+
import opaque_keys.edx.django.models
|
|
4
|
+
from django.db import migrations, models
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Migration(migrations.Migration):
|
|
8
|
+
initial = True
|
|
9
|
+
|
|
10
|
+
dependencies = [] # type: ignore[var-annotated]
|
|
11
|
+
|
|
12
|
+
operations = [
|
|
13
|
+
migrations.CreateModel(
|
|
14
|
+
name="CourseTranslationLog",
|
|
15
|
+
fields=[
|
|
16
|
+
(
|
|
17
|
+
"id",
|
|
18
|
+
models.AutoField(
|
|
19
|
+
auto_created=True,
|
|
20
|
+
primary_key=True,
|
|
21
|
+
serialize=False,
|
|
22
|
+
verbose_name="ID",
|
|
23
|
+
),
|
|
24
|
+
),
|
|
25
|
+
(
|
|
26
|
+
"source_course_id",
|
|
27
|
+
opaque_keys.edx.django.models.CourseKeyField(
|
|
28
|
+
db_index=True, max_length=255
|
|
29
|
+
),
|
|
30
|
+
),
|
|
31
|
+
(
|
|
32
|
+
"source_course_language",
|
|
33
|
+
models.CharField(
|
|
34
|
+
help_text="Source language code (e.g., 'EN')", max_length=10
|
|
35
|
+
),
|
|
36
|
+
),
|
|
37
|
+
(
|
|
38
|
+
"target_course_language",
|
|
39
|
+
models.CharField(
|
|
40
|
+
help_text="Target language code for translation (e.g., 'FR')",
|
|
41
|
+
max_length=10,
|
|
42
|
+
),
|
|
43
|
+
),
|
|
44
|
+
(
|
|
45
|
+
"srt_provider_name",
|
|
46
|
+
models.CharField(
|
|
47
|
+
help_text="LLM Provider used for SRT translation",
|
|
48
|
+
max_length=100,
|
|
49
|
+
),
|
|
50
|
+
),
|
|
51
|
+
(
|
|
52
|
+
"srt_provider_model",
|
|
53
|
+
models.CharField(
|
|
54
|
+
blank=True,
|
|
55
|
+
help_text="LLM provider model used for SRT translation",
|
|
56
|
+
max_length=100,
|
|
57
|
+
),
|
|
58
|
+
),
|
|
59
|
+
(
|
|
60
|
+
"content_provider_name",
|
|
61
|
+
models.CharField(
|
|
62
|
+
help_text="LLM Provider used for content translation",
|
|
63
|
+
max_length=100,
|
|
64
|
+
),
|
|
65
|
+
),
|
|
66
|
+
(
|
|
67
|
+
"content_provider_model",
|
|
68
|
+
models.CharField(
|
|
69
|
+
blank=True,
|
|
70
|
+
help_text="LLM provider model used for content translation",
|
|
71
|
+
max_length=100,
|
|
72
|
+
),
|
|
73
|
+
),
|
|
74
|
+
(
|
|
75
|
+
"command_stats",
|
|
76
|
+
models.TextField(
|
|
77
|
+
blank=True, help_text="Logs from the translation command"
|
|
78
|
+
),
|
|
79
|
+
),
|
|
80
|
+
("created_at", models.DateTimeField(auto_now_add=True)),
|
|
81
|
+
("updated_at", models.DateTimeField(auto_now=True)),
|
|
82
|
+
],
|
|
83
|
+
),
|
|
84
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Models for course translations plugin"""
|
|
2
|
+
|
|
3
|
+
from django.db import models
|
|
4
|
+
from opaque_keys.edx.django.models import CourseKeyField
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CourseTranslationLog(models.Model):
|
|
8
|
+
"""Log entry for course translation operations."""
|
|
9
|
+
|
|
10
|
+
source_course_id = CourseKeyField(max_length=255, db_index=True)
|
|
11
|
+
source_course_language = models.CharField(
|
|
12
|
+
max_length=10,
|
|
13
|
+
help_text="Source language code (e.g., 'EN')",
|
|
14
|
+
)
|
|
15
|
+
target_course_language = models.CharField(
|
|
16
|
+
max_length=10,
|
|
17
|
+
help_text="Target language code for translation (e.g., 'FR')",
|
|
18
|
+
)
|
|
19
|
+
srt_provider_name = models.CharField(
|
|
20
|
+
max_length=100,
|
|
21
|
+
help_text="LLM Provider used for SRT translation",
|
|
22
|
+
)
|
|
23
|
+
srt_provider_model = models.CharField(
|
|
24
|
+
max_length=100,
|
|
25
|
+
blank=True,
|
|
26
|
+
help_text="LLM provider model used for SRT translation",
|
|
27
|
+
)
|
|
28
|
+
content_provider_name = models.CharField(
|
|
29
|
+
max_length=100,
|
|
30
|
+
help_text="LLM Provider used for content translation",
|
|
31
|
+
)
|
|
32
|
+
content_provider_model = models.CharField(
|
|
33
|
+
max_length=100,
|
|
34
|
+
blank=True,
|
|
35
|
+
help_text="LLM provider model used for content translation",
|
|
36
|
+
)
|
|
37
|
+
command_stats = models.TextField(
|
|
38
|
+
blank=True, help_text="Logs from the translation command"
|
|
39
|
+
)
|
|
40
|
+
created_at = models.DateTimeField(
|
|
41
|
+
auto_now_add=True,
|
|
42
|
+
)
|
|
43
|
+
updated_at = models.DateTimeField(
|
|
44
|
+
auto_now=True,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
class Meta:
|
|
48
|
+
"""Meta options for CourseTranslationLog."""
|
|
49
|
+
|
|
50
|
+
app_label = "ol_openedx_course_translations"
|
|
51
|
+
|
|
52
|
+
def __str__(self):
|
|
53
|
+
"""Return a string representation of the translation log."""
|
|
54
|
+
return (
|
|
55
|
+
f"{self.source_course_id} "
|
|
56
|
+
f"({self.source_course_language} → {self.target_course_language})"
|
|
57
|
+
)
|
|
@@ -6,6 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
import srt
|
|
9
|
+
from django.conf import settings
|
|
9
10
|
from litellm import completion
|
|
10
11
|
|
|
11
12
|
from .base import TranslationProvider, load_glossary
|
|
@@ -68,6 +69,7 @@ class LLMProvider(TranslationProvider):
|
|
|
68
69
|
primary_api_key: str,
|
|
69
70
|
repair_api_key: str | None = None,
|
|
70
71
|
model_name: str | None = None,
|
|
72
|
+
timeout: int = settings.LITE_LLM_REQUEST_TIMEOUT,
|
|
71
73
|
):
|
|
72
74
|
"""
|
|
73
75
|
Initialize LLM provider with API keys and model name.
|
|
@@ -79,6 +81,7 @@ class LLMProvider(TranslationProvider):
|
|
|
79
81
|
"""
|
|
80
82
|
super().__init__(primary_api_key, repair_api_key)
|
|
81
83
|
self.model_name = model_name
|
|
84
|
+
self.timeout = timeout
|
|
82
85
|
|
|
83
86
|
def _get_subtitle_system_prompt(
|
|
84
87
|
self,
|
|
@@ -317,6 +320,7 @@ class LLMProvider(TranslationProvider):
|
|
|
317
320
|
model=self.model_name,
|
|
318
321
|
messages=llm_messages,
|
|
319
322
|
api_key=self.primary_api_key,
|
|
323
|
+
timeout=self.timeout,
|
|
320
324
|
**additional_kwargs,
|
|
321
325
|
)
|
|
322
326
|
return llm_response.choices[0].message.content.strip()
|
|
@@ -348,6 +352,9 @@ class LLMProvider(TranslationProvider):
|
|
|
348
352
|
current_batch_size = len(subtitle_list)
|
|
349
353
|
|
|
350
354
|
current_index = 0
|
|
355
|
+
retry_count = 0
|
|
356
|
+
max_retries = 3
|
|
357
|
+
|
|
351
358
|
while current_index < len(subtitle_list):
|
|
352
359
|
subtitle_batch = subtitle_list[
|
|
353
360
|
current_index : current_index + current_batch_size
|
|
@@ -373,18 +380,27 @@ class LLMProvider(TranslationProvider):
|
|
|
373
380
|
)
|
|
374
381
|
translated_subtitle_list.extend(translated_batch)
|
|
375
382
|
current_index += current_batch_size
|
|
383
|
+
retry_count = 0 # Reset retry count on success
|
|
376
384
|
|
|
377
385
|
except Exception as llm_error:
|
|
378
386
|
error_message = str(llm_error).lower()
|
|
379
387
|
if any(
|
|
380
388
|
error_term in error_message for error_term in LLM_ERROR_KEYWORDS
|
|
381
389
|
):
|
|
382
|
-
if current_batch_size <= 1:
|
|
383
|
-
logger.exception(
|
|
390
|
+
if current_batch_size <= 1 or retry_count >= max_retries:
|
|
391
|
+
logger.exception(
|
|
392
|
+
"Failed after %s batch size reduction attempts", retry_count
|
|
393
|
+
)
|
|
384
394
|
raise
|
|
385
395
|
|
|
386
|
-
logger.warning(
|
|
396
|
+
logger.warning(
|
|
397
|
+
"Error: %s. Reducing batch size (attempt %s/%s)...",
|
|
398
|
+
llm_error,
|
|
399
|
+
retry_count + 1,
|
|
400
|
+
max_retries,
|
|
401
|
+
)
|
|
387
402
|
current_batch_size = max(1, current_batch_size // 2)
|
|
403
|
+
retry_count += 1
|
|
388
404
|
continue
|
|
389
405
|
else:
|
|
390
406
|
raise
|
|
@@ -13,3 +13,26 @@ def plugin_settings(settings):
|
|
|
13
13
|
settings.MIDDLEWARE.extend(
|
|
14
14
|
["ol_openedx_course_translations.middleware.CourseLanguageCookieMiddleware"]
|
|
15
15
|
)
|
|
16
|
+
VIDEO_TRANSCRIPT_LANGUAGE_FILTERS = {
|
|
17
|
+
"org.openedx.learning.xblock.render.started.v1": {
|
|
18
|
+
"pipeline": [
|
|
19
|
+
"ol_openedx_course_translations.filters.AddDestLangForVideoBlock"
|
|
20
|
+
],
|
|
21
|
+
"fail_silently": False,
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
existing_filters = getattr(settings, "OPEN_EDX_FILTERS_CONFIG", {})
|
|
25
|
+
|
|
26
|
+
# Merge pipeline lists instead of overwriting
|
|
27
|
+
for filter_name, config in VIDEO_TRANSCRIPT_LANGUAGE_FILTERS.items():
|
|
28
|
+
if filter_name not in existing_filters:
|
|
29
|
+
existing_filters[filter_name] = config
|
|
30
|
+
else:
|
|
31
|
+
existing_filters[filter_name]["pipeline"].extend(config.get("pipeline", []))
|
|
32
|
+
# do not override fail_silently
|
|
33
|
+
if "fail_silently" in config:
|
|
34
|
+
existing_filters[filter_name].setdefault(
|
|
35
|
+
"fail_silently", config["fail_silently"]
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
settings.OPEN_EDX_FILTERS_CONFIG = existing_filters
|
|
@@ -11,6 +11,7 @@ from xml.etree.ElementTree import Element
|
|
|
11
11
|
from defusedxml import ElementTree
|
|
12
12
|
from django.conf import settings
|
|
13
13
|
from django.core.management.base import CommandError
|
|
14
|
+
from opaque_keys.edx.locator import CourseLocator
|
|
14
15
|
|
|
15
16
|
from ol_openedx_course_translations.providers.deepl_provider import DeepLProvider
|
|
16
17
|
from ol_openedx_course_translations.providers.llm_providers import (
|
|
@@ -126,7 +127,7 @@ def translate_xml_display_name(
|
|
|
126
127
|
if display_name:
|
|
127
128
|
translated_name = provider.translate_text(
|
|
128
129
|
display_name,
|
|
129
|
-
target_language
|
|
130
|
+
target_language,
|
|
130
131
|
glossary_directory=glossary_directory,
|
|
131
132
|
)
|
|
132
133
|
xml_root.set("display_name", translated_name)
|
|
@@ -150,7 +151,7 @@ def update_video_xml_transcripts(xml_content: str, target_language: str) -> str:
|
|
|
150
151
|
"""
|
|
151
152
|
try:
|
|
152
153
|
xml_root = ElementTree.fromstring(xml_content)
|
|
153
|
-
target_lang_code = target_language
|
|
154
|
+
target_lang_code = target_language
|
|
154
155
|
|
|
155
156
|
# Update transcripts attribute in <video> tag
|
|
156
157
|
if xml_root.tag == "video" and "transcripts" in xml_root.attrib:
|
|
@@ -579,3 +580,29 @@ def get_translatable_file_paths(
|
|
|
579
580
|
]
|
|
580
581
|
|
|
581
582
|
return translatable_file_paths
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def generate_course_key_from_xml(course_dir_path: Path) -> str:
|
|
586
|
+
"""
|
|
587
|
+
Generate the course id of the source course
|
|
588
|
+
"""
|
|
589
|
+
try:
|
|
590
|
+
about_file_path = course_dir_path / "course" / "course.xml"
|
|
591
|
+
xml_content = about_file_path.read_text(encoding="utf-8")
|
|
592
|
+
xml_root = ElementTree.fromstring(xml_content)
|
|
593
|
+
|
|
594
|
+
org = xml_root.get("org", "")
|
|
595
|
+
course = xml_root.get("course", "")
|
|
596
|
+
url_name = xml_root.get("url_name", "")
|
|
597
|
+
|
|
598
|
+
if not all([org, course, url_name]):
|
|
599
|
+
error_msg = (
|
|
600
|
+
"Missing required attributes in course.xml: org, course, url_name"
|
|
601
|
+
)
|
|
602
|
+
raise CommandError(error_msg)
|
|
603
|
+
except (OSError, ElementTree.ParseError) as e:
|
|
604
|
+
error_msg = f"Failed to read course id from about.xml: {e}"
|
|
605
|
+
raise CommandError(error_msg) from e
|
|
606
|
+
else:
|
|
607
|
+
# URL name is the run ID of the course
|
|
608
|
+
return CourseLocator(org=org, course=course, run=url_name)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ol-openedx-course-translations
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.5
|
|
4
4
|
Summary: An Open edX plugin to translate courses
|
|
5
5
|
Author: MIT Office of Digital Learning
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -10,8 +10,9 @@ Requires-Python: >=3.11
|
|
|
10
10
|
Requires-Dist: deepl>=1.25.0
|
|
11
11
|
Requires-Dist: django>=4.0
|
|
12
12
|
Requires-Dist: djangorestframework>=3.14.0
|
|
13
|
+
Requires-Dist: edx-opaque-keys
|
|
13
14
|
Requires-Dist: gitpython>=3.1.40
|
|
14
|
-
Requires-Dist: litellm>=1.
|
|
15
|
+
Requires-Dist: litellm>=1.80.0
|
|
15
16
|
Requires-Dist: polib>=1.2.0
|
|
16
17
|
Requires-Dist: requests>=2.31.0
|
|
17
18
|
Requires-Dist: srt>=3.5.3
|
|
@@ -69,6 +70,7 @@ Configuration
|
|
|
69
70
|
TRANSLATIONS_GITHUB_TOKEN: <YOUR_GITHUB_TOKEN>
|
|
70
71
|
TRANSLATIONS_REPO_PATH: ""
|
|
71
72
|
TRANSLATIONS_REPO_URL: "https://github.com/mitodl/mitxonline-translations.git"
|
|
73
|
+
LITE_LLM_REQUEST_TIMEOUT: 120 # Timeout for LLM API requests in seconds
|
|
72
74
|
|
|
73
75
|
- For Tutor installations, these values can also be managed through a `custom Tutor plugin <https://docs.tutor.edly.io/tutorials/plugin.html#plugin-development-tutorial>`_.
|
|
74
76
|
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
ol_openedx_course_translations/__init__.py,sha256=qddojx2E0sw3CdZ_iyjwt3iTN_oTPCo1iPtRMSEgyHA,50
|
|
2
|
-
ol_openedx_course_translations/
|
|
3
|
-
ol_openedx_course_translations/
|
|
2
|
+
ol_openedx_course_translations/admin.py,sha256=MXtJCUydPjG6xoAc8oebtkQrmRZH-9IV8y6luzvrIlg,827
|
|
3
|
+
ol_openedx_course_translations/apps.py,sha256=__HhrdS6Tb4A0cVF9kZlJSs1JKhMeLNVnWXPx91k2oo,1057
|
|
4
|
+
ol_openedx_course_translations/filters.py,sha256=m7Gth09EWZelR_5pZGavmxczQ63fsFbCDbogl1-5lMY,1505
|
|
5
|
+
ol_openedx_course_translations/middleware.py,sha256=f5D6ZHlFhsdiOHForrPwHc--ZJWMygVhjXpFpwUJxMs,4608
|
|
6
|
+
ol_openedx_course_translations/models.py,sha256=muqwNYBtbON3Ru6Uq9krorYF1JmbIGOiBzWC-X7CBoU,1787
|
|
4
7
|
ol_openedx_course_translations/tasks.py,sha256=fxxd4mMV8HxtOY-y6kvM-oNjoif4iJE-V2aVyFua0OE,8071
|
|
5
8
|
ol_openedx_course_translations/urls.py,sha256=wDlOrRTtV_YRXAA1lpIhjEEf_fNkyfZUtmVPMD1PEi8,385
|
|
6
9
|
ol_openedx_course_translations/views.py,sha256=ESDmgSFAvyHKQE9s2hthyNgRZmMuLNWvfrGkR4isoec,2173
|
|
@@ -15,21 +18,23 @@ ol_openedx_course_translations/glossaries/machine_learning/ru.txt,sha256=d-p1Ucx
|
|
|
15
18
|
ol_openedx_course_translations/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
19
|
ol_openedx_course_translations/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
20
|
ol_openedx_course_translations/management/commands/sync_and_translate_language.py,sha256=E2sLaXYBTI1qCYcocV022Q-06y9iIA1YTMN5rHsb078,70176
|
|
18
|
-
ol_openedx_course_translations/management/commands/translate_course.py,sha256=
|
|
21
|
+
ol_openedx_course_translations/management/commands/translate_course.py,sha256=QFSR63sO07urXU9BWYj0iv4Xy1Xd94RkV5nvUjpJdeI,22633
|
|
22
|
+
ol_openedx_course_translations/migrations/0001_add_translation_logs.py,sha256=nitP-DJjDIeK9F3IcA1HvE5jpFaGCaoDP00XQxrjcSc,2876
|
|
23
|
+
ol_openedx_course_translations/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
24
|
ol_openedx_course_translations/providers/__init__.py,sha256=ZTWIG9KsYXYElfr-QwMsId7l6Sh6MypDKCWTivk3PME,48
|
|
20
25
|
ol_openedx_course_translations/providers/base.py,sha256=-1QV7w-6Wpwz4EMzCiOOuWOG2kDw4b2gZ1KSK6GFVLM,9264
|
|
21
26
|
ol_openedx_course_translations/providers/deepl_provider.py,sha256=hPuLVEZLU3Dm6a9H-8ztpcX53K9gUA8e57jUzn_HsKU,10076
|
|
22
|
-
ol_openedx_course_translations/providers/llm_providers.py,sha256=
|
|
27
|
+
ol_openedx_course_translations/providers/llm_providers.py,sha256=BIijQgVudTytxtrCvB-NTN9FvA8YPMFiB-77pAHjP-4,19953
|
|
23
28
|
ol_openedx_course_translations/settings/cms.py,sha256=SzoTxVUlvlWvjQ58GCoVygHjoMNAH3InME7T9K6dTQ4,392
|
|
24
|
-
ol_openedx_course_translations/settings/common.py,sha256=
|
|
25
|
-
ol_openedx_course_translations/settings/lms.py,sha256=
|
|
29
|
+
ol_openedx_course_translations/settings/common.py,sha256=NXrBo-GmL-nMZzQvDH-S7p8jy3SIR7spuOI8dZuexe0,1788
|
|
30
|
+
ol_openedx_course_translations/settings/lms.py,sha256=381QL_84D-GxD97rZcsUsdjoYRUqcxWbbrzndkliRRY,1320
|
|
26
31
|
ol_openedx_course_translations/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
32
|
ol_openedx_course_translations/utils/command_utils.py,sha256=GRSGIBKm6ZnRCnv5oVRzxhS6UFwvjmNeJkvCEpLCks0,6482
|
|
28
|
-
ol_openedx_course_translations/utils/constants.py,sha256=
|
|
29
|
-
ol_openedx_course_translations/utils/course_translations.py,sha256=
|
|
33
|
+
ol_openedx_course_translations/utils/constants.py,sha256=geIe61KP4N1piXw-kr8QuACkw79pd8T1xeROq17LHyA,8171
|
|
34
|
+
ol_openedx_course_translations/utils/course_translations.py,sha256=ZGh6YKNFR0MlNeYC1oCABCe0cbAWxLslpXj6jvO7WSs,21639
|
|
30
35
|
ol_openedx_course_translations/utils/translation_sync.py,sha256=IQL-auVk5GwSfUfyeGICUDhd1KPxZs_bthAzRYm9LSs,26739
|
|
31
|
-
ol_openedx_course_translations-0.3.
|
|
32
|
-
ol_openedx_course_translations-0.3.
|
|
33
|
-
ol_openedx_course_translations-0.3.
|
|
34
|
-
ol_openedx_course_translations-0.3.
|
|
35
|
-
ol_openedx_course_translations-0.3.
|
|
36
|
+
ol_openedx_course_translations-0.3.5.dist-info/METADATA,sha256=e08wH5Ldm9qovRG2QTCVGCrc-iG9qwr0Xc135sx0m4I,16828
|
|
37
|
+
ol_openedx_course_translations-0.3.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
38
|
+
ol_openedx_course_translations-0.3.5.dist-info/entry_points.txt,sha256=FPeAXhaGYIZqafskkDD8xPiljXJ6djil27_kTuNP0BI,239
|
|
39
|
+
ol_openedx_course_translations-0.3.5.dist-info/licenses/LICENSE.txt,sha256=iVk4rSDx0SwrA7AriwFblTY0vUxpuVib1oqJEEeejN0,1496
|
|
40
|
+
ol_openedx_course_translations-0.3.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|