edu-rdm-integration 3.21.0__py3-none-any.whl → 3.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/core/consts.py +3 -0
- edu_rdm_integration/pipelines/cleanup_outdated_data/management/commands/rdm_cleanup_outdated_data.py +32 -1
- edu_rdm_integration/pipelines/transfer/tasks.py +5 -0
- edu_rdm_integration/rdm_entities/models.py +5 -0
- edu_rdm_integration/rdm_models/models.py +29 -11
- edu_rdm_integration/stages/collect_data/functions/base/runners.py +7 -0
- edu_rdm_integration/stages/collect_data/migrations/0005_alter_rdmcollectingdatasubstage_previous.py +22 -0
- edu_rdm_integration/stages/collect_data/migrations/0006_fix_fk_constraints.py +59 -0
- edu_rdm_integration/stages/collect_data/models.py +5 -0
- edu_rdm_integration/stages/collect_data/registry/actions.py +4 -4
- edu_rdm_integration/stages/collect_data/registry/templates/ui-js/collect-command-window.js +8 -19
- edu_rdm_integration/stages/collect_data/registry/templates/ui-js/validators.js +0 -15
- edu_rdm_integration/stages/export_data/functions/base/runners.py +11 -0
- edu_rdm_integration/stages/export_data/migrations/0003_alter_rdmexportingdatasubstageattachment_exporting_data_sub_stage.py +22 -0
- edu_rdm_integration/stages/export_data/migrations/0004_fix_fk_constraints.py +76 -0
- edu_rdm_integration/stages/export_data/models.py +12 -1
- edu_rdm_integration/stages/export_data/registry/actions.py +5 -4
- edu_rdm_integration/stages/export_data/registry/templates/ui-js/create-export-command-win.js +15 -31
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/__init__.py +0 -0
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/base.py +305 -0
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/collect_data.py +119 -0
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/consts.py +1 -0
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/export_data.py +174 -0
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/upload_data.py +64 -0
- edu_rdm_integration/stages/service/service_outdated_data/managers.py +80 -0
- edu_rdm_integration/stages/upload_data/enums.py +2 -0
- edu_rdm_integration/stages/upload_data/export_managers.py +3 -1
- edu_rdm_integration/stages/upload_data/management/commands/custom_check_upload_status.py +59 -0
- edu_rdm_integration/stages/upload_data/management/commands/custom_upload_files.py +45 -0
- edu_rdm_integration/stages/upload_data/migrations/0003_auto_20251006_1417.py +28 -0
- edu_rdm_integration/stages/upload_data/migrations/0004_fix_fk_constraints.py +79 -0
- edu_rdm_integration/stages/upload_data/models.py +5 -2
- edu_rdm_integration/stages/upload_data/queues.py +50 -2
- edu_rdm_integration/stages/upload_data/tasks.py +2 -2
- edu_rdm_integration/stages/utils.py +61 -0
- edu_rdm_integration/templates/ui-js/collect-and-export-validators.js +54 -0
- {edu_rdm_integration-3.21.0.dist-info → edu_rdm_integration-3.23.0.dist-info}/METADATA +86 -59
- {edu_rdm_integration-3.21.0.dist-info → edu_rdm_integration-3.23.0.dist-info}/RECORD +41 -24
- {edu_rdm_integration-3.21.0.dist-info → edu_rdm_integration-3.23.0.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.21.0.dist-info → edu_rdm_integration-3.23.0.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.21.0.dist-info → edu_rdm_integration-3.23.0.dist-info}/top_level.txt +0 -0
edu_rdm_integration/pipelines/cleanup_outdated_data/management/commands/rdm_cleanup_outdated_data.py
CHANGED
|
@@ -11,6 +11,9 @@ from edu_rdm_integration.rdm_models.models import (
|
|
|
11
11
|
from edu_rdm_integration.stages.service.model_outdated_data.managers import (
|
|
12
12
|
ModelOutdatedDataCleanerManager,
|
|
13
13
|
)
|
|
14
|
+
from edu_rdm_integration.stages.service.service_outdated_data.managers import (
|
|
15
|
+
ServiceOutdatedDataCleanerManager,
|
|
16
|
+
)
|
|
14
17
|
|
|
15
18
|
|
|
16
19
|
class Command(BaseCommand):
|
|
@@ -33,10 +36,26 @@ class Command(BaseCommand):
|
|
|
33
36
|
'--models',
|
|
34
37
|
action='store',
|
|
35
38
|
dest='models',
|
|
36
|
-
type=lambda ml: [m.strip().upper() for m in ml.
|
|
39
|
+
type=lambda ml: [m.strip().upper() for m in ml.split(',')] if ml else None,
|
|
37
40
|
help=models_help_text,
|
|
38
41
|
)
|
|
39
42
|
|
|
43
|
+
# Аргумент для сервисных стадий
|
|
44
|
+
stages_help_text = (
|
|
45
|
+
'Значением параметра является перечисление стадий сервисных данных, '
|
|
46
|
+
'для которых должна быть произведена зачистка устаревших данных. '
|
|
47
|
+
'Перечисление этапов: collect, export, upload. '
|
|
48
|
+
'Если стадии не указываются, то зачистка будет произведена для всех стадий. '
|
|
49
|
+
'Стадии перечисляются через запятую без пробелов.'
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
'--stages',
|
|
53
|
+
action='store',
|
|
54
|
+
dest='stages',
|
|
55
|
+
type=lambda st: [s.strip().lower() for s in st.split(',')] if st else None,
|
|
56
|
+
help=stages_help_text,
|
|
57
|
+
)
|
|
58
|
+
|
|
40
59
|
parser.add_argument(
|
|
41
60
|
'--safe',
|
|
42
61
|
action='store_true',
|
|
@@ -62,7 +81,19 @@ class Command(BaseCommand):
|
|
|
62
81
|
)
|
|
63
82
|
model_data_cleaner_manager.run()
|
|
64
83
|
|
|
84
|
+
def _cleanup_service_outdated_data(self, options):
|
|
85
|
+
"""Очистка устаревших данных сервисных моделей РВД."""
|
|
86
|
+
service_data_cleaner_manager = ServiceOutdatedDataCleanerManager(
|
|
87
|
+
stages=options['stages'],
|
|
88
|
+
safe=options['safe'],
|
|
89
|
+
log_sql=options['log_sql'],
|
|
90
|
+
)
|
|
91
|
+
service_data_cleaner_manager.run()
|
|
92
|
+
|
|
65
93
|
def handle(self, *args, **options):
|
|
66
94
|
"""Запуск очистки устаревших данных РВД."""
|
|
67
95
|
if settings.RDM_ENABLE_CLEANUP_MODELS_OUTDATED_DATA:
|
|
68
96
|
self._cleanup_model_outdated_data(options)
|
|
97
|
+
|
|
98
|
+
if settings.RDM_ENABLE_CLEANUP_SERVICE_OUTDATED_DATA:
|
|
99
|
+
self._cleanup_service_outdated_data(options)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import time
|
|
1
2
|
from datetime import (
|
|
2
3
|
datetime,
|
|
3
4
|
)
|
|
@@ -37,6 +38,7 @@ from educommon.utils.date import (
|
|
|
37
38
|
from edu_rdm_integration.core.consts import (
|
|
38
39
|
FAST_TRANSFER_TASK_QUEUE_NAME,
|
|
39
40
|
LONG_TRANSFER_TASK_QUEUE_NAME,
|
|
41
|
+
PAUSE_TIME,
|
|
40
42
|
TASK_QUEUE_NAME,
|
|
41
43
|
)
|
|
42
44
|
from edu_rdm_integration.core.enums import (
|
|
@@ -206,6 +208,9 @@ class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataM
|
|
|
206
208
|
|
|
207
209
|
continue
|
|
208
210
|
|
|
211
|
+
# Лаг времени для достаки данных в реплику
|
|
212
|
+
time.sleep(PAUSE_TIME)
|
|
213
|
+
|
|
209
214
|
try:
|
|
210
215
|
if export_enabled:
|
|
211
216
|
self._run_export_entity_data(entity_enum.key, task_id)
|
|
@@ -70,6 +70,11 @@ class RDMEntityEnum(TitledModelEnum):
|
|
|
70
70
|
|
|
71
71
|
return model_enums
|
|
72
72
|
|
|
73
|
+
@classmethod
|
|
74
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
75
|
+
"""Возвращает список кортежей из ключей и ключей перечисления сущностей."""
|
|
76
|
+
return [(key, key) for key in sorted(cls.get_model_enum_keys())]
|
|
77
|
+
|
|
73
78
|
@classmethod
|
|
74
79
|
def extend(
|
|
75
80
|
cls,
|
|
@@ -35,15 +35,6 @@ from m3_db_utils.models import (
|
|
|
35
35
|
class BaseRDMModel(ReprStrPreModelMixin, BaseObjectModel):
|
|
36
36
|
"""Базовая модель РВД."""
|
|
37
37
|
|
|
38
|
-
collecting_sub_stage = ForeignKey(
|
|
39
|
-
verbose_name='Подэтап сбора данных',
|
|
40
|
-
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataSubStage',
|
|
41
|
-
on_delete=CASCADE,
|
|
42
|
-
)
|
|
43
|
-
operation = SmallIntegerField(
|
|
44
|
-
verbose_name='Действие',
|
|
45
|
-
choices=EntityLogOperation.get_choices(),
|
|
46
|
-
)
|
|
47
38
|
created = DateTimeField(
|
|
48
39
|
verbose_name='Дата создания',
|
|
49
40
|
auto_now_add=True,
|
|
@@ -68,10 +59,32 @@ class BaseRDMModel(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
68
59
|
abstract = True
|
|
69
60
|
|
|
70
61
|
|
|
71
|
-
class
|
|
62
|
+
class BaseAdditionalRDMModel(BaseRDMModel):
|
|
63
|
+
"""Абстрактная вспомогательная модель РВД.
|
|
64
|
+
|
|
65
|
+
Является базовым классом для моделей РВД, которые не являются основными для сущностей РВД. Для таких моделей
|
|
66
|
+
производится сбор данных.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
collecting_sub_stage = ForeignKey(
|
|
70
|
+
verbose_name='Подэтап сбора данных',
|
|
71
|
+
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataSubStage',
|
|
72
|
+
on_delete=CASCADE,
|
|
73
|
+
)
|
|
74
|
+
operation = SmallIntegerField(
|
|
75
|
+
verbose_name='Действие',
|
|
76
|
+
choices=EntityLogOperation.get_choices(),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
class Meta:
|
|
80
|
+
abstract = True
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class BaseMainRDMModel(BaseAdditionalRDMModel):
|
|
72
84
|
"""Абстрактная основная модель РВД.
|
|
73
85
|
|
|
74
|
-
Является базовым классом для моделей РВД, которые являются основными для сущностей РВД.
|
|
86
|
+
Является базовым классом для моделей РВД, которые являются основными для сущностей РВД. Для таких моделей
|
|
87
|
+
производится сбор и выгрузка данных.
|
|
75
88
|
"""
|
|
76
89
|
|
|
77
90
|
exporting_sub_stage = ForeignKey(
|
|
@@ -103,6 +116,11 @@ class RDMModelEnum(TitledModelEnum):
|
|
|
103
116
|
verbose_name = 'Модель-перечисление моделей "Региональной витрины данных"'
|
|
104
117
|
verbose_name_plural = 'Модели-перечисления моделей "Региональной витрины данных"'
|
|
105
118
|
|
|
119
|
+
@classmethod
|
|
120
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
121
|
+
"""Возвращает список кортежей из ключей и ключей перечисления моделей."""
|
|
122
|
+
return [(key, key) for key in sorted(cls.get_model_enum_keys())]
|
|
123
|
+
|
|
106
124
|
@classmethod
|
|
107
125
|
def _get_model_relations(cls, model: Type['BaseRDMModel']) -> dict[str, str]:
|
|
108
126
|
"""Получение списка связей модели РВД."""
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
from itertools import (
|
|
2
|
+
islice,
|
|
3
|
+
)
|
|
4
|
+
|
|
1
5
|
from django.conf import (
|
|
2
6
|
settings,
|
|
3
7
|
)
|
|
@@ -27,6 +31,9 @@ class BaseCollectingDataRunner(EduRunner):
|
|
|
27
31
|
size=settings.RDM_COLLECT_CHUNK_SIZE,
|
|
28
32
|
)
|
|
29
33
|
|
|
34
|
+
if settings.DEBUG and settings.RDM_COLLECT_CHUNKS_LIMIT:
|
|
35
|
+
raw_logs_chunks = islice(raw_logs_chunks, settings.RDM_COLLECT_CHUNKS_LIMIT)
|
|
36
|
+
|
|
30
37
|
for chunk_index, raw_logs_chunk in enumerate(raw_logs_chunks, start=1):
|
|
31
38
|
raw_logs = list(raw_logs_chunk)
|
|
32
39
|
for runnable_class in self._prepare_runnable_classes():
|
edu_rdm_integration/stages/collect_data/migrations/0005_alter_rdmcollectingdatasubstage_previous.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-10-08 14:17
|
|
2
|
+
|
|
3
|
+
import django.db.models.deletion
|
|
4
|
+
from django.db import (
|
|
5
|
+
migrations,
|
|
6
|
+
models,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Migration(migrations.Migration):
|
|
11
|
+
|
|
12
|
+
dependencies = [
|
|
13
|
+
('edu_rdm_integration_collect_data_stage', '0004_auto_20250704_0825'),
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
operations = [
|
|
17
|
+
migrations.AlterField(
|
|
18
|
+
model_name='rdmcollectingdatasubstage',
|
|
19
|
+
name='previous',
|
|
20
|
+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration_collect_data_stage.rdmcollectingdatasubstage', verbose_name='Предыдущий сбор данных'),
|
|
21
|
+
),
|
|
22
|
+
]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-10-08 14:43
|
|
2
|
+
|
|
3
|
+
from django.db import (
|
|
4
|
+
migrations,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
from edu_rdm_integration.stages.utils import (
|
|
8
|
+
update_foreign_key_constraint,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def apply_fk_updates(apps, schema_editor):
|
|
13
|
+
"""
|
|
14
|
+
Применяет обновления внешних ключей для конкретных моделей.
|
|
15
|
+
|
|
16
|
+
Здесь задаются таблицы и поведение при удалении записей,
|
|
17
|
+
чтобы синхронизировать фактическое состояние БД с логикой моделей.
|
|
18
|
+
"""
|
|
19
|
+
RDMCollectingDataSubStage = apps.get_model('edu_rdm_integration_collect_data_stage', 'RDMCollectingDataSubStage')
|
|
20
|
+
RDMCollectingDataStage = apps.get_model('edu_rdm_integration_collect_data_stage', 'RDMCollectingDataStage')
|
|
21
|
+
|
|
22
|
+
# Модель: RDMCollectingDataSubStage
|
|
23
|
+
# Поле previous_id должно иметь ON DELETE SET NULL
|
|
24
|
+
update_foreign_key_constraint(
|
|
25
|
+
table_name=RDMCollectingDataSubStage._meta.db_table,
|
|
26
|
+
field_name="previous_id",
|
|
27
|
+
target_table=RDMCollectingDataSubStage._meta.db_table,
|
|
28
|
+
on_delete="SET NULL",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
RDMCollectingDataCommandProgress = apps.get_model('edu_rdm_integration_collect_data_stage', 'RDMCollectingDataCommandProgress')
|
|
32
|
+
# Модель: RDMCollectingDataCommandProgress
|
|
33
|
+
# Поле stage_id должно иметь ON DELETE SET NULL
|
|
34
|
+
update_foreign_key_constraint(
|
|
35
|
+
table_name=RDMCollectingDataCommandProgress._meta.db_table,
|
|
36
|
+
field_name="stage_id",
|
|
37
|
+
target_table=RDMCollectingDataStage._meta.db_table,
|
|
38
|
+
on_delete="SET NULL",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# Таблица: rdm_collecting_data_command_progress
|
|
42
|
+
# Поле stage_id должно иметь ON DELETE SET NULL
|
|
43
|
+
update_foreign_key_constraint(
|
|
44
|
+
table_name='rdm_collecting_data_command_progress',
|
|
45
|
+
field_name="stage_id",
|
|
46
|
+
target_table=RDMCollectingDataStage._meta.db_table,
|
|
47
|
+
on_delete="SET NULL",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Migration(migrations.Migration):
|
|
52
|
+
|
|
53
|
+
dependencies = [
|
|
54
|
+
('edu_rdm_integration_collect_data_stage', '0005_alter_rdmcollectingdatasubstage_previous'),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
operations = [
|
|
58
|
+
migrations.RunPython(apply_fk_updates, reverse_code=migrations.RunPython.noop),
|
|
59
|
+
]
|
|
@@ -65,6 +65,11 @@ class RDMCollectingDataStageStatus(TitledModelEnum):
|
|
|
65
65
|
verbose_name = 'Модель-перечисление статусов этапа сбора данных'
|
|
66
66
|
verbose_name_plural = 'Модели-перечисления статусов этапов сбора данных'
|
|
67
67
|
|
|
68
|
+
@classmethod
|
|
69
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
70
|
+
"""Возвращает список кортежей из ключей и ключей перечисления статусов."""
|
|
71
|
+
return [(key, key) for key in cls.get_model_enum_keys()]
|
|
72
|
+
|
|
68
73
|
|
|
69
74
|
class RDMCollectingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
70
75
|
"""Этап подготовки данных в рамках Функций. За работу Функции отвечает ранер менеджер."""
|
|
@@ -83,7 +83,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
83
83
|
'header': 'Модель',
|
|
84
84
|
'sortable': True,
|
|
85
85
|
'filter': ChoicesFilter(
|
|
86
|
-
choices=
|
|
86
|
+
choices=partial(RDMModelEnum.get_choices),
|
|
87
87
|
parser=str,
|
|
88
88
|
lookup=lambda key: Q(model=key) if key else Q(),
|
|
89
89
|
),
|
|
@@ -93,7 +93,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
93
93
|
'header': 'Статус асинхронной задачи',
|
|
94
94
|
'sortable': True,
|
|
95
95
|
'filter': ChoicesFilter(
|
|
96
|
-
choices=
|
|
96
|
+
choices=partial(AsyncTaskStatus.get_choices),
|
|
97
97
|
parser=str,
|
|
98
98
|
lookup='task__status_id',
|
|
99
99
|
),
|
|
@@ -118,7 +118,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
118
118
|
'header': 'Статус сбора',
|
|
119
119
|
'sortable': True,
|
|
120
120
|
'filter': ChoicesFilter(
|
|
121
|
-
choices=
|
|
121
|
+
choices=partial(RDMCollectingDataStageStatus.get_choices),
|
|
122
122
|
parser=str,
|
|
123
123
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
124
124
|
),
|
|
@@ -278,4 +278,4 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
278
278
|
)
|
|
279
279
|
for command in commands_to_save
|
|
280
280
|
]
|
|
281
|
-
self.model.objects.bulk_create(objs, batch_size=settings.
|
|
281
|
+
self.model.objects.bulk_create(objs, batch_size=settings.RDM_COLLECT_PROGRESS_BATCH_SIZE)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
{% load educommon %}
|
|
2
2
|
{% include "ui-js/validators.js" %}
|
|
3
|
+
{% include "ui-js/collect-and-export-validators.js" %}
|
|
3
4
|
|
|
4
|
-
var
|
|
5
|
+
var logsPeriodStartField = Ext.getCmp('logs_period_started_at');
|
|
6
|
+
var logsPeriodEndField = Ext.getCmp('logs_period_ended_at');
|
|
5
7
|
|
|
6
8
|
Ext.onReady(function() {
|
|
7
9
|
// Инициализация валидаторов
|
|
@@ -41,24 +43,11 @@ function initializeValidators() {
|
|
|
41
43
|
});
|
|
42
44
|
}
|
|
43
45
|
|
|
44
|
-
// Устанавливаем
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
logsPeriodStartField.setMaxValue(now);
|
|
50
|
-
logsPeriodEndField.setMaxValue(now);
|
|
51
|
-
|
|
52
|
-
logsPeriodStartField.validate();
|
|
53
|
-
logsPeriodEndField.validate();
|
|
54
|
-
}, 1000);
|
|
55
|
-
|
|
56
|
-
win.on('destroy', function() {
|
|
57
|
-
clearInterval(validationInterval);
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
logsPeriodStartField.validator = logsPeriodsValidator;
|
|
61
|
-
logsPeriodEndField.validator = logsPeriodsValidator;
|
|
46
|
+
// Устанавливаем время по умолчанию в календаре и доп валидатор
|
|
47
|
+
setupPeriodFields(
|
|
48
|
+
logsPeriodStartField,
|
|
49
|
+
logsPeriodEndField,
|
|
50
|
+
);
|
|
62
51
|
}
|
|
63
52
|
|
|
64
53
|
function initializeBatchSizeSplitByLogic() {
|
|
@@ -47,20 +47,5 @@ var instituteCountValidator = function() {
|
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
this.clearInvalid();
|
|
50
|
-
return true;
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
var logsPeriodStartField = Ext.getCmp('logs_period_started_at');
|
|
54
|
-
var logsPeriodEndField = Ext.getCmp('logs_period_ended_at');
|
|
55
|
-
|
|
56
|
-
var logsPeriodsValidator = function () {
|
|
57
|
-
if (
|
|
58
|
-
logsPeriodStartField.getValue() &&
|
|
59
|
-
logsPeriodEndField.getValue() &&
|
|
60
|
-
logsPeriodStartField.getValue() > logsPeriodEndField.getValue()
|
|
61
|
-
) {
|
|
62
|
-
return 'Дата конца периода не может быть меньше даты начала периода'
|
|
63
|
-
};
|
|
64
|
-
|
|
65
50
|
return true;
|
|
66
51
|
};
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
from itertools import (
|
|
2
|
+
islice,
|
|
3
|
+
)
|
|
4
|
+
|
|
5
|
+
from django.conf import (
|
|
6
|
+
settings,
|
|
7
|
+
)
|
|
8
|
+
|
|
1
9
|
from edu_function_tools.runners import (
|
|
2
10
|
EduRunner,
|
|
3
11
|
)
|
|
@@ -41,6 +49,9 @@ class BaseExportDataRunner(EduRunner):
|
|
|
41
49
|
"""Возвращает генератор запускаемых объектов."""
|
|
42
50
|
model_ids_chunks = self._prepare_model_ids_chunks(*args, **kwargs)
|
|
43
51
|
|
|
52
|
+
if settings.DEBUG and settings.RDM_EXPORT_CHUNKS_LIMIT:
|
|
53
|
+
model_ids_chunks = islice(model_ids_chunks, settings.RDM_EXPORT_CHUNKS_LIMIT)
|
|
54
|
+
|
|
44
55
|
for chunk_index, model_ids_chunk in enumerate(model_ids_chunks, start=1):
|
|
45
56
|
for runnable_class in self._prepare_runnable_classes():
|
|
46
57
|
runnable = runnable_class(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-10-17 14:11
|
|
2
|
+
|
|
3
|
+
import django.db.models.deletion
|
|
4
|
+
from django.db import (
|
|
5
|
+
migrations,
|
|
6
|
+
models,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Migration(migrations.Migration):
|
|
11
|
+
|
|
12
|
+
dependencies = [
|
|
13
|
+
('edu_rdm_integration_export_data_stage', '0002_auto_20250704_0810'),
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
operations = [
|
|
17
|
+
migrations.AlterField(
|
|
18
|
+
model_name='rdmexportingdatasubstageattachment',
|
|
19
|
+
name='exporting_data_sub_stage',
|
|
20
|
+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration_export_data_stage.rdmexportingdatasubstage', verbose_name='Подэтап выгрузки данных'),
|
|
21
|
+
),
|
|
22
|
+
]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-10-17 14:15
|
|
2
|
+
|
|
3
|
+
from django.db import (
|
|
4
|
+
migrations,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
from edu_rdm_integration.stages.utils import (
|
|
8
|
+
update_foreign_key_constraint,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def apply_fk_updates(apps, schema_editor):
|
|
13
|
+
"""
|
|
14
|
+
Применяет обновления внешних ключей для конкретных моделей.
|
|
15
|
+
|
|
16
|
+
Здесь задаются таблицы и поведение при удалении записей,
|
|
17
|
+
чтобы синхронизировать фактическое состояние БД с логикой моделей.
|
|
18
|
+
"""
|
|
19
|
+
RDMExportingDataSubStage = apps.get_model('edu_rdm_integration_export_data_stage', 'RDMExportingDataSubStage')
|
|
20
|
+
RDMExportingDataStage = apps.get_model('edu_rdm_integration_export_data_stage', 'RDMExportingDataStage')
|
|
21
|
+
|
|
22
|
+
RDMExportingDataSubStageAttachment = apps.get_model(
|
|
23
|
+
'edu_rdm_integration_export_data_stage', 'RDMExportingDataSubStageAttachment'
|
|
24
|
+
)
|
|
25
|
+
# Модель: RDMExportingDataSubStageAttachment
|
|
26
|
+
# Поле exporting_data_sub_stage_id должно иметь ON DELETE CASCADE
|
|
27
|
+
update_foreign_key_constraint(
|
|
28
|
+
table_name=RDMExportingDataSubStageAttachment._meta.db_table,
|
|
29
|
+
field_name='exporting_data_sub_stage_id',
|
|
30
|
+
target_table=RDMExportingDataSubStage._meta.db_table,
|
|
31
|
+
on_delete='SET NULL',
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
RDMExportingDataSubStageEntity = apps.get_model(
|
|
35
|
+
'edu_rdm_integration_export_data_stage', 'RDMExportingDataSubStageEntity'
|
|
36
|
+
)
|
|
37
|
+
# Модель: CollectDataCommandProgress
|
|
38
|
+
# Поле exporting_data_sub_stage_id должно иметь ON DELETE CASCADE
|
|
39
|
+
update_foreign_key_constraint(
|
|
40
|
+
table_name=RDMExportingDataSubStageEntity._meta.db_table,
|
|
41
|
+
field_name='exporting_data_sub_stage_id',
|
|
42
|
+
target_table=RDMExportingDataSubStage._meta.db_table,
|
|
43
|
+
on_delete='CASCADE',
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
RDMExportingDataCommandProgress = apps.get_model(
|
|
47
|
+
'edu_rdm_integration_export_data_stage', 'RDMExportingDataCommandProgress'
|
|
48
|
+
)
|
|
49
|
+
# Модель: RDMExportingDataCommandProgress
|
|
50
|
+
# Поле stage_id должно иметь ON DELETE SET NULL
|
|
51
|
+
update_foreign_key_constraint(
|
|
52
|
+
table_name=RDMExportingDataCommandProgress._meta.db_table,
|
|
53
|
+
field_name='stage_id',
|
|
54
|
+
target_table=RDMExportingDataStage._meta.db_table,
|
|
55
|
+
on_delete='SET NULL',
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# Таблица: rdm_exporting_data_command_progress
|
|
59
|
+
# Поле stage_id должно иметь ON DELETE SET NULL
|
|
60
|
+
update_foreign_key_constraint(
|
|
61
|
+
table_name='rdm_exporting_data_command_progress',
|
|
62
|
+
field_name="stage_id",
|
|
63
|
+
target_table=RDMExportingDataStage._meta.db_table,
|
|
64
|
+
on_delete="SET NULL",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Migration(migrations.Migration):
|
|
69
|
+
|
|
70
|
+
dependencies = [
|
|
71
|
+
('edu_rdm_integration_export_data_stage', '0003_alter_rdmexportingdatasubstageattachment_exporting_data_sub_stage'),
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
operations = [
|
|
75
|
+
migrations.RunPython(apply_fk_updates, reverse_code=migrations.RunPython.noop),
|
|
76
|
+
]
|
|
@@ -69,6 +69,11 @@ class RDMExportingDataStageStatus(TitledModelEnum):
|
|
|
69
69
|
title='Завершено',
|
|
70
70
|
)
|
|
71
71
|
|
|
72
|
+
@classmethod
|
|
73
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
74
|
+
"""Возвращает список кортежей из ключей и ключей перечисления статусов."""
|
|
75
|
+
return [(key, key) for key in cls.get_model_enum_keys()]
|
|
76
|
+
|
|
72
77
|
class Meta:
|
|
73
78
|
db_table = 'rdm_exporting_data_stage_status'
|
|
74
79
|
verbose_name = 'Модель-перечисление статусов этапа выгрузки данных'
|
|
@@ -163,6 +168,10 @@ class RDMExportingDataSubStageStatus(TitledModelEnum):
|
|
|
163
168
|
)
|
|
164
169
|
PROCESS_ERROR = ModelEnumValue(title='Ошибка обработки витриной')
|
|
165
170
|
|
|
171
|
+
IN_EXPORT = ModelEnumValue(
|
|
172
|
+
title='Экспортируется',
|
|
173
|
+
)
|
|
174
|
+
|
|
166
175
|
class Meta:
|
|
167
176
|
db_table = 'rdm_exporting_data_sub_stage_status'
|
|
168
177
|
verbose_name = 'Модель-перечисление статусов подэтапа выгрузки данных'
|
|
@@ -248,7 +257,9 @@ class RDMExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
248
257
|
exporting_data_sub_stage = ForeignKey(
|
|
249
258
|
to=RDMExportingDataSubStage,
|
|
250
259
|
verbose_name='Подэтап выгрузки данных',
|
|
251
|
-
|
|
260
|
+
null=True,
|
|
261
|
+
blank=True,
|
|
262
|
+
on_delete=SET_NULL,
|
|
252
263
|
)
|
|
253
264
|
|
|
254
265
|
# TODO PYTD-22 В зависимости от принятого решения по инструменту ограничения доступа к media-файлам, нужно будет
|
|
@@ -86,7 +86,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
86
86
|
'header': 'Сущность',
|
|
87
87
|
'sortable': True,
|
|
88
88
|
'filter': ChoicesFilter(
|
|
89
|
-
choices=
|
|
89
|
+
choices=partial(RDMEntityEnum.get_choices),
|
|
90
90
|
parser=str,
|
|
91
91
|
lookup=lambda key: Q(entity=key) if key else Q(),
|
|
92
92
|
),
|
|
@@ -96,7 +96,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
96
96
|
'header': 'Статус асинхронной задачи',
|
|
97
97
|
'sortable': True,
|
|
98
98
|
'filter': ChoicesFilter(
|
|
99
|
-
choices=
|
|
99
|
+
choices=partial(AsyncTaskStatus.get_choices),
|
|
100
100
|
parser=str,
|
|
101
101
|
lookup='task__status_id',
|
|
102
102
|
),
|
|
@@ -131,7 +131,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
131
131
|
'header': 'Статус экспорта',
|
|
132
132
|
'sortable': True,
|
|
133
133
|
'filter': ChoicesFilter(
|
|
134
|
-
choices=
|
|
134
|
+
choices=partial(RDMExportingDataStageStatus.get_choices),
|
|
135
135
|
parser=str,
|
|
136
136
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
137
137
|
),
|
|
@@ -228,7 +228,8 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
228
228
|
ready_sub_stages=Subquery(
|
|
229
229
|
RDMExportingDataSubStage.objects.filter(
|
|
230
230
|
stage_id=OuterRef('stage_id'),
|
|
231
|
-
|
|
231
|
+
status__in=(RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key,
|
|
232
|
+
RDMExportingDataSubStageStatus.IN_EXPORT.key)
|
|
232
233
|
)
|
|
233
234
|
.annotate(
|
|
234
235
|
ready_count=Func(F('id'), function='Count', output_field=IntegerField()),
|
edu_rdm_integration/stages/export_data/registry/templates/ui-js/create-export-command-win.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
{% include "ui-js/collect-and-export-validators.js" %}
|
|
2
|
+
|
|
2
3
|
var periodStartField = Ext.getCmp('period_started_at');
|
|
3
4
|
var periodEndField = Ext.getCmp('period_ended_at');
|
|
4
5
|
|
|
@@ -8,34 +9,17 @@ Ext.onReady(function() {
|
|
|
8
9
|
});
|
|
9
10
|
|
|
10
11
|
function initializeValidators() {
|
|
11
|
-
//
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
clearInterval(validationInterval);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
var periodsValidator = function () {
|
|
28
|
-
if (
|
|
29
|
-
periodStartField.getValue() &&
|
|
30
|
-
periodEndField.getValue() &&
|
|
31
|
-
periodStartField.getValue() > periodEndField.getValue()
|
|
32
|
-
) {
|
|
33
|
-
return 'Дата конца периода не может быть меньше даты начала периода'
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
return true;
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
periodStartField.validator = periodsValidator;
|
|
40
|
-
periodEndField.validator = periodsValidator;
|
|
12
|
+
// Установка максимальной даты (завтра 23:59:59)
|
|
13
|
+
const maxDate = new Date();
|
|
14
|
+
maxDate.setDate(maxDate.getDate() + 1);
|
|
15
|
+
maxDate.setHours(23, 59, 59);
|
|
16
|
+
|
|
17
|
+
periodStartField.setMaxValue(maxDate);
|
|
18
|
+
periodEndField.setMaxValue(maxDate);
|
|
19
|
+
|
|
20
|
+
// Устанавливаем время по умолчанию в календаре и доп валидатор
|
|
21
|
+
setupPeriodFields(
|
|
22
|
+
periodStartField,
|
|
23
|
+
periodEndField,
|
|
24
|
+
);
|
|
41
25
|
}
|
|
File without changes
|