edu-rdm-integration 3.10.4__py3-none-any.whl → 3.11.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/adapters/apps.py +3 -3
- edu_rdm_integration/adapters/caches.py +4 -4
- edu_rdm_integration/adapters/errors.py +2 -2
- edu_rdm_integration/adapters/functions.py +9 -9
- edu_rdm_integration/adapters/helpers.py +7 -7
- edu_rdm_integration/adapters/receivers.py +3 -3
- edu_rdm_integration/adapters/runners.py +13 -13
- edu_rdm_integration/adapters/strategies.py +7 -7
- edu_rdm_integration/adapters/validators.py +6 -6
- edu_rdm_integration/collect_and_export_data/migrations/0003_auto_20250704_0725.py +37 -0
- edu_rdm_integration/collect_and_export_data/models.py +1 -1
- edu_rdm_integration/core/helpers.py +6 -4
- edu_rdm_integration/core/registry/apps.py +1 -1
- edu_rdm_integration/core/utils.py +3 -3
- edu_rdm_integration/migrations/0018_auto_20250704_0725.py +154 -0
- edu_rdm_integration/models.py +1 -0
- edu_rdm_integration/pipelines/transfer/actions.py +2 -2
- edu_rdm_integration/pipelines/transfer/mixins.py +2 -2
- edu_rdm_integration/pipelines/transfer/models.py +2 -2
- edu_rdm_integration/pipelines/transfer/tasks.py +8 -8
- edu_rdm_integration/rdm_entities/entities.py +6 -6
- edu_rdm_integration/rdm_entities/migrations/0002_rename_regionaldatamartentityenum_rdmentityenum.py +20 -0
- edu_rdm_integration/rdm_entities/mixins.py +2 -2
- edu_rdm_integration/rdm_entities/models.py +1 -1
- edu_rdm_integration/rdm_models/migrations/0002_rename_regionaldatamartmodelenum_rdmmodelenum.py +19 -0
- edu_rdm_integration/rdm_models/mixins.py +2 -2
- edu_rdm_integration/rdm_models/models.py +7 -7
- edu_rdm_integration/stages/collect_data/function_templates/function_collect_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/collect_data/functions/base/functions.py +9 -9
- edu_rdm_integration/stages/collect_data/functions/base/managers.py +6 -6
- edu_rdm_integration/stages/collect_data/functions/base/mixins.py +14 -16
- edu_rdm_integration/stages/collect_data/generators.py +3 -3
- edu_rdm_integration/stages/collect_data/helpers.py +18 -18
- edu_rdm_integration/stages/collect_data/management/base.py +2 -4
- edu_rdm_integration/stages/collect_data/migrations/0002_edurdmcollectdatacommandprogress.py +1 -0
- edu_rdm_integration/stages/collect_data/migrations/0003_auto_20250704_0810.py +35 -0
- edu_rdm_integration/stages/collect_data/migrations/0004_auto_20250704_0825.py +40 -0
- edu_rdm_integration/stages/collect_data/mixins.py +4 -4
- edu_rdm_integration/stages/collect_data/models.py +35 -64
- edu_rdm_integration/stages/collect_data/operations.py +7 -7
- edu_rdm_integration/stages/collect_data/registry/actions.py +10 -10
- edu_rdm_integration/stages/collect_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/collect_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/collect_data/tests.py +5 -5
- edu_rdm_integration/stages/export_data/function_templates/function_export_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/export_data/functions/base/functions.py +13 -13
- edu_rdm_integration/stages/export_data/functions/base/managers.py +10 -10
- edu_rdm_integration/stages/export_data/generators.py +3 -5
- edu_rdm_integration/stages/export_data/helpers.py +19 -19
- edu_rdm_integration/stages/export_data/management/base.py +2 -4
- edu_rdm_integration/stages/export_data/migrations/0001_initial.py +1 -0
- edu_rdm_integration/stages/export_data/migrations/0002_auto_20250704_0810.py +47 -0
- edu_rdm_integration/stages/export_data/mixins.py +5 -5
- edu_rdm_integration/stages/export_data/models.py +39 -52
- edu_rdm_integration/stages/export_data/operations.py +8 -10
- edu_rdm_integration/stages/export_data/registry/actions.py +18 -18
- edu_rdm_integration/stages/export_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/export_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/upload_data/consts.py +4 -4
- edu_rdm_integration/stages/upload_data/export_managers.py +23 -23
- edu_rdm_integration/stages/upload_data/helpers.py +10 -10
- edu_rdm_integration/stages/upload_data/management/commands/async_fix_attachment_size.py +2 -2
- edu_rdm_integration/stages/upload_data/management/commands/check_upload_status.py +2 -2
- edu_rdm_integration/stages/upload_data/migrations/0002_auto_20250704_0810.py +28 -0
- edu_rdm_integration/stages/upload_data/models.py +9 -9
- edu_rdm_integration/stages/upload_data/tasks.py +2 -2
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.1.dist-info}/METADATA +3 -2
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.1.dist-info}/RECORD +71 -62
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.1.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.1.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.1.dist-info}/top_level.txt +0 -0
|
@@ -59,7 +59,7 @@ if TYPE_CHECKING:
|
|
|
59
59
|
WebEduEntityCache,
|
|
60
60
|
)
|
|
61
61
|
from edu_rdm_integration.rdm_models.models import (
|
|
62
|
-
|
|
62
|
+
BaseRDMModel,
|
|
63
63
|
)
|
|
64
64
|
from edu_rdm_integration.stages.collect_data.functions.base.caches import (
|
|
65
65
|
IgnoreLogDependency,
|
|
@@ -243,21 +243,19 @@ class FilteredSaveEntitiesFunctionMixin(metaclass=ABCMeta):
|
|
|
243
243
|
|
|
244
244
|
Attributes:
|
|
245
245
|
_filtered_operations (tuple[EntityLogOperation]): : Операции, подлежащие фильтрации (по умолчанию только UPDATE)
|
|
246
|
-
_saved_entities_to_cache_map (dict[type[
|
|
246
|
+
_saved_entities_to_cache_map (dict[type[BaseRDMModel], WebEduEntityCache]): Копия маппинга моделей на кэши,
|
|
247
247
|
используемая для сравнения изменений
|
|
248
248
|
_ignored_fields (dict[EntityLogOperation, tuple[str]]): Словарь полей, которые следует игнорировать при
|
|
249
249
|
сравнении для каждой операции
|
|
250
250
|
"""
|
|
251
251
|
|
|
252
252
|
_filtered_operations: tuple[EntityLogOperation, ...] = (EntityLogOperation.UPDATE,)
|
|
253
|
-
_saved_entities_to_cache_map: dict[type['
|
|
253
|
+
_saved_entities_to_cache_map: dict[type['BaseRDMModel'], 'WebEduEntityCache'] = {}
|
|
254
254
|
_ignored_fields = {
|
|
255
255
|
EntityLogOperation.UPDATE: ('modified', 'operation', 'collecting_sub_stage', 'exporting_sub_stage'),
|
|
256
256
|
}
|
|
257
257
|
|
|
258
|
-
def _is_object_to_save(
|
|
259
|
-
self, saved_object: 'BaseEntityModel', ignore_fields: Optional[Iterable[str]] = None
|
|
260
|
-
) -> bool:
|
|
258
|
+
def _is_object_to_save(self, saved_object: 'BaseRDMModel', ignore_fields: Optional[Iterable[str]] = None) -> bool:
|
|
261
259
|
"""Проверяет, должен ли объект быть добавлен в очередь на сохранение.
|
|
262
260
|
|
|
263
261
|
Args:
|
|
@@ -282,8 +280,8 @@ class FilteredSaveEntitiesFunctionMixin(metaclass=ABCMeta):
|
|
|
282
280
|
|
|
283
281
|
def _is_save_object_modified(
|
|
284
282
|
self,
|
|
285
|
-
saved_object: '
|
|
286
|
-
original_object: '
|
|
283
|
+
saved_object: 'BaseRDMModel',
|
|
284
|
+
original_object: 'BaseRDMModel',
|
|
287
285
|
ignore_fields: Optional[Iterable[str]] = None,
|
|
288
286
|
) -> bool:
|
|
289
287
|
"""Проверяет, были ли изменения в указанном объекте относительно оригинала.
|
|
@@ -340,38 +338,38 @@ class FilteredSaveEntitiesFunctionMixin(metaclass=ABCMeta):
|
|
|
340
338
|
|
|
341
339
|
@property
|
|
342
340
|
@abstractmethod
|
|
343
|
-
def _model_to_cache_map(self) -> dict[type['
|
|
341
|
+
def _model_to_cache_map(self) -> dict[type['BaseRDMModel'], 'WebEduEntityCache']:
|
|
344
342
|
"""Абстрактное свойство, возвращающее соответствие между типами моделей и их кэшами.
|
|
345
343
|
|
|
346
344
|
Должно быть реализовано в дочернем классе. Используется для получения оригинальных данных объектов
|
|
347
345
|
перед сравнением.
|
|
348
346
|
|
|
349
347
|
Returns:
|
|
350
|
-
dict[type[
|
|
348
|
+
dict[type[BaseRDMModel], WebEduEntityCache]: Словарь, где ключ — это тип модели, а значение —
|
|
351
349
|
соответствующий кэш этой модели.
|
|
352
350
|
"""
|
|
353
351
|
|
|
354
352
|
@property
|
|
355
353
|
@abstractmethod
|
|
356
|
-
def _model_to_save_queue_map(self) -> dict[type['
|
|
354
|
+
def _model_to_save_queue_map(self) -> dict[type['BaseRDMModel'], dict]:
|
|
357
355
|
"""Абстрактное свойство, возвращающее соответствие между типами моделей и очередями сохранения.
|
|
358
356
|
|
|
359
357
|
Должно быть реализовано в дочернем классе. Используется для определения, в какую очередь будет добавляться
|
|
360
358
|
объект для сохранения.
|
|
361
359
|
|
|
362
360
|
Returns:
|
|
363
|
-
dict[type[
|
|
361
|
+
dict[type[BaseRDMModel], dict]: Словарь, где ключ — это тип модели, а значение — словарь, представляющий
|
|
364
362
|
очередь сохранения для этой модели.
|
|
365
363
|
"""
|
|
366
364
|
|
|
367
365
|
def _add_to_save_entities(
|
|
368
366
|
self,
|
|
369
|
-
save_object: '
|
|
367
|
+
save_object: 'BaseRDMModel',
|
|
370
368
|
operation: EntityLogOperation,
|
|
371
369
|
):
|
|
372
370
|
"""Добавление в очередь сущности на сохранение."""
|
|
373
371
|
try:
|
|
374
|
-
to_save_queue: dict[EntityLogOperation, dict[int, '
|
|
372
|
+
to_save_queue: dict[EntityLogOperation, dict[int, 'BaseRDMModel']] = self._model_to_save_queue_map[
|
|
375
373
|
type(save_object)
|
|
376
374
|
]
|
|
377
375
|
except KeyError:
|
|
@@ -386,11 +384,11 @@ class FilteredSaveEntitiesFunctionMixin(metaclass=ABCMeta):
|
|
|
386
384
|
if self._is_object_to_save(save_object, self._ignored_fields.get(operation)):
|
|
387
385
|
to_save_queue[EntityLogOperation.UPDATE][save_object.id] = save_object
|
|
388
386
|
|
|
389
|
-
def _get_entity_saver(self, model: '
|
|
387
|
+
def _get_entity_saver(self, model: 'BaseRDMModel') -> EntitySaver:
|
|
390
388
|
"""Возвращает экземпляр saver'а для указанной модели.
|
|
391
389
|
|
|
392
390
|
Args:
|
|
393
|
-
model (
|
|
391
|
+
model (BaseRDMModel): Тип модели, для которой требуется получить saver.
|
|
394
392
|
|
|
395
393
|
Returns:
|
|
396
394
|
EntitySaver: Объект, отвечающий за сохранение сущностей указанного типа.
|
|
@@ -34,7 +34,7 @@ from edu_rdm_integration.core.consts import (
|
|
|
34
34
|
DATE_FORMAT,
|
|
35
35
|
)
|
|
36
36
|
from edu_rdm_integration.rdm_models.models import (
|
|
37
|
-
|
|
37
|
+
RDMModelEnum,
|
|
38
38
|
)
|
|
39
39
|
from edu_rdm_integration.stages.collect_data.consts import (
|
|
40
40
|
ALL_UNITS_IN_COMMAND,
|
|
@@ -102,9 +102,9 @@ class BaseFirstCollectModelsDataCommandsGenerator:
|
|
|
102
102
|
):
|
|
103
103
|
"""Инициализация."""
|
|
104
104
|
# Если модели не указаны, берется значение по умолчанию - все модели:
|
|
105
|
-
models = models if models else
|
|
105
|
+
models = models if models else RDMModelEnum.get_enum_data().keys()
|
|
106
106
|
self.regional_data_mart_models: list[ModelEnumValue] = [
|
|
107
|
-
|
|
107
|
+
RDMModelEnum.get_model_enum_value(model) for model in models
|
|
108
108
|
]
|
|
109
109
|
|
|
110
110
|
self.logs_period_started_at = logs_period_started_at
|
|
@@ -33,16 +33,16 @@ from educommon import (
|
|
|
33
33
|
)
|
|
34
34
|
|
|
35
35
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
36
|
+
RDMCollectingDataStage,
|
|
37
|
+
RDMCollectingDataStageStatus,
|
|
38
|
+
RDMCollectingDataSubStage,
|
|
39
|
+
RDMCollectingDataSubStageStatus,
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
if TYPE_CHECKING:
|
|
44
44
|
from edu_rdm_integration.rdm_models.models import (
|
|
45
|
-
|
|
45
|
+
BaseRDMModel,
|
|
46
46
|
)
|
|
47
47
|
from edu_rdm_integration.stages.collect_data.functions.non_calculated.base.managers import (
|
|
48
48
|
BaseCollectingExportedDataRunnerManager,
|
|
@@ -66,10 +66,10 @@ def set_failed_status_suspended_collecting_data_stages() -> dict[str, int]:
|
|
|
66
66
|
suspended_time_at = current_datetime - timedelta(minutes=settings.RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT)
|
|
67
67
|
|
|
68
68
|
suspended_stage_ids = set(
|
|
69
|
-
|
|
69
|
+
RDMCollectingDataStage.objects.annotate(
|
|
70
70
|
last_sub_stage_started_at=Coalesce(
|
|
71
71
|
Subquery(
|
|
72
|
-
|
|
72
|
+
RDMCollectingDataSubStage.objects.filter(stage_id=OuterRef('pk'))
|
|
73
73
|
.values('started_at')
|
|
74
74
|
.order_by('-started_at')[:1]
|
|
75
75
|
),
|
|
@@ -79,27 +79,27 @@ def set_failed_status_suspended_collecting_data_stages() -> dict[str, int]:
|
|
|
79
79
|
.filter(
|
|
80
80
|
last_sub_stage_started_at__lt=suspended_time_at,
|
|
81
81
|
status__in=(
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
RDMCollectingDataStageStatus.CREATED.key,
|
|
83
|
+
RDMCollectingDataStageStatus.IN_PROGRESS.key,
|
|
84
84
|
),
|
|
85
85
|
)
|
|
86
86
|
.values_list('pk', flat=True)
|
|
87
87
|
)
|
|
88
88
|
|
|
89
89
|
if suspended_stage_ids:
|
|
90
|
-
logger.info(f'find suspended
|
|
90
|
+
logger.info(f'find suspended CollectingDataStage: {", ".join(map(str, suspended_stage_ids))}..')
|
|
91
91
|
|
|
92
|
-
change_stage_count =
|
|
92
|
+
change_stage_count = RDMCollectingDataStage.objects.filter(
|
|
93
93
|
pk__in=suspended_stage_ids,
|
|
94
94
|
).update(
|
|
95
|
-
status=
|
|
95
|
+
status=RDMCollectingDataStageStatus.FAILED.key,
|
|
96
96
|
ended_at=current_datetime,
|
|
97
97
|
)
|
|
98
98
|
|
|
99
|
-
change_sub_stage_count =
|
|
99
|
+
change_sub_stage_count = RDMCollectingDataSubStage.objects.filter(
|
|
100
100
|
stage_id__in=suspended_stage_ids,
|
|
101
101
|
).update(
|
|
102
|
-
status=
|
|
102
|
+
status=RDMCollectingDataSubStageStatus.FAILED.key,
|
|
103
103
|
ended_at=current_datetime,
|
|
104
104
|
)
|
|
105
105
|
|
|
@@ -118,12 +118,12 @@ def get_collecting_managers_max_period_ended_dates(
|
|
|
118
118
|
) -> dict[str, 'datetime']:
|
|
119
119
|
"""Возвращает дату и время завершения последнего успешного этапа сбора для менеджеров Функций сбора."""
|
|
120
120
|
managers_last_period_ended = (
|
|
121
|
-
|
|
121
|
+
RDMCollectingDataStage.objects.filter(
|
|
122
122
|
manager_id__in=[manager.uuid for manager in collecting_managers],
|
|
123
123
|
id=Subquery(
|
|
124
|
-
|
|
124
|
+
RDMCollectingDataStage.objects.filter(
|
|
125
125
|
manager_id=OuterRef('manager_id'),
|
|
126
|
-
status_id=
|
|
126
|
+
status_id=RDMCollectingDataStageStatus.FINISHED.key,
|
|
127
127
|
)
|
|
128
128
|
.order_by('-id')
|
|
129
129
|
.values('id')[:1]
|
|
@@ -142,7 +142,7 @@ def get_collecting_managers_max_period_ended_dates(
|
|
|
142
142
|
return {manager_id: last_period_ended_at for manager_id, last_period_ended_at in managers_last_period_ended}
|
|
143
143
|
|
|
144
144
|
|
|
145
|
-
def update_fields(entity: '
|
|
145
|
+
def update_fields(entity: 'BaseRDMModel', field_values: dict[str, Any], mapping: dict[str, str]) -> None:
|
|
146
146
|
"""Обновление значений полей сущности по измененным полям модели.
|
|
147
147
|
|
|
148
148
|
:param entity: Выгружаемая сущность
|
|
@@ -13,7 +13,7 @@ from edu_rdm_integration.core.consts import (
|
|
|
13
13
|
DATETIME_FORMAT,
|
|
14
14
|
)
|
|
15
15
|
from edu_rdm_integration.rdm_models.models import (
|
|
16
|
-
|
|
16
|
+
RDMModelEnum,
|
|
17
17
|
)
|
|
18
18
|
from edu_rdm_integration.stages.collect_data.operations import (
|
|
19
19
|
BaseCollectModelsData,
|
|
@@ -25,9 +25,7 @@ class BaseCollectModelDataCommand(BaseCommand):
|
|
|
25
25
|
|
|
26
26
|
def add_arguments(self, parser: 'CommandParser'):
|
|
27
27
|
"""Добавление параметров."""
|
|
28
|
-
models = ', '.join(
|
|
29
|
-
[f'{key} - {value.title}' for key, value in RegionalDataMartModelEnum.get_enum_data().items()]
|
|
30
|
-
)
|
|
28
|
+
models = ', '.join([f'{key} - {value.title}' for key, value in RDMModelEnum.get_enum_data().items()])
|
|
31
29
|
models_help_text = (
|
|
32
30
|
f'Значением параметра является перечисление моделей РВД, для которых должен быть произведен сбор данных. '
|
|
33
31
|
f'Перечисление моделей:\n{models}. Если модели не указываются, то сбор данных производится для всех '
|
|
@@ -19,6 +19,7 @@ class Migration(migrations.Migration):
|
|
|
19
19
|
('edu_rdm_integration_models', '0001_initial'),
|
|
20
20
|
('async_task', '0003_alter_runningtask_options'),
|
|
21
21
|
('edu_rdm_integration_collect_data_stage', '0001_initial'),
|
|
22
|
+
('rdm_collect_and_export_data', '0003_auto_20250704_0725'),
|
|
22
23
|
]
|
|
23
24
|
|
|
24
25
|
operations = [
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-07-04 08:10
|
|
2
|
+
|
|
3
|
+
from django.db import (
|
|
4
|
+
migrations,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Migration(migrations.Migration):
|
|
9
|
+
dependencies = [
|
|
10
|
+
('function_tools', '0005_auto_20220724_0050'),
|
|
11
|
+
('edu_rdm_integration_collect_data_stage', '0002_edurdmcollectdatacommandprogress'),
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
operations = [
|
|
15
|
+
migrations.RenameModel(
|
|
16
|
+
old_name='CollectingDataStageStatus',
|
|
17
|
+
new_name='RDMCollectingDataStageStatus',
|
|
18
|
+
),
|
|
19
|
+
migrations.RenameModel(
|
|
20
|
+
old_name='CollectingExportedDataStage',
|
|
21
|
+
new_name='RDMCollectingDataStage',
|
|
22
|
+
),
|
|
23
|
+
migrations.RenameModel(
|
|
24
|
+
old_name='CollectingDataSubStageStatus',
|
|
25
|
+
new_name='RDMCollectingDataSubStageStatus',
|
|
26
|
+
),
|
|
27
|
+
migrations.RenameModel(
|
|
28
|
+
old_name='CollectingExportedDataSubStage',
|
|
29
|
+
new_name='RDMCollectingDataSubStage',
|
|
30
|
+
),
|
|
31
|
+
migrations.RenameModel(
|
|
32
|
+
old_name='EduRdmCollectDataCommandProgress',
|
|
33
|
+
new_name='RDMCollectingDataCommandProgress',
|
|
34
|
+
),
|
|
35
|
+
]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-07-04 08:25
|
|
2
|
+
|
|
3
|
+
import django.db.models.deletion
|
|
4
|
+
from django.db import (
|
|
5
|
+
migrations,
|
|
6
|
+
models,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Migration(migrations.Migration):
|
|
11
|
+
dependencies = [
|
|
12
|
+
('edu_rdm_integration_collect_data_stage', '0003_auto_20250704_0810'),
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
operations = [
|
|
16
|
+
migrations.AlterModelOptions(
|
|
17
|
+
name='rdmcollectingdatasubstage',
|
|
18
|
+
options={'verbose_name': 'Подэтап сбора данных', 'verbose_name_plural': 'Подэтапы сбора данных'},
|
|
19
|
+
),
|
|
20
|
+
migrations.AlterField(
|
|
21
|
+
model_name='rdmcollectingdatacommandprogress',
|
|
22
|
+
name='stage',
|
|
23
|
+
field=models.ForeignKey(
|
|
24
|
+
blank=True,
|
|
25
|
+
null=True,
|
|
26
|
+
on_delete=django.db.models.deletion.SET_NULL,
|
|
27
|
+
to='edu_rdm_integration_collect_data_stage.rdmcollectingdatastage',
|
|
28
|
+
verbose_name='Этап сбора данных',
|
|
29
|
+
),
|
|
30
|
+
),
|
|
31
|
+
migrations.AlterField(
|
|
32
|
+
model_name='rdmcollectingdatasubstage',
|
|
33
|
+
name='stage',
|
|
34
|
+
field=models.ForeignKey(
|
|
35
|
+
on_delete=django.db.models.deletion.PROTECT,
|
|
36
|
+
to='edu_rdm_integration_collect_data_stage.rdmcollectingdatastage',
|
|
37
|
+
verbose_name='Этап сбора данных',
|
|
38
|
+
),
|
|
39
|
+
),
|
|
40
|
+
]
|
|
@@ -9,7 +9,7 @@ from edu_rdm_integration.core.helpers import (
|
|
|
9
9
|
save_command_log_link,
|
|
10
10
|
)
|
|
11
11
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
12
|
-
|
|
12
|
+
RDMCollectingDataCommandProgress,
|
|
13
13
|
)
|
|
14
14
|
from edu_rdm_integration.stages.collect_data.operations import (
|
|
15
15
|
BaseCollectModelsDataByGeneratingLogs,
|
|
@@ -24,9 +24,9 @@ class CollectCommandMixin:
|
|
|
24
24
|
description = 'Сбор данных моделей РВД'
|
|
25
25
|
task_type = AsyncTaskType.SYSTEM
|
|
26
26
|
|
|
27
|
-
def get_collect_command(self, command_id: int) ->
|
|
27
|
+
def get_collect_command(self, command_id: int) -> RDMCollectingDataCommandProgress:
|
|
28
28
|
"""Возвращает экземпляр модели команды запуска."""
|
|
29
|
-
command =
|
|
29
|
+
command = RDMCollectingDataCommandProgress.objects.get(id=command_id)
|
|
30
30
|
|
|
31
31
|
return command
|
|
32
32
|
|
|
@@ -49,7 +49,7 @@ class CollectCommandMixin:
|
|
|
49
49
|
"""Сохранение ссылки на файл логов в команде."""
|
|
50
50
|
try:
|
|
51
51
|
command = self.get_collect_command(command_id)
|
|
52
|
-
except
|
|
52
|
+
except RDMCollectingDataCommandProgress.DoesNotExist:
|
|
53
53
|
command = None
|
|
54
54
|
|
|
55
55
|
if command:
|
|
@@ -44,7 +44,7 @@ from edu_rdm_integration.core.utils import (
|
|
|
44
44
|
)
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class
|
|
47
|
+
class RDMCollectingDataStageStatus(TitledModelEnum):
|
|
48
48
|
"""Статус этапа сбора данных."""
|
|
49
49
|
|
|
50
50
|
CREATED = ModelEnumValue(
|
|
@@ -69,7 +69,7 @@ class CollectingDataStageStatus(TitledModelEnum):
|
|
|
69
69
|
verbose_name_plural = 'Модели-перечисления статусов этапов сбора данных'
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
class
|
|
72
|
+
class RDMCollectingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
73
73
|
"""Этап подготовки данных в рамках Функций. За работу Функции отвечает ранер менеджер."""
|
|
74
74
|
|
|
75
75
|
manager = ForeignKey(
|
|
@@ -106,10 +106,10 @@ class CollectingExportedDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
106
106
|
)
|
|
107
107
|
|
|
108
108
|
status = ForeignKey(
|
|
109
|
-
to='edu_rdm_integration_collect_data_stage.
|
|
109
|
+
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataStageStatus',
|
|
110
110
|
verbose_name='Статус',
|
|
111
111
|
on_delete=PROTECT,
|
|
112
|
-
default=
|
|
112
|
+
default=RDMCollectingDataStageStatus.CREATED.key,
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
class Meta:
|
|
@@ -125,7 +125,7 @@ class CollectingExportedDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
125
125
|
def save(self, *args, **kwargs):
|
|
126
126
|
"""Сохранение этапа сбора данных модели РВД."""
|
|
127
127
|
if (
|
|
128
|
-
self.status_id in (
|
|
128
|
+
self.status_id in (RDMCollectingDataStageStatus.FAILED.key, RDMCollectingDataStageStatus.FINISHED.key)
|
|
129
129
|
and not self.ended_at
|
|
130
130
|
):
|
|
131
131
|
self.ended_at = datetime.now()
|
|
@@ -133,7 +133,7 @@ class CollectingExportedDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
133
133
|
super().save(*args, **kwargs)
|
|
134
134
|
|
|
135
135
|
|
|
136
|
-
class
|
|
136
|
+
class RDMCollectingDataSubStageStatus(TitledModelEnum):
|
|
137
137
|
"""Статус этапа сбора данных."""
|
|
138
138
|
|
|
139
139
|
CREATED = ModelEnumValue(
|
|
@@ -166,12 +166,12 @@ class CollectingDataSubStageStatus(TitledModelEnum):
|
|
|
166
166
|
verbose_name_plural = 'Модели-перечисления статусов подэтапов сбора данных'
|
|
167
167
|
|
|
168
168
|
|
|
169
|
-
class
|
|
169
|
+
class RDMCollectingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
170
170
|
"""Подэтап сбора данных для сущностей в рамках функции."""
|
|
171
171
|
|
|
172
172
|
stage = ForeignKey(
|
|
173
|
-
to=
|
|
174
|
-
verbose_name='Этап
|
|
173
|
+
to=RDMCollectingDataStage,
|
|
174
|
+
verbose_name='Этап сбора данных',
|
|
175
175
|
on_delete=PROTECT,
|
|
176
176
|
)
|
|
177
177
|
|
|
@@ -203,16 +203,16 @@ class CollectingExportedDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
203
203
|
)
|
|
204
204
|
|
|
205
205
|
status = ForeignKey(
|
|
206
|
-
to='edu_rdm_integration_collect_data_stage.
|
|
206
|
+
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataSubStageStatus',
|
|
207
207
|
verbose_name='Статус',
|
|
208
208
|
on_delete=PROTECT,
|
|
209
|
-
default=
|
|
209
|
+
default=RDMCollectingDataSubStageStatus.CREATED.key,
|
|
210
210
|
)
|
|
211
211
|
|
|
212
212
|
class Meta:
|
|
213
213
|
db_table = 'rdm_collecting_exported_data_sub_stage'
|
|
214
|
-
verbose_name = 'Подэтап
|
|
215
|
-
verbose_name_plural = 'Подэтапы
|
|
214
|
+
verbose_name = 'Подэтап сбора данных'
|
|
215
|
+
verbose_name_plural = 'Подэтапы сбора данных'
|
|
216
216
|
|
|
217
217
|
@property
|
|
218
218
|
def attrs_for_repr_str(self):
|
|
@@ -223,7 +223,7 @@ class CollectingExportedDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
223
223
|
"""Сохранение подэтапа сбора данных."""
|
|
224
224
|
if (
|
|
225
225
|
self.status_id
|
|
226
|
-
in (
|
|
226
|
+
in (RDMCollectingDataSubStageStatus.FAILED.key, RDMCollectingDataSubStageStatus.READY_TO_EXPORT.key)
|
|
227
227
|
and not self.ended_at
|
|
228
228
|
):
|
|
229
229
|
self.ended_at = datetime.now()
|
|
@@ -231,38 +231,34 @@ class CollectingExportedDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
231
231
|
super().save(*args, **kwargs)
|
|
232
232
|
|
|
233
233
|
|
|
234
|
-
class
|
|
235
|
-
"""Модель, хранящая данные для формирования и отслеживания асинхронных задач по сбору данных.
|
|
236
|
-
|
|
237
|
-
В реализации необходимо определить поля:
|
|
238
|
-
1. Ссылку на асинхронную задачу, например:
|
|
239
|
-
task = ForeignKey(
|
|
240
|
-
to=RunningTask,
|
|
241
|
-
verbose_name='Асинхронная задача',
|
|
242
|
-
blank=True, null=True,
|
|
243
|
-
on_delete=SET_NULL,
|
|
244
|
-
)
|
|
245
|
-
2. Поле хранения лога:
|
|
246
|
-
logs_link = FileField(
|
|
247
|
-
upload_to=upload_file_handler,
|
|
248
|
-
max_length=255,
|
|
249
|
-
verbose_name='Ссылка на файл логов',
|
|
250
|
-
)
|
|
251
|
-
"""
|
|
252
|
-
|
|
253
|
-
task = ...
|
|
254
|
-
|
|
255
|
-
logs_link = ...
|
|
234
|
+
class RDMCollectingDataCommandProgress(ReprStrPreModelMixin, BaseObjectModel):
|
|
235
|
+
"""Модель, хранящая данные для формирования и отслеживания асинхронных задач по сбору данных."""
|
|
256
236
|
|
|
237
|
+
task = ForeignKey(
|
|
238
|
+
to='async_task.RunningTask',
|
|
239
|
+
verbose_name='Асинхронная задача',
|
|
240
|
+
blank=True,
|
|
241
|
+
null=True,
|
|
242
|
+
on_delete=SET_NULL,
|
|
243
|
+
)
|
|
244
|
+
logs_link = FileField(
|
|
245
|
+
upload_to=get_data_command_progress_attachment_path,
|
|
246
|
+
max_length=255,
|
|
247
|
+
verbose_name='Ссылка на файл логов',
|
|
248
|
+
)
|
|
249
|
+
type = PositiveSmallIntegerField( # noqa: A003
|
|
250
|
+
verbose_name='Тип команды',
|
|
251
|
+
choices=CommandType.get_choices(),
|
|
252
|
+
)
|
|
257
253
|
stage = ForeignKey(
|
|
258
|
-
to='edu_rdm_integration_collect_data_stage.
|
|
259
|
-
verbose_name='Этап
|
|
254
|
+
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataStage',
|
|
255
|
+
verbose_name='Этап сбора данных',
|
|
260
256
|
blank=True,
|
|
261
257
|
null=True,
|
|
262
258
|
on_delete=SET_NULL,
|
|
263
259
|
)
|
|
264
260
|
model = ForeignKey(
|
|
265
|
-
to='edu_rdm_integration_models.
|
|
261
|
+
to='edu_rdm_integration_models.RDMModelEnum',
|
|
266
262
|
verbose_name='Модель РВД',
|
|
267
263
|
on_delete=PROTECT,
|
|
268
264
|
)
|
|
@@ -288,31 +284,6 @@ class AbstractCollectDataCommandProgress(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
288
284
|
)
|
|
289
285
|
|
|
290
286
|
class Meta:
|
|
291
|
-
abstract = True
|
|
292
|
-
db_table = 'rdm_collecting_data_command_progress'
|
|
293
287
|
verbose_name = 'Задача по сбору данных'
|
|
294
288
|
verbose_name_plural = 'Задачи по сбору данных'
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
class EduRdmCollectDataCommandProgress(AbstractCollectDataCommandProgress):
|
|
298
|
-
"""Модель, хранящая данные для формирования и отслеживания асинхронных задач по сбору данных."""
|
|
299
|
-
|
|
300
|
-
task = ForeignKey(
|
|
301
|
-
to='async_task.RunningTask',
|
|
302
|
-
verbose_name='Асинхронная задача',
|
|
303
|
-
blank=True,
|
|
304
|
-
null=True,
|
|
305
|
-
on_delete=SET_NULL,
|
|
306
|
-
)
|
|
307
|
-
logs_link = FileField(
|
|
308
|
-
upload_to=get_data_command_progress_attachment_path,
|
|
309
|
-
max_length=255,
|
|
310
|
-
verbose_name='Ссылка на файл логов',
|
|
311
|
-
)
|
|
312
|
-
type = PositiveSmallIntegerField( # noqa: A003
|
|
313
|
-
verbose_name='Тип команды',
|
|
314
|
-
choices=CommandType.get_choices(),
|
|
315
|
-
)
|
|
316
|
-
|
|
317
|
-
class Meta(AbstractCollectDataCommandProgress.Meta):
|
|
318
289
|
db_table = 'edu_rdm_collecting_data_command_progress'
|
|
@@ -46,14 +46,14 @@ from edu_rdm_integration.core.storages import (
|
|
|
46
46
|
RegionalDataMartEntityStorage,
|
|
47
47
|
)
|
|
48
48
|
from edu_rdm_integration.rdm_models.models import (
|
|
49
|
-
|
|
49
|
+
RDMModelEnum,
|
|
50
50
|
)
|
|
51
51
|
from edu_rdm_integration.stages.collect_data.helpers import (
|
|
52
52
|
get_collecting_managers_max_period_ended_dates,
|
|
53
53
|
)
|
|
54
54
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
RDMCollectingDataStage,
|
|
56
|
+
RDMCollectingDataStageStatus,
|
|
57
57
|
)
|
|
58
58
|
|
|
59
59
|
|
|
@@ -80,8 +80,8 @@ class BaseCollectModelsData(BaseOperationData):
|
|
|
80
80
|
super().__init__(**kwargs)
|
|
81
81
|
|
|
82
82
|
# Если модели не указаны, берется значение по умолчанию - все модели:
|
|
83
|
-
models = models if models else
|
|
84
|
-
self.models: list[ModelEnumValue] = [
|
|
83
|
+
models = models if models else RDMModelEnum.get_enum_data().keys()
|
|
84
|
+
self.models: list[ModelEnumValue] = [RDMModelEnum.get_model_enum_value(model) for model in models]
|
|
85
85
|
|
|
86
86
|
self.logs_period_started_at = logs_period_started_at
|
|
87
87
|
self.logs_period_ended_at = logs_period_ended_at
|
|
@@ -241,9 +241,9 @@ class BaseCollectLatestModelsData(BaseCollectModelsData):
|
|
|
241
241
|
|
|
242
242
|
def _has_stage_created_or_in_progress(self, manager_id: str, model: str) -> bool:
|
|
243
243
|
"""Проверяет есть ли готовый к работе stage или в работе для данной модели."""
|
|
244
|
-
stage_created_or_in_progress =
|
|
244
|
+
stage_created_or_in_progress = RDMCollectingDataStage.objects.filter(
|
|
245
245
|
manager_id=manager_id,
|
|
246
|
-
status_id__in=(
|
|
246
|
+
status_id__in=(RDMCollectingDataStageStatus.CREATED.key, RDMCollectingDataStageStatus.IN_PROGRESS.key),
|
|
247
247
|
).exists()
|
|
248
248
|
|
|
249
249
|
if stage_created_or_in_progress:
|
|
@@ -41,16 +41,16 @@ from edu_rdm_integration.core.registry.actions import (
|
|
|
41
41
|
BaseStartTaskAction,
|
|
42
42
|
)
|
|
43
43
|
from edu_rdm_integration.rdm_models.models import (
|
|
44
|
-
|
|
44
|
+
RDMModelEnum,
|
|
45
45
|
)
|
|
46
46
|
from edu_rdm_integration.stages.collect_data.generators import (
|
|
47
47
|
FirstCollectModelsDataCommandsGenerator,
|
|
48
48
|
)
|
|
49
49
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
RDMCollectingDataCommandProgress,
|
|
51
|
+
RDMCollectingDataStageStatus,
|
|
52
|
+
RDMCollectingDataSubStage,
|
|
53
|
+
RDMCollectingDataSubStageStatus,
|
|
54
54
|
)
|
|
55
55
|
from edu_rdm_integration.stages.collect_data.registry.ui import (
|
|
56
56
|
CreateCollectCommandWindow,
|
|
@@ -62,7 +62,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
62
62
|
"""Базовый пак команд сбора данных моделей РВД."""
|
|
63
63
|
|
|
64
64
|
title = 'Сбор данных моделей РВД'
|
|
65
|
-
model =
|
|
65
|
+
model = RDMCollectingDataCommandProgress
|
|
66
66
|
|
|
67
67
|
add_window = CreateCollectCommandWindow
|
|
68
68
|
edit_window = DetailCollectCommandWindow
|
|
@@ -80,7 +80,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
80
80
|
'header': 'Модель',
|
|
81
81
|
'sortable': True,
|
|
82
82
|
'filter': ChoicesFilter(
|
|
83
|
-
choices=[(key, key) for key in
|
|
83
|
+
choices=[(key, key) for key in RDMModelEnum.get_model_enum_keys()],
|
|
84
84
|
parser=str,
|
|
85
85
|
lookup=lambda key: Q(model=key) if key else Q(),
|
|
86
86
|
),
|
|
@@ -115,7 +115,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
115
115
|
'header': 'Статус сбора',
|
|
116
116
|
'sortable': True,
|
|
117
117
|
'filter': ChoicesFilter(
|
|
118
|
-
choices=[(key, key) for key in
|
|
118
|
+
choices=[(key, key) for key in RDMCollectingDataStageStatus.get_model_enum_keys()],
|
|
119
119
|
parser=str,
|
|
120
120
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
121
121
|
),
|
|
@@ -206,9 +206,9 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
206
206
|
# Необходимо также рассчитать прогресс сбора:
|
|
207
207
|
query = query.annotate(
|
|
208
208
|
ready_to_export_sub_stages=Subquery(
|
|
209
|
-
|
|
209
|
+
RDMCollectingDataSubStage.objects.filter(
|
|
210
210
|
stage_id=OuterRef('stage_id'),
|
|
211
|
-
status_id=
|
|
211
|
+
status_id=RDMCollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
|
212
212
|
)
|
|
213
213
|
.annotate(ready_to_export_sub_stages=Func(F('id'), function='Count'))
|
|
214
214
|
.values('ready_to_export_sub_stages')
|
|
@@ -8,4 +8,4 @@ class RDMCollectDataRegistryAppConfig(AppConfig):
|
|
|
8
8
|
|
|
9
9
|
name = 'edu_rdm_integration.stages.collect_data.registry'
|
|
10
10
|
label = 'edu_rdm_integration_collect_data_registry'
|
|
11
|
-
verbose_name = 'Сбор данных моделей РВД из интерфейса'
|
|
11
|
+
verbose_name = 'Сбор данных моделей РВД из интерфейса'
|