edu-rdm-integration 3.10.3__py3-none-any.whl → 3.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of edu-rdm-integration might be problematic. Click here for more details.
- edu_rdm_integration/adapters/apps.py +3 -3
- edu_rdm_integration/adapters/caches.py +4 -4
- edu_rdm_integration/adapters/errors.py +2 -2
- edu_rdm_integration/adapters/functions.py +9 -9
- edu_rdm_integration/adapters/helpers.py +7 -7
- edu_rdm_integration/adapters/receivers.py +3 -3
- edu_rdm_integration/adapters/runners.py +13 -13
- edu_rdm_integration/adapters/strategies.py +7 -7
- edu_rdm_integration/adapters/validators.py +6 -6
- edu_rdm_integration/collect_and_export_data/migrations/0003_auto_20250704_0725.py +37 -0
- edu_rdm_integration/collect_and_export_data/models.py +1 -1
- edu_rdm_integration/core/helpers.py +6 -4
- edu_rdm_integration/core/registry/apps.py +1 -1
- edu_rdm_integration/core/utils.py +3 -3
- edu_rdm_integration/migrations/0018_auto_20250704_0725.py +154 -0
- edu_rdm_integration/pipelines/transfer/actions.py +2 -2
- edu_rdm_integration/pipelines/transfer/mixins.py +2 -2
- edu_rdm_integration/pipelines/transfer/models.py +2 -2
- edu_rdm_integration/pipelines/transfer/tasks.py +8 -8
- edu_rdm_integration/rdm_entities/entities.py +6 -6
- edu_rdm_integration/rdm_entities/migrations/0001_initial.py +1 -1
- edu_rdm_integration/rdm_entities/migrations/0002_rename_regionaldatamartentityenum_rdmentityenum.py +20 -0
- edu_rdm_integration/rdm_entities/mixins.py +2 -2
- edu_rdm_integration/rdm_entities/models.py +1 -1
- edu_rdm_integration/rdm_models/migrations/0001_initial.py +1 -1
- edu_rdm_integration/rdm_models/migrations/0002_rename_regionaldatamartmodelenum_rdmmodelenum.py +19 -0
- edu_rdm_integration/rdm_models/mixins.py +2 -2
- edu_rdm_integration/rdm_models/models.py +7 -7
- edu_rdm_integration/stages/collect_data/function_templates/function_collect_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/collect_data/functions/base/functions.py +9 -9
- edu_rdm_integration/stages/collect_data/functions/base/managers.py +6 -6
- edu_rdm_integration/stages/collect_data/functions/base/mixins.py +14 -16
- edu_rdm_integration/stages/collect_data/generators.py +3 -3
- edu_rdm_integration/stages/collect_data/helpers.py +18 -18
- edu_rdm_integration/stages/collect_data/management/base.py +2 -4
- edu_rdm_integration/stages/collect_data/migrations/0001_initial.py +1 -1
- edu_rdm_integration/stages/collect_data/migrations/0002_edurdmcollectdatacommandprogress.py +1 -0
- edu_rdm_integration/stages/collect_data/migrations/0003_auto_20250704_0810.py +35 -0
- edu_rdm_integration/stages/collect_data/migrations/0004_auto_20250704_0825.py +40 -0
- edu_rdm_integration/stages/collect_data/mixins.py +4 -4
- edu_rdm_integration/stages/collect_data/models.py +35 -64
- edu_rdm_integration/stages/collect_data/operations.py +7 -7
- edu_rdm_integration/stages/collect_data/registry/actions.py +10 -10
- edu_rdm_integration/stages/collect_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/collect_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/collect_data/tests.py +5 -5
- edu_rdm_integration/stages/export_data/function_templates/function_export_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/export_data/functions/base/functions.py +13 -13
- edu_rdm_integration/stages/export_data/functions/base/managers.py +10 -10
- edu_rdm_integration/stages/export_data/generators.py +3 -5
- edu_rdm_integration/stages/export_data/helpers.py +19 -19
- edu_rdm_integration/stages/export_data/management/base.py +2 -4
- edu_rdm_integration/stages/export_data/migrations/0001_initial.py +1 -0
- edu_rdm_integration/stages/export_data/migrations/0002_auto_20250704_0810.py +47 -0
- edu_rdm_integration/stages/export_data/mixins.py +5 -5
- edu_rdm_integration/stages/export_data/models.py +39 -52
- edu_rdm_integration/stages/export_data/operations.py +8 -10
- edu_rdm_integration/stages/export_data/registry/actions.py +18 -18
- edu_rdm_integration/stages/export_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/export_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/upload_data/consts.py +4 -4
- edu_rdm_integration/stages/upload_data/export_managers.py +23 -23
- edu_rdm_integration/stages/upload_data/helpers.py +10 -10
- edu_rdm_integration/stages/upload_data/management/commands/async_fix_attachment_size.py +2 -2
- edu_rdm_integration/stages/upload_data/management/commands/check_upload_status.py +2 -2
- edu_rdm_integration/stages/upload_data/migrations/0002_auto_20250704_0810.py +28 -0
- edu_rdm_integration/stages/upload_data/models.py +9 -9
- edu_rdm_integration/stages/upload_data/tasks.py +2 -2
- {edu_rdm_integration-3.10.3.dist-info → edu_rdm_integration-3.11.0.dist-info}/METADATA +6 -5
- {edu_rdm_integration-3.10.3.dist-info → edu_rdm_integration-3.11.0.dist-info}/RECORD +73 -65
- {edu_rdm_integration-3.10.3.dist-info → edu_rdm_integration-3.11.0.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.10.3.dist-info → edu_rdm_integration-3.11.0.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.10.3.dist-info → edu_rdm_integration-3.11.0.dist-info}/top_level.txt +0 -0
|
@@ -8,4 +8,4 @@ class RDMCollectDataRegistryAppConfig(AppConfig):
|
|
|
8
8
|
|
|
9
9
|
name = 'edu_rdm_integration.stages.collect_data.registry'
|
|
10
10
|
label = 'edu_rdm_integration_collect_data_registry'
|
|
11
|
-
verbose_name = 'Сбор данных моделей РВД из интерфейса'
|
|
11
|
+
verbose_name = 'Сбор данных моделей РВД из интерфейса'
|
|
@@ -29,7 +29,7 @@ from edu_rdm_integration.core.registry.ui import (
|
|
|
29
29
|
BaseCreateCommandWindow,
|
|
30
30
|
)
|
|
31
31
|
from edu_rdm_integration.rdm_models.models import (
|
|
32
|
-
|
|
32
|
+
RDMModelEnum,
|
|
33
33
|
)
|
|
34
34
|
from edu_rdm_integration.stages.collect_data.consts import (
|
|
35
35
|
ALL_UNITS_IN_COMMAND,
|
|
@@ -52,9 +52,7 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
|
|
|
52
52
|
trigger_action_all=True,
|
|
53
53
|
allow_blank=False,
|
|
54
54
|
)
|
|
55
|
-
model.set_store(
|
|
56
|
-
ExtDataStore((idx, key) for idx, key in enumerate(RegionalDataMartModelEnum.get_model_enum_keys()))
|
|
57
|
-
)
|
|
55
|
+
model.set_store(ExtDataStore((idx, key) for idx, key in enumerate(RDMModelEnum.get_model_enum_keys())))
|
|
58
56
|
logs_period_started_at = ExtDateTimeField(
|
|
59
57
|
name='logs_period_started_at',
|
|
60
58
|
label='Начало периода',
|
|
@@ -42,8 +42,8 @@ from edu_rdm_integration.core.consts import (
|
|
|
42
42
|
REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
|
|
43
43
|
)
|
|
44
44
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
RDMCollectingDataStage,
|
|
46
|
+
RDMCollectingDataSubStage,
|
|
47
47
|
)
|
|
48
48
|
|
|
49
49
|
|
|
@@ -127,7 +127,7 @@ class BaseCollectingFunctionTestCase(TestCase):
|
|
|
127
127
|
operation=operation,
|
|
128
128
|
)
|
|
129
129
|
|
|
130
|
-
def create_sub_stage(self, *class_names: str) ->
|
|
130
|
+
def create_sub_stage(self, *class_names: str) -> RDMCollectingDataSubStage:
|
|
131
131
|
"""Создает подэтап сбора данных."""
|
|
132
132
|
function_tools_entities = dict(
|
|
133
133
|
Entity.objects.filter(
|
|
@@ -136,11 +136,11 @@ class BaseCollectingFunctionTestCase(TestCase):
|
|
|
136
136
|
class_name__in=class_names,
|
|
137
137
|
).values_list('type', 'uuid'),
|
|
138
138
|
)
|
|
139
|
-
stage =
|
|
139
|
+
stage = RDMCollectingDataStage.objects.create(
|
|
140
140
|
manager_id=function_tools_entities[EntityType.MANAGER.key],
|
|
141
141
|
)
|
|
142
142
|
|
|
143
|
-
return
|
|
143
|
+
return RDMCollectingDataSubStage.objects.create(
|
|
144
144
|
function_id=function_tools_entities[EntityType.FUNCTION.key],
|
|
145
145
|
stage=stage,
|
|
146
146
|
)
|
|
@@ -7,7 +7,7 @@ from edu_rdm_integration.core.consts import (
|
|
|
7
7
|
REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
|
|
8
8
|
)
|
|
9
9
|
from edu_rdm_integration.rdm_entities.models import (
|
|
10
|
-
|
|
10
|
+
RDMEntityEnum,
|
|
11
11
|
)
|
|
12
12
|
|
|
13
13
|
from {{ strategy.function_class_module }} import (
|
|
@@ -51,7 +51,7 @@ class {{ camel_case_function_name }}Function({{ strategy.function_class_name }})
|
|
|
51
51
|
def _prepare_entities(cls) -> list['ModelEnumValue']:
|
|
52
52
|
"""Формирование списка ключей модели-перечисления сущностей."""
|
|
53
53
|
entities = [
|
|
54
|
-
|
|
54
|
+
RDMEntityEnum.,
|
|
55
55
|
]
|
|
56
56
|
|
|
57
57
|
return entities
|
|
@@ -47,11 +47,11 @@ from edu_rdm_integration.stages.export_data.helpers import (
|
|
|
47
47
|
get_exporting_data_stage_attachment_path,
|
|
48
48
|
)
|
|
49
49
|
from edu_rdm_integration.stages.export_data.models import (
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
RDMExportingDataStage,
|
|
51
|
+
RDMExportingDataSubStage,
|
|
52
|
+
RDMExportingDataSubStageAttachment,
|
|
53
|
+
RDMExportingDataSubStageEntity,
|
|
54
|
+
RDMExportingDataSubStageStatus,
|
|
55
55
|
)
|
|
56
56
|
|
|
57
57
|
|
|
@@ -62,14 +62,14 @@ class BaseExportDataFunction(
|
|
|
62
62
|
):
|
|
63
63
|
"""Базовый класс функций выгрузки данных для интеграции с "Региональная витрина данных"."""
|
|
64
64
|
|
|
65
|
-
def __init__(self, *args, stage:
|
|
65
|
+
def __init__(self, *args, stage: RDMExportingDataStage, model_ids: list[Union[int, str]], **kwargs):
|
|
66
66
|
super().__init__(*args, entities=self.entities, model_ids=model_ids, **kwargs)
|
|
67
67
|
|
|
68
|
-
self._sub_stage =
|
|
68
|
+
self._sub_stage = RDMExportingDataSubStage.objects.create(
|
|
69
69
|
stage=stage,
|
|
70
70
|
function_id=self.uuid,
|
|
71
71
|
)
|
|
72
|
-
|
|
72
|
+
RDMExportingDataSubStageEntity.objects.create(
|
|
73
73
|
exporting_data_sub_stage=self._sub_stage, entity_id=self.first_entity.key
|
|
74
74
|
)
|
|
75
75
|
# Проставление подэтапа выгрузки
|
|
@@ -105,7 +105,7 @@ class BaseExportDataFunction(
|
|
|
105
105
|
|
|
106
106
|
def _before_prepare(self, *args, **kwargs):
|
|
107
107
|
"""Выполнение действий функций системы."""
|
|
108
|
-
self._sub_stage.status_id =
|
|
108
|
+
self._sub_stage.status_id = RDMExportingDataSubStageStatus.IN_PROGRESS.key
|
|
109
109
|
self._sub_stage.save()
|
|
110
110
|
|
|
111
111
|
logger.info(f'{LOGS_DELIMITER * 3}change status {repr(self._sub_stage)}')
|
|
@@ -167,7 +167,7 @@ class BaseExportDataFunction(
|
|
|
167
167
|
|
|
168
168
|
joined_records = '\n'.join([title_record, *[f'{DELIMITER}'.join(record) for record in records]])
|
|
169
169
|
|
|
170
|
-
sub_stage_attachment =
|
|
170
|
+
sub_stage_attachment = RDMExportingDataSubStageAttachment(
|
|
171
171
|
exporting_data_sub_stage=self._sub_stage,
|
|
172
172
|
operation=operation,
|
|
173
173
|
)
|
|
@@ -213,12 +213,12 @@ class BaseExportDataFunction(
|
|
|
213
213
|
|
|
214
214
|
if self.result.has_not_errors:
|
|
215
215
|
self._sub_stage.status_id = (
|
|
216
|
-
|
|
216
|
+
RDMExportingDataSubStageStatus.FINISHED.key
|
|
217
217
|
if not self.has_data
|
|
218
|
-
else
|
|
218
|
+
else RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key
|
|
219
219
|
)
|
|
220
220
|
else:
|
|
221
|
-
self._sub_stage.status_id =
|
|
221
|
+
self._sub_stage.status_id = RDMExportingDataSubStageStatus.FAILED.key
|
|
222
222
|
|
|
223
223
|
self._sub_stage.save()
|
|
224
224
|
|
|
@@ -27,14 +27,14 @@ from edu_rdm_integration.core.storages import (
|
|
|
27
27
|
RegionalDataMartEntityStorage,
|
|
28
28
|
)
|
|
29
29
|
from edu_rdm_integration.stages.collect_data.models import (
|
|
30
|
-
|
|
30
|
+
RDMCollectingDataStageStatus,
|
|
31
31
|
)
|
|
32
32
|
from edu_rdm_integration.stages.export_data.functions.base.runners import (
|
|
33
33
|
BaseExportDataRunner,
|
|
34
34
|
)
|
|
35
35
|
from edu_rdm_integration.stages.export_data.models import (
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
RDMExportingDataStage,
|
|
37
|
+
RDMExportingDataStageStatus,
|
|
38
38
|
)
|
|
39
39
|
|
|
40
40
|
|
|
@@ -56,7 +56,7 @@ class BaseExportDataRunnerManager(WebEduRunnerManager, metaclass=ABCMeta):
|
|
|
56
56
|
|
|
57
57
|
self._period_started_at, self._period_ended_at = self._prepare_period(*args, **kwargs)
|
|
58
58
|
|
|
59
|
-
self._stage =
|
|
59
|
+
self._stage = RDMExportingDataStage.objects.create(
|
|
60
60
|
manager_id=self.uuid,
|
|
61
61
|
period_started_at=self._period_started_at,
|
|
62
62
|
period_ended_at=self._period_ended_at,
|
|
@@ -74,11 +74,11 @@ class BaseExportDataRunnerManager(WebEduRunnerManager, metaclass=ABCMeta):
|
|
|
74
74
|
entity_storage = RegionalDataMartEntityStorage()
|
|
75
75
|
entity_storage.prepare()
|
|
76
76
|
|
|
77
|
-
exporting_data_stage =
|
|
77
|
+
exporting_data_stage = RDMExportingDataStage.objects.filter(
|
|
78
78
|
manager_id=self.uuid,
|
|
79
79
|
status_id__in=(
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
RDMCollectingDataStageStatus.FAILED.key,
|
|
81
|
+
RDMCollectingDataStageStatus.FINISHED.key,
|
|
82
82
|
),
|
|
83
83
|
).latest('period_ended_at')
|
|
84
84
|
|
|
@@ -123,7 +123,7 @@ class BaseExportDataRunnerManager(WebEduRunnerManager, metaclass=ABCMeta):
|
|
|
123
123
|
"""
|
|
124
124
|
model_ids_map = self._prepare_model_ids_map()
|
|
125
125
|
|
|
126
|
-
self._stage.status_id =
|
|
126
|
+
self._stage.status_id = RDMExportingDataStageStatus.IN_PROGRESS.key
|
|
127
127
|
self._stage.save()
|
|
128
128
|
|
|
129
129
|
super()._create_runner(
|
|
@@ -146,9 +146,9 @@ class BaseExportDataRunnerManager(WebEduRunnerManager, metaclass=ABCMeta):
|
|
|
146
146
|
def _after_start_runner(self, *args, **kwargs):
|
|
147
147
|
"""Точка расширения поведения менеджера ранера после запуска ранера."""
|
|
148
148
|
if self._runner.result.has_not_errors:
|
|
149
|
-
self._stage.status_id =
|
|
149
|
+
self._stage.status_id = RDMExportingDataStageStatus.FINISHED.key
|
|
150
150
|
else:
|
|
151
|
-
self._stage.status_id =
|
|
151
|
+
self._stage.status_id = RDMExportingDataStageStatus.FAILED.key
|
|
152
152
|
|
|
153
153
|
self._stage.save()
|
|
154
154
|
|
|
@@ -21,7 +21,7 @@ from edu_rdm_integration.core.consts import (
|
|
|
21
21
|
DATE_FORMAT,
|
|
22
22
|
)
|
|
23
23
|
from edu_rdm_integration.rdm_entities.models import (
|
|
24
|
-
|
|
24
|
+
RDMEntityEnum,
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
|
|
@@ -38,10 +38,8 @@ class BaseFirstExportEntitiesDataCommandsGenerator:
|
|
|
38
38
|
):
|
|
39
39
|
"""Инициализация."""
|
|
40
40
|
# Если сущности не указаны, берется значение по умолчанию - все сущности:
|
|
41
|
-
entities = entities if entities else
|
|
42
|
-
self.entities: list[ModelEnumValue] = [
|
|
43
|
-
RegionalDataMartEntityEnum.get_model_enum_value(entity) for entity in entities
|
|
44
|
-
]
|
|
41
|
+
entities = entities if entities else RDMEntityEnum.get_enum_data().keys()
|
|
42
|
+
self.entities: list[ModelEnumValue] = [RDMEntityEnum.get_model_enum_value(entity) for entity in entities]
|
|
45
43
|
|
|
46
44
|
self.period_started_at = period_started_at
|
|
47
45
|
self.period_ended_at = period_ended_at
|
|
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
|
|
|
48
48
|
)
|
|
49
49
|
|
|
50
50
|
from edu_rdm_integration.rdm_models.models import (
|
|
51
|
-
|
|
51
|
+
BaseRDMModel,
|
|
52
52
|
)
|
|
53
53
|
from edu_rdm_integration.stages.export_data.functions.base.managers import (
|
|
54
54
|
BaseExportDataRunnerManager,
|
|
@@ -64,10 +64,10 @@ def set_failed_status_suspended_exporting_data_stages() -> dict[str, int]:
|
|
|
64
64
|
в настройках приложения.
|
|
65
65
|
"""
|
|
66
66
|
from edu_rdm_integration.stages.export_data.models import (
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
67
|
+
RDMExportingDataStage,
|
|
68
|
+
RDMExportingDataStageStatus,
|
|
69
|
+
RDMExportingDataSubStage,
|
|
70
|
+
RDMExportingDataSubStageStatus,
|
|
71
71
|
)
|
|
72
72
|
|
|
73
73
|
changed_status_result = {
|
|
@@ -79,10 +79,10 @@ def set_failed_status_suspended_exporting_data_stages() -> dict[str, int]:
|
|
|
79
79
|
suspended_time_at = current_datetime - timedelta(minutes=settings.RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT)
|
|
80
80
|
|
|
81
81
|
suspended_stage_ids = set(
|
|
82
|
-
|
|
82
|
+
RDMExportingDataStage.objects.annotate(
|
|
83
83
|
last_sub_stage_started_at=Coalesce(
|
|
84
84
|
Subquery(
|
|
85
|
-
|
|
85
|
+
RDMExportingDataSubStage.objects.filter(stage_id=OuterRef('pk'))
|
|
86
86
|
.values('started_at')
|
|
87
87
|
.order_by('-started_at')[:1]
|
|
88
88
|
),
|
|
@@ -92,8 +92,8 @@ def set_failed_status_suspended_exporting_data_stages() -> dict[str, int]:
|
|
|
92
92
|
.filter(
|
|
93
93
|
last_sub_stage_started_at__lt=suspended_time_at,
|
|
94
94
|
status__in=(
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
RDMExportingDataStageStatus.CREATED.key,
|
|
96
|
+
RDMExportingDataStageStatus.IN_PROGRESS.key,
|
|
97
97
|
),
|
|
98
98
|
)
|
|
99
99
|
.values_list('pk', flat=True)
|
|
@@ -102,17 +102,17 @@ def set_failed_status_suspended_exporting_data_stages() -> dict[str, int]:
|
|
|
102
102
|
if suspended_stage_ids:
|
|
103
103
|
logger.info(f'find suspended ExportingDataStage: {", ".join(map(str, suspended_stage_ids))}..')
|
|
104
104
|
|
|
105
|
-
change_stage_count =
|
|
105
|
+
change_stage_count = RDMExportingDataStage.objects.filter(
|
|
106
106
|
pk__in=suspended_stage_ids,
|
|
107
107
|
).update(
|
|
108
|
-
status=
|
|
108
|
+
status=RDMExportingDataStageStatus.FAILED.key,
|
|
109
109
|
ended_at=current_datetime,
|
|
110
110
|
)
|
|
111
111
|
|
|
112
|
-
change_sub_stage_count =
|
|
112
|
+
change_sub_stage_count = RDMExportingDataSubStage.objects.filter(
|
|
113
113
|
stage_id__in=suspended_stage_ids,
|
|
114
114
|
).update(
|
|
115
|
-
status=
|
|
115
|
+
status=RDMExportingDataSubStageStatus.FAILED.key,
|
|
116
116
|
ended_at=current_datetime,
|
|
117
117
|
)
|
|
118
118
|
|
|
@@ -131,17 +131,17 @@ def get_exporting_managers_max_period_ended_dates(
|
|
|
131
131
|
) -> dict[str, 'datetime']:
|
|
132
132
|
"""Возвращает дату и время последнего успешного этапа экспорта для менеджеров Функций экспорта."""
|
|
133
133
|
from edu_rdm_integration.stages.export_data.models import (
|
|
134
|
-
|
|
135
|
-
|
|
134
|
+
RDMExportingDataStage,
|
|
135
|
+
RDMExportingDataStageStatus,
|
|
136
136
|
)
|
|
137
137
|
|
|
138
138
|
managers_last_period_ended = (
|
|
139
|
-
|
|
139
|
+
RDMExportingDataStage.objects.filter(
|
|
140
140
|
manager_id__in=[manager.uuid for manager in exporting_managers],
|
|
141
141
|
id=Subquery(
|
|
142
|
-
|
|
142
|
+
RDMExportingDataStage.objects.filter(
|
|
143
143
|
manager_id=OuterRef('manager_id'),
|
|
144
|
-
status_id=
|
|
144
|
+
status_id=RDMExportingDataStageStatus.FINISHED.key,
|
|
145
145
|
)
|
|
146
146
|
.order_by('-id')
|
|
147
147
|
.values('id')[:1]
|
|
@@ -184,7 +184,7 @@ def get_exporting_data_stage_attachment_path(instance, filename):
|
|
|
184
184
|
|
|
185
185
|
|
|
186
186
|
def set_entity_field_by_model_object(
|
|
187
|
-
entity: '
|
|
187
|
+
entity: 'BaseRDMModel', model_object: Union['Model', 'NamedTuple'], mapping: dict[str, str]
|
|
188
188
|
) -> None:
|
|
189
189
|
"""Обновление значений полей сущности по измененным полям модели.
|
|
190
190
|
|
|
@@ -16,7 +16,7 @@ from edu_rdm_integration.core.consts import (
|
|
|
16
16
|
DATETIME_FORMAT,
|
|
17
17
|
)
|
|
18
18
|
from edu_rdm_integration.rdm_entities.models import (
|
|
19
|
-
|
|
19
|
+
RDMEntityEnum,
|
|
20
20
|
)
|
|
21
21
|
from edu_rdm_integration.stages.export_data.operations import (
|
|
22
22
|
BaseExportEntitiesData,
|
|
@@ -34,9 +34,7 @@ class BaseExportEntityDataCommand(BaseCommand):
|
|
|
34
34
|
|
|
35
35
|
def add_arguments(self, parser: 'CommandParser'):
|
|
36
36
|
"""Добавление параметров."""
|
|
37
|
-
entities = ', '.join(
|
|
38
|
-
[f'{key} - {value.title}' for key, value in RegionalDataMartEntityEnum.get_enum_data().items()]
|
|
39
|
-
)
|
|
37
|
+
entities = ', '.join([f'{key} - {value.title}' for key, value in RDMEntityEnum.get_enum_data().items()])
|
|
40
38
|
entities_help_text = (
|
|
41
39
|
f'Значением параметра является перечисление сущностей РВД, для которых должена быть произведена выгрузка '
|
|
42
40
|
f'данных. Перечисление сущностей:\n{entities}. Если сущности не указываются, то выгрузка данных '
|
|
@@ -22,6 +22,7 @@ class Migration(migrations.Migration):
|
|
|
22
22
|
('edu_rdm_integration_entities', '0001_initial'),
|
|
23
23
|
('function_tools', '0005_auto_20220724_0050'),
|
|
24
24
|
('async_task', '0003_alter_runningtask_options'),
|
|
25
|
+
('rdm_collect_and_export_data', '0003_auto_20250704_0725'),
|
|
25
26
|
]
|
|
26
27
|
|
|
27
28
|
operations = [
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-07-04 08:10
|
|
2
|
+
|
|
3
|
+
from django.db import (
|
|
4
|
+
migrations,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Migration(migrations.Migration):
|
|
9
|
+
dependencies = [
|
|
10
|
+
('function_tools', '0005_auto_20220724_0050'),
|
|
11
|
+
('edu_rdm_integration_upload_data_stage', '0001_initial'),
|
|
12
|
+
('async_task', '0003_alter_runningtask_options'),
|
|
13
|
+
('regional_data_mart_integration', '0163_auto_20250704_0725'),
|
|
14
|
+
('edu_rdm_integration_entities', '0002_rename_regionaldatamartentityenum_rdmentityenum'),
|
|
15
|
+
('edu_rdm_integration_export_data_stage', '0001_initial'),
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
operations = [
|
|
19
|
+
migrations.RenameModel(
|
|
20
|
+
old_name='EduRdmExportDataCommandProgress',
|
|
21
|
+
new_name='RDMExportingDataCommandProgress',
|
|
22
|
+
),
|
|
23
|
+
migrations.RenameModel(
|
|
24
|
+
old_name='ExportingDataStage',
|
|
25
|
+
new_name='RDMExportingDataStage',
|
|
26
|
+
),
|
|
27
|
+
migrations.RenameModel(
|
|
28
|
+
old_name='ExportingDataStageStatus',
|
|
29
|
+
new_name='RDMExportingDataStageStatus',
|
|
30
|
+
),
|
|
31
|
+
migrations.RenameModel(
|
|
32
|
+
old_name='ExportingDataSubStage',
|
|
33
|
+
new_name='RDMExportingDataSubStage',
|
|
34
|
+
),
|
|
35
|
+
migrations.RenameModel(
|
|
36
|
+
old_name='ExportingDataSubStageAttachment',
|
|
37
|
+
new_name='RDMExportingDataSubStageAttachment',
|
|
38
|
+
),
|
|
39
|
+
migrations.RenameModel(
|
|
40
|
+
old_name='ExportingDataSubStageEntity',
|
|
41
|
+
new_name='RDMExportingDataSubStageEntity',
|
|
42
|
+
),
|
|
43
|
+
migrations.RenameModel(
|
|
44
|
+
old_name='ExportingDataSubStageStatus',
|
|
45
|
+
new_name='RDMExportingDataSubStageStatus',
|
|
46
|
+
),
|
|
47
|
+
]
|
|
@@ -9,7 +9,7 @@ from edu_rdm_integration.core.helpers import (
|
|
|
9
9
|
save_command_log_link,
|
|
10
10
|
)
|
|
11
11
|
from edu_rdm_integration.stages.export_data.models import (
|
|
12
|
-
|
|
12
|
+
RDMExportingDataCommandProgress,
|
|
13
13
|
)
|
|
14
14
|
from edu_rdm_integration.stages.export_data.operations import (
|
|
15
15
|
ExportEntitiesData,
|
|
@@ -24,13 +24,13 @@ class ExportCommandMixin:
|
|
|
24
24
|
description = 'Экспорт данных сущностей РВД'
|
|
25
25
|
task_type = AsyncTaskType.SYSTEM
|
|
26
26
|
|
|
27
|
-
def get_export_command(self, command_id: int) ->
|
|
27
|
+
def get_export_command(self, command_id: int) -> RDMExportingDataCommandProgress:
|
|
28
28
|
"""Возвращает экземпляр модели команды запуска."""
|
|
29
|
-
command =
|
|
29
|
+
command = RDMExportingDataCommandProgress.objects.get(id=command_id)
|
|
30
30
|
|
|
31
31
|
return command
|
|
32
32
|
|
|
33
|
-
def run_export_command(self, command:
|
|
33
|
+
def run_export_command(self, command: RDMExportingDataCommandProgress) -> None:
|
|
34
34
|
"""Запуск команды выгрузки."""
|
|
35
35
|
ExportEntitiesData(
|
|
36
36
|
entities=(command.entity_id,),
|
|
@@ -43,7 +43,7 @@ class ExportCommandMixin:
|
|
|
43
43
|
"""Сохранение ссылки на файл логов в команде."""
|
|
44
44
|
try:
|
|
45
45
|
command = self.get_export_command(command_id)
|
|
46
|
-
except
|
|
46
|
+
except RDMExportingDataCommandProgress.DoesNotExist:
|
|
47
47
|
command = None
|
|
48
48
|
|
|
49
49
|
if command:
|
|
@@ -43,14 +43,14 @@ from edu_rdm_integration.core.utils import (
|
|
|
43
43
|
get_data_command_progress_attachment_path,
|
|
44
44
|
)
|
|
45
45
|
from edu_rdm_integration.rdm_entities.models import (
|
|
46
|
-
|
|
46
|
+
RDMEntityEnum,
|
|
47
47
|
)
|
|
48
48
|
from edu_rdm_integration.stages.export_data.helpers import (
|
|
49
49
|
get_exporting_data_stage_attachment_path,
|
|
50
50
|
)
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
class
|
|
53
|
+
class RDMExportingDataStageStatus(TitledModelEnum):
|
|
54
54
|
"""Статус этапа выгрузки данных."""
|
|
55
55
|
|
|
56
56
|
CREATED = ModelEnumValue(
|
|
@@ -75,7 +75,7 @@ class ExportingDataStageStatus(TitledModelEnum):
|
|
|
75
75
|
verbose_name_plural = 'Модели-перечисления статусов этапов выгрузки данных'
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
class
|
|
78
|
+
class RDMExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
79
79
|
"""Этап выгрузки данных."""
|
|
80
80
|
|
|
81
81
|
manager = ForeignKey(
|
|
@@ -108,10 +108,10 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
108
108
|
)
|
|
109
109
|
|
|
110
110
|
status = ForeignKey(
|
|
111
|
-
to=
|
|
111
|
+
to=RDMExportingDataStageStatus,
|
|
112
112
|
verbose_name='Статус',
|
|
113
113
|
on_delete=PROTECT,
|
|
114
|
-
default=
|
|
114
|
+
default=RDMExportingDataStageStatus.CREATED.key,
|
|
115
115
|
)
|
|
116
116
|
|
|
117
117
|
class Meta:
|
|
@@ -127,7 +127,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
127
127
|
def save(self, *args, **kwargs):
|
|
128
128
|
"""Сохранение этапа экспорта данных."""
|
|
129
129
|
if (
|
|
130
|
-
self.status_id in (
|
|
130
|
+
self.status_id in (RDMExportingDataStageStatus.FAILED.key, RDMExportingDataStageStatus.FINISHED.key)
|
|
131
131
|
and not self.ended_at
|
|
132
132
|
):
|
|
133
133
|
self.ended_at = datetime.now()
|
|
@@ -135,7 +135,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
135
135
|
super().save(*args, **kwargs)
|
|
136
136
|
|
|
137
137
|
|
|
138
|
-
class
|
|
138
|
+
class RDMExportingDataSubStageStatus(TitledModelEnum):
|
|
139
139
|
"""Модель-перечисление статусов этапа выгрузки данных."""
|
|
140
140
|
|
|
141
141
|
CREATED = ModelEnumValue(
|
|
@@ -164,7 +164,7 @@ class ExportingDataSubStageStatus(TitledModelEnum):
|
|
|
164
164
|
verbose_name_plural = 'Модели-перечисления статусов подэтапов выгрузки данных'
|
|
165
165
|
|
|
166
166
|
|
|
167
|
-
class
|
|
167
|
+
class RDMExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
168
168
|
"""Подэтап выгрузки данных."""
|
|
169
169
|
|
|
170
170
|
function = ForeignKey(
|
|
@@ -176,7 +176,7 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
176
176
|
)
|
|
177
177
|
|
|
178
178
|
stage = ForeignKey(
|
|
179
|
-
to=
|
|
179
|
+
to=RDMExportingDataStage,
|
|
180
180
|
verbose_name='Этап выгрузки данных',
|
|
181
181
|
on_delete=CASCADE,
|
|
182
182
|
)
|
|
@@ -195,10 +195,10 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
195
195
|
)
|
|
196
196
|
|
|
197
197
|
status = ForeignKey(
|
|
198
|
-
to=
|
|
198
|
+
to=RDMExportingDataSubStageStatus,
|
|
199
199
|
verbose_name='Статус',
|
|
200
200
|
on_delete=PROTECT,
|
|
201
|
-
default=
|
|
201
|
+
default=RDMExportingDataSubStageStatus.CREATED.key,
|
|
202
202
|
)
|
|
203
203
|
|
|
204
204
|
class Meta:
|
|
@@ -216,9 +216,9 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
216
216
|
if (
|
|
217
217
|
self.status_id
|
|
218
218
|
in {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
219
|
+
RDMExportingDataSubStageStatus.FAILED.key,
|
|
220
|
+
RDMExportingDataSubStageStatus.FINISHED.key,
|
|
221
|
+
RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key,
|
|
222
222
|
}
|
|
223
223
|
and not self.ended_at
|
|
224
224
|
):
|
|
@@ -227,11 +227,11 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
227
227
|
super().save(*args, **kwargs)
|
|
228
228
|
|
|
229
229
|
|
|
230
|
-
class
|
|
230
|
+
class RDMExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
|
|
231
231
|
"""Сгенерированный файл для дальнейшей выгрузки в "Региональная витрина данных"."""
|
|
232
232
|
|
|
233
233
|
exporting_data_sub_stage = ForeignKey(
|
|
234
|
-
to=
|
|
234
|
+
to=RDMExportingDataSubStage,
|
|
235
235
|
verbose_name='Подэтап выгрузки данных',
|
|
236
236
|
on_delete=CASCADE,
|
|
237
237
|
)
|
|
@@ -276,17 +276,17 @@ class ExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
276
276
|
return ['exporting_data_sub_stage_id', 'attachment', 'operation', 'created', 'modified']
|
|
277
277
|
|
|
278
278
|
|
|
279
|
-
class
|
|
279
|
+
class RDMExportingDataSubStageEntity(BaseObjectModel):
|
|
280
280
|
"""Модель связи сущности и подэтапа выгрузки."""
|
|
281
281
|
|
|
282
282
|
entity = ForeignKey(
|
|
283
|
-
to=
|
|
283
|
+
to=RDMEntityEnum,
|
|
284
284
|
verbose_name='Сущность РВД',
|
|
285
285
|
on_delete=PROTECT,
|
|
286
286
|
)
|
|
287
287
|
|
|
288
288
|
exporting_data_sub_stage = ForeignKey(
|
|
289
|
-
to=
|
|
289
|
+
to=RDMExportingDataSubStage,
|
|
290
290
|
verbose_name='Подэтап выгрузки данных',
|
|
291
291
|
on_delete=CASCADE,
|
|
292
292
|
)
|
|
@@ -297,22 +297,34 @@ class ExportingDataSubStageEntity(BaseObjectModel):
|
|
|
297
297
|
verbose_name_plural = 'Связи сущности и подэтапа выгрузки'
|
|
298
298
|
|
|
299
299
|
|
|
300
|
-
class
|
|
300
|
+
class RDMExportingDataCommandProgress(ReprStrPreModelMixin, BaseObjectModel):
|
|
301
301
|
"""Команда экспорта данных."""
|
|
302
302
|
|
|
303
|
-
task =
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
303
|
+
task = ForeignKey(
|
|
304
|
+
to='async_task.RunningTask',
|
|
305
|
+
verbose_name='Асинхронная задача',
|
|
306
|
+
null=True,
|
|
307
|
+
blank=True,
|
|
308
|
+
on_delete=SET_NULL,
|
|
309
|
+
)
|
|
310
|
+
logs_link = FileField(
|
|
311
|
+
upload_to=get_data_command_progress_attachment_path,
|
|
312
|
+
max_length=255,
|
|
313
|
+
verbose_name='Файл лога',
|
|
314
|
+
)
|
|
315
|
+
type = PositiveSmallIntegerField( # noqa: A003
|
|
316
|
+
verbose_name='Тип команды',
|
|
317
|
+
choices=CommandType.get_choices(),
|
|
318
|
+
)
|
|
307
319
|
stage = ForeignKey(
|
|
308
|
-
to=
|
|
320
|
+
to=RDMExportingDataStage,
|
|
309
321
|
verbose_name='Этап выгрузки данных',
|
|
310
322
|
null=True,
|
|
311
323
|
blank=True,
|
|
312
324
|
on_delete=SET_NULL,
|
|
313
325
|
)
|
|
314
326
|
entity = ForeignKey(
|
|
315
|
-
to=
|
|
327
|
+
to=RDMEntityEnum,
|
|
316
328
|
verbose_name='Сущность РВД',
|
|
317
329
|
on_delete=PROTECT,
|
|
318
330
|
)
|
|
@@ -332,31 +344,6 @@ class AbstractExportDataCommandProgress(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
332
344
|
)
|
|
333
345
|
|
|
334
346
|
class Meta:
|
|
335
|
-
|
|
336
|
-
db_table = 'rdm_exporting_data_command_progress'
|
|
347
|
+
db_table = 'edu_rdm_exporting_data_command_progress'
|
|
337
348
|
verbose_name = 'Команда экспорта данных'
|
|
338
349
|
verbose_name_plural = 'Команды экспорта данных'
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
class EduRdmExportDataCommandProgress(AbstractExportDataCommandProgress):
|
|
342
|
-
"""Команда экспорта данных."""
|
|
343
|
-
|
|
344
|
-
task = ForeignKey(
|
|
345
|
-
to='async_task.RunningTask',
|
|
346
|
-
verbose_name='Асинхронная задача',
|
|
347
|
-
null=True,
|
|
348
|
-
blank=True,
|
|
349
|
-
on_delete=SET_NULL,
|
|
350
|
-
)
|
|
351
|
-
logs_link = FileField(
|
|
352
|
-
upload_to=get_data_command_progress_attachment_path,
|
|
353
|
-
max_length=255,
|
|
354
|
-
verbose_name='Файл лога',
|
|
355
|
-
)
|
|
356
|
-
type = PositiveSmallIntegerField( # noqa: A003
|
|
357
|
-
verbose_name='Тип команды',
|
|
358
|
-
choices=CommandType.get_choices(),
|
|
359
|
-
)
|
|
360
|
-
|
|
361
|
-
class Meta(AbstractExportDataCommandProgress.Meta):
|
|
362
|
-
db_table = 'edu_rdm_exporting_data_command_progress'
|