edu-rdm-integration 3.10.4__py3-none-any.whl → 3.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of edu-rdm-integration might be problematic. Click here for more details.
- edu_rdm_integration/adapters/apps.py +3 -3
- edu_rdm_integration/adapters/caches.py +4 -4
- edu_rdm_integration/adapters/errors.py +2 -2
- edu_rdm_integration/adapters/functions.py +9 -9
- edu_rdm_integration/adapters/helpers.py +7 -7
- edu_rdm_integration/adapters/receivers.py +3 -3
- edu_rdm_integration/adapters/runners.py +13 -13
- edu_rdm_integration/adapters/strategies.py +7 -7
- edu_rdm_integration/adapters/validators.py +6 -6
- edu_rdm_integration/collect_and_export_data/migrations/0003_auto_20250704_0725.py +37 -0
- edu_rdm_integration/collect_and_export_data/models.py +1 -1
- edu_rdm_integration/core/helpers.py +6 -4
- edu_rdm_integration/core/registry/apps.py +1 -1
- edu_rdm_integration/core/utils.py +3 -3
- edu_rdm_integration/migrations/0018_auto_20250704_0725.py +154 -0
- edu_rdm_integration/pipelines/transfer/actions.py +2 -2
- edu_rdm_integration/pipelines/transfer/mixins.py +2 -2
- edu_rdm_integration/pipelines/transfer/models.py +2 -2
- edu_rdm_integration/pipelines/transfer/tasks.py +8 -8
- edu_rdm_integration/rdm_entities/entities.py +6 -6
- edu_rdm_integration/rdm_entities/migrations/0001_initial.py +1 -1
- edu_rdm_integration/rdm_entities/migrations/0002_rename_regionaldatamartentityenum_rdmentityenum.py +20 -0
- edu_rdm_integration/rdm_entities/mixins.py +2 -2
- edu_rdm_integration/rdm_entities/models.py +1 -1
- edu_rdm_integration/rdm_models/migrations/0001_initial.py +1 -1
- edu_rdm_integration/rdm_models/migrations/0002_rename_regionaldatamartmodelenum_rdmmodelenum.py +19 -0
- edu_rdm_integration/rdm_models/mixins.py +2 -2
- edu_rdm_integration/rdm_models/models.py +7 -7
- edu_rdm_integration/stages/collect_data/function_templates/function_collect_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/collect_data/functions/base/functions.py +9 -9
- edu_rdm_integration/stages/collect_data/functions/base/managers.py +6 -6
- edu_rdm_integration/stages/collect_data/functions/base/mixins.py +14 -16
- edu_rdm_integration/stages/collect_data/generators.py +3 -3
- edu_rdm_integration/stages/collect_data/helpers.py +18 -18
- edu_rdm_integration/stages/collect_data/management/base.py +2 -4
- edu_rdm_integration/stages/collect_data/migrations/0001_initial.py +1 -1
- edu_rdm_integration/stages/collect_data/migrations/0002_edurdmcollectdatacommandprogress.py +1 -0
- edu_rdm_integration/stages/collect_data/migrations/0003_auto_20250704_0810.py +35 -0
- edu_rdm_integration/stages/collect_data/migrations/0004_auto_20250704_0825.py +40 -0
- edu_rdm_integration/stages/collect_data/mixins.py +4 -4
- edu_rdm_integration/stages/collect_data/models.py +35 -64
- edu_rdm_integration/stages/collect_data/operations.py +7 -7
- edu_rdm_integration/stages/collect_data/registry/actions.py +10 -10
- edu_rdm_integration/stages/collect_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/collect_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/collect_data/tests.py +5 -5
- edu_rdm_integration/stages/export_data/function_templates/function_export_data_template/functions.py-tpl +2 -2
- edu_rdm_integration/stages/export_data/functions/base/functions.py +13 -13
- edu_rdm_integration/stages/export_data/functions/base/managers.py +10 -10
- edu_rdm_integration/stages/export_data/generators.py +3 -5
- edu_rdm_integration/stages/export_data/helpers.py +19 -19
- edu_rdm_integration/stages/export_data/management/base.py +2 -4
- edu_rdm_integration/stages/export_data/migrations/0001_initial.py +1 -0
- edu_rdm_integration/stages/export_data/migrations/0002_auto_20250704_0810.py +47 -0
- edu_rdm_integration/stages/export_data/mixins.py +5 -5
- edu_rdm_integration/stages/export_data/models.py +39 -52
- edu_rdm_integration/stages/export_data/operations.py +8 -10
- edu_rdm_integration/stages/export_data/registry/actions.py +18 -18
- edu_rdm_integration/stages/export_data/registry/apps.py +1 -1
- edu_rdm_integration/stages/export_data/registry/ui.py +2 -4
- edu_rdm_integration/stages/upload_data/consts.py +4 -4
- edu_rdm_integration/stages/upload_data/export_managers.py +23 -23
- edu_rdm_integration/stages/upload_data/helpers.py +10 -10
- edu_rdm_integration/stages/upload_data/management/commands/async_fix_attachment_size.py +2 -2
- edu_rdm_integration/stages/upload_data/management/commands/check_upload_status.py +2 -2
- edu_rdm_integration/stages/upload_data/migrations/0002_auto_20250704_0810.py +28 -0
- edu_rdm_integration/stages/upload_data/models.py +9 -9
- edu_rdm_integration/stages/upload_data/tasks.py +2 -2
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.0.dist-info}/METADATA +3 -2
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.0.dist-info}/RECORD +73 -65
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.0.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.0.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.10.4.dist-info → edu_rdm_integration-3.11.0.dist-info}/top_level.txt +0 -0
|
@@ -70,14 +70,14 @@ from edu_rdm_integration.core.storages import (
|
|
|
70
70
|
RegionalDataMartEntityStorage,
|
|
71
71
|
)
|
|
72
72
|
from edu_rdm_integration.rdm_entities.models import (
|
|
73
|
-
|
|
73
|
+
RDMEntityEnum,
|
|
74
74
|
)
|
|
75
75
|
from edu_rdm_integration.stages.export_data.helpers import (
|
|
76
76
|
get_exporting_managers_max_period_ended_dates,
|
|
77
77
|
)
|
|
78
78
|
from edu_rdm_integration.stages.export_data.models import (
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
RDMExportingDataStage,
|
|
80
|
+
RDMExportingDataSubStageStatus,
|
|
81
81
|
)
|
|
82
82
|
|
|
83
83
|
|
|
@@ -100,10 +100,8 @@ class BaseExportEntitiesData(BaseOperationData):
|
|
|
100
100
|
super().__init__(**kwargs)
|
|
101
101
|
|
|
102
102
|
# Если сущности не указаны, берется значение по умолчанию - все сущности:
|
|
103
|
-
entities = entities if entities else
|
|
104
|
-
self.entities: list[ModelEnumValue] = [
|
|
105
|
-
RegionalDataMartEntityEnum.get_model_enum_value(entity) for entity in entities
|
|
106
|
-
]
|
|
103
|
+
entities = entities if entities else RDMEntityEnum.get_enum_data().keys()
|
|
104
|
+
self.entities: list[ModelEnumValue] = [RDMEntityEnum.get_model_enum_value(entity) for entity in entities]
|
|
107
105
|
|
|
108
106
|
self.period_started_at = period_started_at
|
|
109
107
|
self.period_ended_at = period_ended_at
|
|
@@ -124,9 +122,9 @@ class BaseExportEntitiesData(BaseOperationData):
|
|
|
124
122
|
|
|
125
123
|
def _has_stage_created_or_in_progress(self, manager_id: str, entity: str) -> bool:
|
|
126
124
|
"""Проверяет есть ли готовый к работе stage или в работе для данной сущности."""
|
|
127
|
-
stage_created_or_in_progress =
|
|
125
|
+
stage_created_or_in_progress = RDMExportingDataStage.objects.filter(
|
|
128
126
|
manager_id=manager_id,
|
|
129
|
-
status_id__in=(
|
|
127
|
+
status_id__in=(RDMExportingDataSubStageStatus.CREATED.key, RDMExportingDataSubStageStatus.IN_PROGRESS.key),
|
|
130
128
|
).exists()
|
|
131
129
|
|
|
132
130
|
if stage_created_or_in_progress:
|
|
@@ -265,7 +263,7 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
|
265
263
|
# и modified записи модели < левой границы периода команды latest (даты последней удачной выгрузки):
|
|
266
264
|
model.objects.filter(
|
|
267
265
|
exporting_sub_stage__ended_at__gt=now - timedelta(days=365),
|
|
268
|
-
exporting_sub_stage__status_id=
|
|
266
|
+
exporting_sub_stage__status_id=RDMExportingDataSubStageStatus.FAILED.key,
|
|
269
267
|
modified__gt=F('exporting_sub_stage__ended_at'),
|
|
270
268
|
modified__lt=last_finished_export_data,
|
|
271
269
|
),
|
|
@@ -42,16 +42,16 @@ from edu_rdm_integration.core.registry.actions import (
|
|
|
42
42
|
BaseStartTaskAction,
|
|
43
43
|
)
|
|
44
44
|
from edu_rdm_integration.rdm_entities.models import (
|
|
45
|
-
|
|
45
|
+
RDMEntityEnum,
|
|
46
46
|
)
|
|
47
47
|
from edu_rdm_integration.stages.export_data.generators import (
|
|
48
48
|
BaseFirstExportEntitiesDataCommandsGenerator,
|
|
49
49
|
)
|
|
50
50
|
from edu_rdm_integration.stages.export_data.models import (
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
51
|
+
RDMExportingDataCommandProgress,
|
|
52
|
+
RDMExportingDataStageStatus,
|
|
53
|
+
RDMExportingDataSubStage,
|
|
54
|
+
RDMExportingDataSubStageStatus,
|
|
55
55
|
)
|
|
56
56
|
from edu_rdm_integration.stages.export_data.registry.ui import (
|
|
57
57
|
CreateExportCommandWindow,
|
|
@@ -63,7 +63,7 @@ from edu_rdm_integration.stages.export_data.registry.ui import (
|
|
|
63
63
|
class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
64
64
|
"""Базовый пак команд экспорта данных сущностей РВД."""
|
|
65
65
|
|
|
66
|
-
model =
|
|
66
|
+
model = RDMExportingDataCommandProgress
|
|
67
67
|
title = 'Экспорт данных сущностей РВД'
|
|
68
68
|
|
|
69
69
|
add_window = CreateExportCommandWindow
|
|
@@ -83,7 +83,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
83
83
|
'header': 'Сущность',
|
|
84
84
|
'sortable': True,
|
|
85
85
|
'filter': ChoicesFilter(
|
|
86
|
-
choices=[(key, key) for key in
|
|
86
|
+
choices=[(key, key) for key in RDMEntityEnum.get_model_enum_keys()],
|
|
87
87
|
parser=str,
|
|
88
88
|
lookup=lambda key: Q(entity=key) if key else Q(),
|
|
89
89
|
),
|
|
@@ -128,7 +128,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
128
128
|
'header': 'Статус экспорта',
|
|
129
129
|
'sortable': True,
|
|
130
130
|
'filter': ChoicesFilter(
|
|
131
|
-
choices=[(key, key) for key in
|
|
131
|
+
choices=[(key, key) for key in RDMExportingDataStageStatus.get_model_enum_keys()],
|
|
132
132
|
parser=str,
|
|
133
133
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
134
134
|
),
|
|
@@ -213,9 +213,9 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
213
213
|
|
|
214
214
|
return query.annotate(
|
|
215
215
|
finished_sub_stages=Subquery(
|
|
216
|
-
|
|
216
|
+
RDMExportingDataSubStage.objects.filter(
|
|
217
217
|
stage_id=OuterRef('stage_id'),
|
|
218
|
-
status=
|
|
218
|
+
status=RDMExportingDataSubStageStatus.FINISHED.key,
|
|
219
219
|
)
|
|
220
220
|
.annotate(
|
|
221
221
|
finished_count=Func(F('id'), function='Count'),
|
|
@@ -223,9 +223,9 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
223
223
|
.values('finished_count'),
|
|
224
224
|
),
|
|
225
225
|
ready_sub_stages=Subquery(
|
|
226
|
-
|
|
226
|
+
RDMExportingDataSubStage.objects.filter(
|
|
227
227
|
stage_id=OuterRef('stage_id'),
|
|
228
|
-
status=
|
|
228
|
+
status=RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key,
|
|
229
229
|
)
|
|
230
230
|
.annotate(
|
|
231
231
|
ready_count=Func(F('id'), function='Count', output_field=IntegerField()),
|
|
@@ -233,9 +233,9 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
233
233
|
.values('ready_count'),
|
|
234
234
|
),
|
|
235
235
|
process_errors_sub_stages=Subquery(
|
|
236
|
-
|
|
236
|
+
RDMExportingDataSubStage.objects.filter(
|
|
237
237
|
stage_id=OuterRef('stage_id'),
|
|
238
|
-
status=
|
|
238
|
+
status=RDMExportingDataSubStageStatus.PROCESS_ERROR.key,
|
|
239
239
|
)
|
|
240
240
|
.annotate(process_errors_count=Func(F('id'), function='Count', output_field=IntegerField()))
|
|
241
241
|
.values('process_errors_count'),
|
|
@@ -275,14 +275,14 @@ class PrepareSubStageForExportAction(BaseAction):
|
|
|
275
275
|
def run(self, request, context):
|
|
276
276
|
"""Обновление статусов подэтапов не принятых витриной."""
|
|
277
277
|
command_ids = context.commands
|
|
278
|
-
stage_ids =
|
|
278
|
+
stage_ids = RDMExportingDataCommandProgress.objects.filter(id__in=command_ids).values_list(
|
|
279
279
|
'stage_id', flat=True
|
|
280
280
|
)
|
|
281
281
|
|
|
282
|
-
updated_count =
|
|
282
|
+
updated_count = RDMExportingDataSubStage.objects.filter(
|
|
283
283
|
stage_id__in=stage_ids,
|
|
284
|
-
status=
|
|
285
|
-
).update(status=
|
|
284
|
+
status=RDMExportingDataSubStageStatus.PROCESS_ERROR.key,
|
|
285
|
+
).update(status=RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key)
|
|
286
286
|
if updated_count:
|
|
287
287
|
message = f'Будет переотправлено {updated_count} подэтапов.'
|
|
288
288
|
else:
|
|
@@ -8,4 +8,4 @@ class RDMExportDataRegistryAppConfig(AppConfig):
|
|
|
8
8
|
|
|
9
9
|
name = 'edu_rdm_integration.stages.export_data.registry'
|
|
10
10
|
label = 'edu_rdm_integration_export_data_registry'
|
|
11
|
-
verbose_name = 'Экспорт данных моделей РВД из интерфейса'
|
|
11
|
+
verbose_name = 'Экспорт данных моделей РВД из интерфейса'
|
|
@@ -37,7 +37,7 @@ from edu_rdm_integration.core.registry.ui import (
|
|
|
37
37
|
CommandProgressListWindow,
|
|
38
38
|
)
|
|
39
39
|
from edu_rdm_integration.rdm_entities.models import (
|
|
40
|
-
|
|
40
|
+
RDMEntityEnum,
|
|
41
41
|
)
|
|
42
42
|
|
|
43
43
|
|
|
@@ -57,9 +57,7 @@ class CreateExportCommandWindow(BaseCreateCommandWindow):
|
|
|
57
57
|
trigger_action_all=True,
|
|
58
58
|
allow_blank=False,
|
|
59
59
|
)
|
|
60
|
-
entity.set_store(
|
|
61
|
-
ExtDataStore((idx, key) for idx, key in enumerate(RegionalDataMartEntityEnum.get_model_enum_keys()))
|
|
62
|
-
)
|
|
60
|
+
entity.set_store(ExtDataStore((idx, key) for idx, key in enumerate(RDMEntityEnum.get_model_enum_keys())))
|
|
63
61
|
period_started_at = ExtDateTimeField(
|
|
64
62
|
name='period_started_at',
|
|
65
63
|
label='Начало периода',
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
from edu_rdm_integration.stages.upload_data.models import (
|
|
2
|
-
|
|
2
|
+
RDMRequestStatus,
|
|
3
3
|
)
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
FAILED_STATUSES = {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
RDMRequestStatus.FAILED_PROCESSING,
|
|
8
|
+
RDMRequestStatus.REQUEST_ID_NOT_FOUND,
|
|
9
|
+
RDMRequestStatus.FLC_ERROR,
|
|
10
10
|
}
|
|
@@ -43,9 +43,9 @@ from edu_rdm_integration.stages.export_data.functions.base.requests import (
|
|
|
43
43
|
RegionalDataMartEntityRequest,
|
|
44
44
|
)
|
|
45
45
|
from edu_rdm_integration.stages.export_data.models import (
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
RDMExportingDataSubStage,
|
|
47
|
+
RDMExportingDataSubStageAttachment,
|
|
48
|
+
RDMExportingDataSubStageStatus,
|
|
49
49
|
)
|
|
50
50
|
from edu_rdm_integration.stages.upload_data.consts import (
|
|
51
51
|
FAILED_STATUSES,
|
|
@@ -60,9 +60,9 @@ from edu_rdm_integration.stages.upload_data.helpers import (
|
|
|
60
60
|
UploadStatusHelper,
|
|
61
61
|
)
|
|
62
62
|
from edu_rdm_integration.stages.upload_data.models import (
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
RDMExportingDataSubStageUploaderClientLog,
|
|
64
|
+
RDMRequestStatus,
|
|
65
|
+
RDMUploadStatusRequestLog,
|
|
66
66
|
)
|
|
67
67
|
from edu_rdm_integration.stages.upload_data.queues import (
|
|
68
68
|
Queue,
|
|
@@ -91,10 +91,10 @@ class ExportQueueSender:
|
|
|
91
91
|
|
|
92
92
|
При расчете используются только файлы, отмеченные как отправленные, но не обработанные витриной.
|
|
93
93
|
"""
|
|
94
|
-
sended_attachment_ids =
|
|
94
|
+
sended_attachment_ids = RDMExportingDataSubStageUploaderClientLog.objects.filter(
|
|
95
95
|
file_upload_status=FileUploadStatusEnum.IN_PROGRESS
|
|
96
96
|
).values_list('attachment_id', flat=True)
|
|
97
|
-
file_size =
|
|
97
|
+
file_size = RDMExportingDataSubStageAttachment.objects.filter(id__in=sended_attachment_ids).aggregate(
|
|
98
98
|
Sum('attachment_size')
|
|
99
99
|
)['attachment_size__sum']
|
|
100
100
|
|
|
@@ -102,18 +102,18 @@ class ExportQueueSender:
|
|
|
102
102
|
|
|
103
103
|
def _make_stage_filter(self) -> Q:
|
|
104
104
|
"""Формирование фильтра для выборки подэтапов."""
|
|
105
|
-
return Q(status_id=
|
|
105
|
+
return Q(status_id=RDMExportingDataSubStageStatus.READY_FOR_EXPORT.key)
|
|
106
106
|
|
|
107
107
|
def get_sub_stages_attachments_to_export(self):
|
|
108
108
|
"""Выборка готовых к экспорту подэтапов."""
|
|
109
109
|
sub_stage_ids = (
|
|
110
|
-
|
|
110
|
+
RDMExportingDataSubStage.objects.filter(self._make_stage_filter())
|
|
111
111
|
.order_by('started_at')
|
|
112
112
|
.values_list('id', flat=True)[: self.limit]
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
return (
|
|
116
|
-
|
|
116
|
+
RDMExportingDataSubStage.objects.filter(id__in=sub_stage_ids)
|
|
117
117
|
.annotate(
|
|
118
118
|
attachment_id=F('exportingdatasubstageattachment__id'),
|
|
119
119
|
attachment_name=F('exportingdatasubstageattachment__attachment'),
|
|
@@ -201,7 +201,7 @@ class WorkerSender:
|
|
|
201
201
|
self.entities = set()
|
|
202
202
|
self.received_file_size = 0
|
|
203
203
|
|
|
204
|
-
def get_file_upload_status(self, request_id: str) -> tuple[Optional[int], Optional[
|
|
204
|
+
def get_file_upload_status(self, request_id: str) -> tuple[Optional[int], Optional[RDMRequestStatus], 'Entry']:
|
|
205
205
|
"""Возвращает статус файла в витрине по запросу."""
|
|
206
206
|
file_upload_status = None
|
|
207
207
|
|
|
@@ -209,7 +209,7 @@ class WorkerSender:
|
|
|
209
209
|
request_status = None
|
|
210
210
|
|
|
211
211
|
if response:
|
|
212
|
-
request_status =
|
|
212
|
+
request_status = RDMRequestStatus.get_values_to_enum_data().get(response.get('code'))
|
|
213
213
|
|
|
214
214
|
if not request_status:
|
|
215
215
|
logger.error(
|
|
@@ -220,7 +220,7 @@ class WorkerSender:
|
|
|
220
220
|
if request_status in FAILED_STATUSES:
|
|
221
221
|
file_upload_status = FileUploadStatusEnum.ERROR
|
|
222
222
|
|
|
223
|
-
elif request_status ==
|
|
223
|
+
elif request_status == RDMRequestStatus.SUCCESSFULLY_PROCESSED:
|
|
224
224
|
file_upload_status = FileUploadStatusEnum.FINISHED
|
|
225
225
|
|
|
226
226
|
return file_upload_status, request_status, log_entry
|
|
@@ -229,14 +229,14 @@ class WorkerSender:
|
|
|
229
229
|
"""Отправка файлов."""
|
|
230
230
|
sub_stages = self.queue.dequeue()
|
|
231
231
|
for (sub_stage_id, entity_key), upload_files in sub_stages.items():
|
|
232
|
-
sub_stage =
|
|
232
|
+
sub_stage = RDMExportingDataSubStage.objects.filter(id=sub_stage_id).first()
|
|
233
233
|
if not sub_stage:
|
|
234
234
|
return
|
|
235
235
|
|
|
236
236
|
result_to_save = []
|
|
237
237
|
upload_status_requests_to_save = []
|
|
238
238
|
errors = []
|
|
239
|
-
status =
|
|
239
|
+
status = RDMExportingDataSubStageStatus.FINISHED.key
|
|
240
240
|
for file in upload_files:
|
|
241
241
|
if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
|
|
242
242
|
logger.warning(
|
|
@@ -250,7 +250,7 @@ class WorkerSender:
|
|
|
250
250
|
file_data = file_path.open('rb').read()
|
|
251
251
|
except (OSError, IOError, FileNotFoundError) as error:
|
|
252
252
|
logger.exception(f'Ошибка чтения файла {file_path} - {str(error)} ')
|
|
253
|
-
status =
|
|
253
|
+
status = RDMExportingDataSubStageStatus.FAILED.key
|
|
254
254
|
continue
|
|
255
255
|
|
|
256
256
|
request = RegionalDataMartEntityRequest(
|
|
@@ -274,7 +274,7 @@ class WorkerSender:
|
|
|
274
274
|
|
|
275
275
|
file_upload_status = FileUploadStatusEnum.IN_PROGRESS if request_id else FileUploadStatusEnum.ERROR
|
|
276
276
|
|
|
277
|
-
sub_stage_uploader_client_log =
|
|
277
|
+
sub_stage_uploader_client_log = RDMExportingDataSubStageUploaderClientLog(
|
|
278
278
|
entry=result.log,
|
|
279
279
|
sub_stage_id=sub_stage_id,
|
|
280
280
|
attachment_id=file.attachment_id,
|
|
@@ -286,7 +286,7 @@ class WorkerSender:
|
|
|
286
286
|
if request_id:
|
|
287
287
|
updated_file_upload_status, request_status, log_entry = self.get_file_upload_status(request_id)
|
|
288
288
|
upload_status_requests_to_save.append(
|
|
289
|
-
|
|
289
|
+
RDMUploadStatusRequestLog(
|
|
290
290
|
upload=sub_stage_uploader_client_log,
|
|
291
291
|
entry=log_entry,
|
|
292
292
|
request_status_id=getattr(request_status, 'key', None),
|
|
@@ -299,7 +299,7 @@ class WorkerSender:
|
|
|
299
299
|
self.received_file_size += file.attachment_size
|
|
300
300
|
|
|
301
301
|
if updated_file_upload_status == FileUploadStatusEnum.ERROR:
|
|
302
|
-
status =
|
|
302
|
+
status = RDMExportingDataSubStageStatus.PROCESS_ERROR.key
|
|
303
303
|
|
|
304
304
|
result_to_save.append(sub_stage_uploader_client_log)
|
|
305
305
|
if result.error:
|
|
@@ -308,15 +308,15 @@ class WorkerSender:
|
|
|
308
308
|
)
|
|
309
309
|
|
|
310
310
|
errors.append(result.error)
|
|
311
|
-
status =
|
|
311
|
+
status = RDMExportingDataSubStageStatus.FAILED.key
|
|
312
312
|
else:
|
|
313
313
|
logger.info(f'Response with {result.response.status_code} code and content {result.response.text}')
|
|
314
314
|
self.entities.add(entity_key)
|
|
315
315
|
|
|
316
316
|
# Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
|
|
317
317
|
with atomic():
|
|
318
|
-
|
|
319
|
-
|
|
318
|
+
RDMExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
|
|
319
|
+
RDMUploadStatusRequestLog.objects.bulk_create(upload_status_requests_to_save)
|
|
320
320
|
sub_stage.status_id = status
|
|
321
321
|
sub_stage.save()
|
|
322
322
|
|
|
@@ -37,7 +37,7 @@ from edu_rdm_integration.stages.export_data.functions.base.requests import (
|
|
|
37
37
|
RegionalDataMartStatusRequest,
|
|
38
38
|
)
|
|
39
39
|
from edu_rdm_integration.stages.export_data.models import (
|
|
40
|
-
|
|
40
|
+
RDMExportingDataSubStageStatus,
|
|
41
41
|
)
|
|
42
42
|
from edu_rdm_integration.stages.upload_data.consts import (
|
|
43
43
|
FAILED_STATUSES,
|
|
@@ -46,9 +46,9 @@ from edu_rdm_integration.stages.upload_data.enums import (
|
|
|
46
46
|
FileUploadStatusEnum,
|
|
47
47
|
)
|
|
48
48
|
from edu_rdm_integration.stages.upload_data.models import (
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
RDMExportingDataSubStageUploaderClientLog,
|
|
50
|
+
RDMRequestStatus,
|
|
51
|
+
RDMUploadStatusRequestLog,
|
|
52
52
|
)
|
|
53
53
|
|
|
54
54
|
|
|
@@ -113,7 +113,7 @@ class UploadStatusHelper:
|
|
|
113
113
|
@classmethod
|
|
114
114
|
def update_upload_status(
|
|
115
115
|
cls,
|
|
116
|
-
upload:
|
|
116
|
+
upload: RDMExportingDataSubStageUploaderClientLog,
|
|
117
117
|
response: Optional[dict[str, Any]],
|
|
118
118
|
log_entry: 'Entry',
|
|
119
119
|
) -> None:
|
|
@@ -121,7 +121,7 @@ class UploadStatusHelper:
|
|
|
121
121
|
request_status = None
|
|
122
122
|
|
|
123
123
|
if isinstance(response, dict):
|
|
124
|
-
request_status =
|
|
124
|
+
request_status = RDMRequestStatus.get_values_to_enum_data().get(response.get('code'))
|
|
125
125
|
|
|
126
126
|
if not request_status:
|
|
127
127
|
logger.error(
|
|
@@ -130,7 +130,7 @@ class UploadStatusHelper:
|
|
|
130
130
|
)
|
|
131
131
|
|
|
132
132
|
with transaction.atomic():
|
|
133
|
-
|
|
133
|
+
RDMUploadStatusRequestLog.objects.create(
|
|
134
134
|
upload=upload,
|
|
135
135
|
entry=log_entry,
|
|
136
136
|
request_status_id=getattr(request_status, 'key', None),
|
|
@@ -138,16 +138,16 @@ class UploadStatusHelper:
|
|
|
138
138
|
|
|
139
139
|
if request_status in FAILED_STATUSES:
|
|
140
140
|
upload.file_upload_status = FileUploadStatusEnum.ERROR
|
|
141
|
-
upload.sub_stage.status_id =
|
|
141
|
+
upload.sub_stage.status_id = RDMExportingDataSubStageStatus.PROCESS_ERROR.key
|
|
142
142
|
upload.sub_stage.save()
|
|
143
143
|
|
|
144
|
-
elif request_status ==
|
|
144
|
+
elif request_status == RDMRequestStatus.SUCCESSFULLY_PROCESSED:
|
|
145
145
|
upload.file_upload_status = FileUploadStatusEnum.FINISHED
|
|
146
146
|
|
|
147
147
|
if upload.file_upload_status != FileUploadStatusEnum.IN_PROGRESS:
|
|
148
148
|
upload.save()
|
|
149
149
|
|
|
150
|
-
def _process_upload(self, upload:
|
|
150
|
+
def _process_upload(self, upload: RDMExportingDataSubStageUploaderClientLog) -> None:
|
|
151
151
|
"""Обрабатывает запись загрузки данных в витрину."""
|
|
152
152
|
response, log_entry = self.send_upload_status_request(upload.request_id)
|
|
153
153
|
self.update_upload_status(upload, response, log_entry)
|
|
@@ -38,7 +38,7 @@ from educommon.utils.seqtools import (
|
|
|
38
38
|
)
|
|
39
39
|
|
|
40
40
|
from edu_rdm_integration.stages.export_data.models import (
|
|
41
|
-
|
|
41
|
+
RDMExportingDataSubStageAttachment,
|
|
42
42
|
)
|
|
43
43
|
from edu_rdm_integration.stages.upload_data.enums import (
|
|
44
44
|
FileUploadStatusEnum,
|
|
@@ -106,7 +106,7 @@ class Command(BaseCommand):
|
|
|
106
106
|
) -> list[tuple[int, int]]:
|
|
107
107
|
"""Формирование id файлов, которым надо обновить размер."""
|
|
108
108
|
logger.info('Определение файлов, которым надо обновить размер.')
|
|
109
|
-
attachment_query =
|
|
109
|
+
attachment_query = RDMExportingDataSubStageAttachment.objects.filter(
|
|
110
110
|
exportingdatasubstageuploaderclientlog__file_upload_status=FileUploadStatusEnum.IN_PROGRESS,
|
|
111
111
|
exportingdatasubstageuploaderclientlog__is_emulation=False,
|
|
112
112
|
attachment_size__isnull=True,
|
|
@@ -36,7 +36,7 @@ from edu_rdm_integration.stages.upload_data.helpers import (
|
|
|
36
36
|
UploadStatusHelper,
|
|
37
37
|
)
|
|
38
38
|
from edu_rdm_integration.stages.upload_data.models import (
|
|
39
|
-
|
|
39
|
+
RDMExportingDataSubStageUploaderClientLog,
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
|
|
@@ -90,7 +90,7 @@ class Command(BaseCommand):
|
|
|
90
90
|
if thread_count < 1:
|
|
91
91
|
raise ValueError(f'Количество потоков {thread_count} должно быть больше 0.')
|
|
92
92
|
|
|
93
|
-
in_progress_attachment_uploads =
|
|
93
|
+
in_progress_attachment_uploads = RDMExportingDataSubStageUploaderClientLog.objects.filter(
|
|
94
94
|
created__gte=kwargs['period_started_at'],
|
|
95
95
|
created__lte=kwargs['period_ended_at'],
|
|
96
96
|
is_emulation=False,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-07-04 08:10
|
|
2
|
+
|
|
3
|
+
from django.db import (
|
|
4
|
+
migrations,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Migration(migrations.Migration):
|
|
9
|
+
dependencies = [
|
|
10
|
+
('edu_rdm_integration_export_data_stage', '0002_auto_20250704_0810'),
|
|
11
|
+
('uploader_client', '0001_initial'),
|
|
12
|
+
('edu_rdm_integration_upload_data_stage', '0001_initial'),
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
operations = [
|
|
16
|
+
migrations.RenameModel(
|
|
17
|
+
old_name='ExportingDataSubStageUploaderClientLog',
|
|
18
|
+
new_name='RDMExportingDataSubStageUploaderClientLog',
|
|
19
|
+
),
|
|
20
|
+
migrations.RenameModel(
|
|
21
|
+
old_name='DataMartRequestStatus',
|
|
22
|
+
new_name='RDMRequestStatus',
|
|
23
|
+
),
|
|
24
|
+
migrations.RenameModel(
|
|
25
|
+
old_name='UploadStatusRequestLog',
|
|
26
|
+
new_name='RDMUploadStatusRequestLog',
|
|
27
|
+
),
|
|
28
|
+
]
|
|
@@ -23,15 +23,15 @@ from m3_db_utils.models import (
|
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
from edu_rdm_integration.stages.export_data.models import (
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
RDMExportingDataSubStage,
|
|
27
|
+
RDMExportingDataSubStageAttachment,
|
|
28
28
|
)
|
|
29
29
|
from edu_rdm_integration.stages.upload_data.enums import (
|
|
30
30
|
FileUploadStatusEnum,
|
|
31
31
|
)
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class
|
|
34
|
+
class RDMRequestStatus(TitledIntegerModelEnum):
|
|
35
35
|
"""Модель-перечисление статусов загрузки данных в Витрину."""
|
|
36
36
|
|
|
37
37
|
UPLOAD_TO_BUFFER = ModelEnumValue(
|
|
@@ -85,7 +85,7 @@ class DataMartRequestStatus(TitledIntegerModelEnum):
|
|
|
85
85
|
verbose_name_plural = 'Статусы загрузки данных в Витрину'
|
|
86
86
|
|
|
87
87
|
|
|
88
|
-
class
|
|
88
|
+
class RDMExportingDataSubStageUploaderClientLog(ReprStrPreModelMixin, BaseObjectModel):
|
|
89
89
|
"""Связь лога Загрузчика данных с подэтапом выгрузки данных."""
|
|
90
90
|
|
|
91
91
|
entry = ForeignKey(
|
|
@@ -96,13 +96,13 @@ class ExportingDataSubStageUploaderClientLog(ReprStrPreModelMixin, BaseObjectMod
|
|
|
96
96
|
)
|
|
97
97
|
|
|
98
98
|
sub_stage = ForeignKey(
|
|
99
|
-
to=
|
|
99
|
+
to=RDMExportingDataSubStage,
|
|
100
100
|
verbose_name='Подэтап выгрузки данных',
|
|
101
101
|
on_delete=CASCADE,
|
|
102
102
|
)
|
|
103
103
|
|
|
104
104
|
attachment = ForeignKey(
|
|
105
|
-
to=
|
|
105
|
+
to=RDMExportingDataSubStageAttachment,
|
|
106
106
|
verbose_name='Прикрепленный файл',
|
|
107
107
|
on_delete=CASCADE,
|
|
108
108
|
)
|
|
@@ -147,12 +147,12 @@ class ExportingDataSubStageUploaderClientLog(ReprStrPreModelMixin, BaseObjectMod
|
|
|
147
147
|
verbose_name_plural = 'Лог запроса подэтапа выгрузки данных'
|
|
148
148
|
|
|
149
149
|
|
|
150
|
-
class
|
|
150
|
+
class RDMUploadStatusRequestLog(ReprStrPreModelMixin, BaseObjectModel):
|
|
151
151
|
"""Модель связывающая статусы, загрузку файла в витрину и http-запрос к витрине."""
|
|
152
152
|
|
|
153
153
|
upload = ForeignKey(
|
|
154
154
|
verbose_name='Cвязь запроса статуса с загрузкой файла в витрину',
|
|
155
|
-
to=
|
|
155
|
+
to=RDMExportingDataSubStageUploaderClientLog,
|
|
156
156
|
on_delete=CASCADE,
|
|
157
157
|
)
|
|
158
158
|
entry = ForeignKey(
|
|
@@ -163,7 +163,7 @@ class UploadStatusRequestLog(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
163
163
|
)
|
|
164
164
|
request_status = ForeignKey(
|
|
165
165
|
verbose_name='Статус запроса в витрине',
|
|
166
|
-
to=
|
|
166
|
+
to=RDMRequestStatus,
|
|
167
167
|
on_delete=PROTECT,
|
|
168
168
|
null=True,
|
|
169
169
|
blank=True,
|
|
@@ -26,7 +26,7 @@ from edu_rdm_integration.stages.upload_data.helpers import (
|
|
|
26
26
|
UploadStatusHelper,
|
|
27
27
|
)
|
|
28
28
|
from edu_rdm_integration.stages.upload_data.models import (
|
|
29
|
-
|
|
29
|
+
RDMExportingDataSubStageUploaderClientLog,
|
|
30
30
|
)
|
|
31
31
|
from edu_rdm_integration.stages.upload_data.operations import (
|
|
32
32
|
UploadData,
|
|
@@ -55,7 +55,7 @@ class RDMCheckUploadStatus(UniquePeriodicAsyncTask):
|
|
|
55
55
|
super().process(*args, **kwargs)
|
|
56
56
|
|
|
57
57
|
# Получаем незавершенные загрузки данных в витрину
|
|
58
|
-
in_progress_uploads =
|
|
58
|
+
in_progress_uploads = RDMExportingDataSubStageUploaderClientLog.objects.filter(
|
|
59
59
|
file_upload_status=FileUploadStatusEnum.IN_PROGRESS,
|
|
60
60
|
is_emulation=False,
|
|
61
61
|
).select_related('attachment')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edu-rdm-integration
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.11.0
|
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
|
5
5
|
Author-email: BARS Group <education_dev@bars.group>
|
|
6
6
|
Project-URL: Homepage, https://stash.bars-open.ru/projects/EDUBASE/repos/edu-rdm-integration/browse
|
|
@@ -15,13 +15,14 @@ Classifier: Framework :: Django :: 2.2
|
|
|
15
15
|
Classifier: Framework :: Django :: 3.0
|
|
16
16
|
Classifier: Framework :: Django :: 3.1
|
|
17
17
|
Classifier: Framework :: Django :: 3.2
|
|
18
|
+
Classifier: Framework :: Django :: 4.0
|
|
18
19
|
Classifier: Intended Audience :: Developers
|
|
19
20
|
Classifier: Environment :: Web Environment
|
|
20
21
|
Requires-Python: >=3.9
|
|
21
22
|
Description-Content-Type: text/markdown
|
|
22
23
|
License-File: LICENSE
|
|
23
24
|
Requires-Dist: transliterate<2
|
|
24
|
-
Requires-Dist: Django<
|
|
25
|
+
Requires-Dist: Django<5.0,>=3.1
|
|
25
26
|
Requires-Dist: celery<5.3,>=4.4.7
|
|
26
27
|
Requires-Dist: asyncpg==0.23.0
|
|
27
28
|
Requires-Dist: educommon<4,>=3.11.0
|