edu-rdm-integration 2.1.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ default_app_config = 'edu_rdm_integration.collect_and_export_data.apps.AppConfig'
@@ -0,0 +1,9 @@
1
+ from django.apps import (
2
+ AppConfig as AppConfigBase,
3
+ )
4
+
5
+
6
+ class AppConfig(AppConfigBase):
7
+
8
+ name = __package__
9
+ label = 'rdm_collect_and_export_data'
@@ -0,0 +1,67 @@
1
+ # Generated by Django 3.0.14 on 2024-03-11 16:10
2
+ # flake8: noqa
3
+ import uuid
4
+
5
+ import django.contrib.postgres.fields.jsonb
6
+ import django.db.models.deletion
7
+ import django.utils.timezone
8
+ from django.db import (
9
+ migrations,
10
+ models,
11
+ )
12
+
13
+ import educommon.django.db.mixins
14
+
15
+ import edu_rdm_integration.utils
16
+
17
+
18
+ class Migration(migrations.Migration):
19
+
20
+ initial = True
21
+
22
+ dependencies = [
23
+ ('edu_rdm_integration', '0008_transferredentity'),
24
+ ('async_task', '0002_task_type_and_status_data'),
25
+ ]
26
+
27
+ operations = [
28
+ migrations.CreateModel(
29
+ name='EduRdmExportDataCommandProgress',
30
+ fields=[
31
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
32
+ ('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
33
+ ('period_started_at', models.DateTimeField(verbose_name='Левая граница периода выборки данных для выгрузки')),
34
+ ('period_ended_at', models.DateTimeField(verbose_name='Правая граница периода выборки данных для выгрузки')),
35
+ ('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
36
+ ('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Файл лога')),
37
+ ('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
38
+ ('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.RegionalDataMartEntityEnum', verbose_name='Сущность РВД')),
39
+ ('stage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration.ExportingDataStage', verbose_name='Этап выгрузки данных')),
40
+ ('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.RunningTask', verbose_name='Асинхронная задача')),
41
+ ],
42
+ options={
43
+ 'db_table': 'edu_rdm_exporting_data_command_progress',
44
+ },
45
+ bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
46
+ ),
47
+ migrations.CreateModel(
48
+ name='EduRdmCollectDataCommandProgress',
49
+ fields=[
50
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
51
+ ('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
52
+ ('logs_period_started_at', models.DateTimeField(verbose_name='Левая граница периода обрабатываемых логов')),
53
+ ('logs_period_ended_at', models.DateTimeField(verbose_name='Правая граница периода обрабатываемых логов')),
54
+ ('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
55
+ ('institute_ids', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, null=True, verbose_name='id учебного заведения')),
56
+ ('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Ссылка на файл логов')),
57
+ ('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
58
+ ('model', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.RegionalDataMartModelEnum', verbose_name='Модель РВД')),
59
+ ('stage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration.CollectingExportedDataStage', verbose_name='Этап формирования данных для выгрузки')),
60
+ ('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.RunningTask', verbose_name='Асинхронная задача')),
61
+ ],
62
+ options={
63
+ 'db_table': 'edu_rdm_collecting_data_command_progress',
64
+ },
65
+ bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
66
+ ),
67
+ ]
@@ -0,0 +1,67 @@
1
+ from django.db.models import (
2
+ SET_NULL,
3
+ FileField,
4
+ ForeignKey,
5
+ PositiveSmallIntegerField,
6
+ )
7
+
8
+ from edu_rdm_integration.enums import (
9
+ CommandType,
10
+ )
11
+ from edu_rdm_integration.models import (
12
+ AbstractCollectDataCommandProgress,
13
+ AbstractExportDataCommandProgress,
14
+ )
15
+ from edu_rdm_integration.utils import (
16
+ get_data_command_progress_attachment_path,
17
+ )
18
+
19
+
20
+ class EduRdmCollectDataCommandProgress(AbstractCollectDataCommandProgress):
21
+ """
22
+ Модель, хранящая данные для формирования и отслеживания асинхронных задач по сбору данных.
23
+ """
24
+
25
+ task = ForeignKey(
26
+ to='async_task.RunningTask',
27
+ verbose_name='Асинхронная задача',
28
+ blank=True,
29
+ null=True,
30
+ on_delete=SET_NULL,
31
+ )
32
+ logs_link = FileField(
33
+ upload_to=get_data_command_progress_attachment_path,
34
+ max_length=255,
35
+ verbose_name='Ссылка на файл логов',
36
+ )
37
+ type = PositiveSmallIntegerField( # noqa: A003
38
+ verbose_name='Тип команды',
39
+ choices=CommandType.get_choices(),
40
+ )
41
+
42
+ class Meta(AbstractCollectDataCommandProgress.Meta):
43
+ db_table = 'edu_rdm_collecting_data_command_progress'
44
+
45
+
46
+ class EduRdmExportDataCommandProgress(AbstractExportDataCommandProgress):
47
+ """Команда экспорта данных."""
48
+
49
+ task = ForeignKey(
50
+ to='async_task.RunningTask',
51
+ verbose_name='Асинхронная задача',
52
+ null=True,
53
+ blank=True,
54
+ on_delete=SET_NULL,
55
+ )
56
+ logs_link = FileField(
57
+ upload_to=get_data_command_progress_attachment_path,
58
+ max_length=255,
59
+ verbose_name='Файл лога',
60
+ )
61
+ type = PositiveSmallIntegerField( # noqa: A003
62
+ verbose_name='Тип команды',
63
+ choices=CommandType.get_choices(),
64
+ )
65
+
66
+ class Meta(AbstractExportDataCommandProgress.Meta):
67
+ db_table = 'edu_rdm_exporting_data_command_progress'
@@ -0,0 +1,77 @@
1
+ from edu_rdm_integration.collect_and_export_data.models import (
2
+ EduRdmCollectDataCommandProgress,
3
+ EduRdmExportDataCommandProgress,
4
+ )
5
+ from edu_rdm_integration.collect_data.collect import (
6
+ BaseCollectModelsDataByGeneratingLogs,
7
+ )
8
+ from edu_rdm_integration.export_data.export import (
9
+ ExportEntitiesData,
10
+ )
11
+ from edu_rdm_integration.helpers import (
12
+ save_command_log_link,
13
+ )
14
+
15
+
16
+ class CollectCommandMixin:
17
+ """Класс-примесь для запуска команды сборки моделей."""
18
+
19
+ def get_collect_command(self, command_id: int) -> EduRdmCollectDataCommandProgress:
20
+ """Возвращает экземпляр модели команды запуска."""
21
+ command = EduRdmCollectDataCommandProgress.objects.get(id=command_id)
22
+
23
+ return command
24
+
25
+ def get_collect_models_class(self):
26
+ """Возвращает класс для сбора данных."""
27
+ return BaseCollectModelsDataByGeneratingLogs
28
+
29
+ def run_collect_command(self, command) -> None:
30
+ """Запуск команды сбора."""
31
+ collect_models_data_class = self.get_collect_models_class()
32
+ collect_models_data_class(
33
+ models=(command.model_id,),
34
+ logs_period_started_at=command.logs_period_started_at,
35
+ logs_period_ended_at=command.logs_period_ended_at,
36
+ command_id=command.id,
37
+ institute_ids=command.institute_ids or (),
38
+ ).collect()
39
+
40
+ def save_collect_command_logs(self, command_id: int, log_dir: str):
41
+ """Сохранение ссылки на файл логов в команде."""
42
+ try:
43
+ command = self.get_collect_command(command_id)
44
+ except EduRdmCollectDataCommandProgress.DoesNotExist:
45
+ command = None
46
+
47
+ if command:
48
+ save_command_log_link(command, log_dir)
49
+
50
+
51
+ class ExportCommandMixin:
52
+ """Класс-примесь для запуска команды выгрузки сущностей."""
53
+
54
+ def get_export_command(self, command_id: int) -> EduRdmExportDataCommandProgress:
55
+ """Возвращает экземпляр модели команды запуска."""
56
+ command = EduRdmExportDataCommandProgress.objects.get(id=command_id)
57
+
58
+ return command
59
+
60
+ def run_export_command(self, command: EduRdmExportDataCommandProgress) -> None:
61
+ """Запуск команды выгрузки."""
62
+ ExportEntitiesData(
63
+ entities=(command.entity_id,),
64
+ period_started_at=command.period_started_at,
65
+ period_ended_at=command.period_ended_at,
66
+ command_id=command.id,
67
+ ).export()
68
+
69
+ def save_export_command_logs(self, command_id: int, log_dir: str):
70
+ """Сохранение ссылки на файл логов в команде."""
71
+ try:
72
+ command = self.get_export_command(command_id)
73
+ except EduRdmExportDataCommandProgress.DoesNotExist:
74
+ command = None
75
+
76
+ if command:
77
+ save_command_log_link(command, log_dir)
@@ -154,3 +154,15 @@ class EntityRetiredStatusEnum(ValueCodeEnumerate):
154
154
  RETIRED: 'Прекращена',
155
155
  ENTERED_IN_ERROR: 'Создана по ошибке',
156
156
  }
157
+
158
+
159
+ class CommandType(BaseEnumerate):
160
+ """Тип команды."""
161
+
162
+ AUTO = 1
163
+ MANUAL = 2
164
+
165
+ values = {
166
+ AUTO: 'Автоматический',
167
+ MANUAL: 'Ручной',
168
+ }
@@ -1,4 +1,5 @@
1
1
  import heapq
2
+ import os
2
3
  from concurrent.futures import (
3
4
  ThreadPoolExecutor,
4
5
  )
@@ -12,8 +13,12 @@ from typing import (
12
13
  List,
13
14
  Optional,
14
15
  Tuple,
16
+ Union,
15
17
  )
16
18
 
19
+ from django.conf import (
20
+ settings,
21
+ )
17
22
  from django.db import (
18
23
  transaction,
19
24
  )
@@ -28,6 +33,10 @@ from educommon import (
28
33
  logger,
29
34
  )
30
35
 
36
+ from edu_rdm_integration.collect_and_export_data.models import (
37
+ EduRdmCollectDataCommandProgress,
38
+ EduRdmExportDataCommandProgress,
39
+ )
31
40
  from edu_rdm_integration.enums import (
32
41
  FileUploadStatusEnum,
33
42
  )
@@ -151,7 +160,7 @@ class Graph:
151
160
  """
152
161
 
153
162
  def __init__(self):
154
- self.vertices: Dict[str, Dict[str, Optional[str]]] = {}
163
+ self.vertices: Dict[str, Dict[str, Optional[str]]] = {}
155
164
  """Словарь для хранения данных графа."""
156
165
 
157
166
  def add_vertex(self, vertex: str):
@@ -261,3 +270,14 @@ class Graph:
261
270
  path.reverse()
262
271
 
263
272
  return path
273
+
274
+
275
+ def save_command_log_link(
276
+ command: Union[EduRdmCollectDataCommandProgress, EduRdmExportDataCommandProgress],
277
+ log_dir: str
278
+ ) -> None:
279
+ """Сохраняет ссылку на лог команды."""
280
+ log_file = os.path.join(settings.MEDIA_ROOT, log_dir, f'{command.id}.log')
281
+ if os.path.exists(log_file):
282
+ command.logs_link = os.path.join(log_dir, f'{command.id}.log')
283
+ command.save()
@@ -1,3 +1,13 @@
1
+ from itertools import (
2
+ chain,
3
+ )
4
+ from typing import (
5
+ TYPE_CHECKING,
6
+ Dict,
7
+ Optional,
8
+ Type,
9
+ )
10
+
1
11
  import celery
2
12
  from celery.schedules import (
3
13
  crontab,
@@ -5,34 +15,84 @@ from celery.schedules import (
5
15
  from django.conf import (
6
16
  settings,
7
17
  )
18
+ from django.db.models import (
19
+ CharField,
20
+ Max,
21
+ )
22
+ from django.db.models.functions import (
23
+ Cast,
24
+ )
25
+ from django.utils import (
26
+ timezone,
27
+ )
8
28
 
9
29
  from educommon.async_task.models import (
10
30
  AsyncTaskType,
31
+ RunningTask,
11
32
  )
12
33
  from educommon.async_task.tasks import (
13
34
  PeriodicAsyncTask,
14
35
  )
36
+ from educommon.utils.date import (
37
+ get_today_min_datetime,
38
+ )
39
+ from m3_db_utils.consts import (
40
+ DEFAULT_ORDER_NUMBER,
41
+ )
15
42
 
43
+ from edu_rdm_integration.collect_and_export_data.models import (
44
+ EduRdmCollectDataCommandProgress,
45
+ EduRdmExportDataCommandProgress,
46
+ )
47
+ from edu_rdm_integration.collect_data.collect import (
48
+ BaseCollectLatestModelsData,
49
+ )
16
50
  from edu_rdm_integration.collect_data.helpers import (
17
51
  set_failed_status_suspended_collecting_data_stages,
18
52
  )
19
53
  from edu_rdm_integration.consts import (
54
+ REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
55
+ REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
20
56
  TASK_QUEUE_NAME,
21
57
  )
22
58
  from edu_rdm_integration.enums import (
59
+ CommandType,
23
60
  FileUploadStatusEnum,
24
61
  )
62
+ from edu_rdm_integration.export_data.export import (
63
+ ExportLatestEntitiesData,
64
+ )
25
65
  from edu_rdm_integration.export_data.helpers import (
26
66
  set_failed_status_suspended_exporting_data_stages,
27
67
  )
28
68
  from edu_rdm_integration.helpers import (
29
69
  UploadStatusHelper,
70
+ save_command_log_link,
30
71
  )
31
72
  from edu_rdm_integration.models import (
73
+ CollectingDataSubStageStatus,
74
+ CollectingExportedDataSubStage,
75
+ ExportingDataSubStage,
76
+ ExportingDataSubStageStatus,
32
77
  ExportingDataSubStageUploaderClientLog,
78
+ RegionalDataMartEntityEnum,
79
+ TransferredEntity,
80
+ )
81
+ from edu_rdm_integration.storages import (
82
+ RegionalDataMartEntityStorage,
33
83
  )
34
84
 
35
85
 
86
+ if TYPE_CHECKING:
87
+ from datetime import (
88
+ datetime,
89
+ )
90
+
91
+ from function_tools.managers import (
92
+ RunnerManager,
93
+ )
94
+
95
+
36
96
  class RDMCheckUploadStatus(PeriodicAsyncTask):
37
97
  """Периодическая задача для сбора статусов по загрузке файла в витрину."""
38
98
 
@@ -95,6 +155,185 @@ class CheckSuspendedExportedStagePeriodicTask(PeriodicAsyncTask):
95
155
  )
96
156
 
97
157
 
158
+ class TransferLatestEntitiesDataPeriodicTask(PeriodicAsyncTask):
159
+ """Периодическая задача сбора и выгрузки данных."""
160
+
161
+ queue = TASK_QUEUE_NAME
162
+ routing_key = TASK_QUEUE_NAME
163
+ description = 'Периодическая задача сбора и экспорта данных РВД'
164
+ task_type = AsyncTaskType.UNKNOWN
165
+ run_every = crontab(
166
+ minute=settings.RDM_TRANSFER_TASK_MINUTE,
167
+ hour=settings.RDM_TRANSFER_TASK_HOUR,
168
+ day_of_week=settings.RDM_TRANSFER_TASK_DAY_OF_WEEK,
169
+ )
170
+
171
+ def __init__(self) -> None:
172
+ super().__init__()
173
+
174
+ self._collecting_data_managers: Dict[str, Type['RunnerManager']] = {}
175
+ self._collecting_data_manager_to_logs_period_end: Dict[str, datetime] = {}
176
+
177
+ self._exporting_data_managers: Dict[str, Type['RunnerManager']] = {}
178
+ self._exporting_data_manager_to_period_end: Dict[str, datetime] = {}
179
+
180
+ self._transferred_entities = []
181
+
182
+ def process(self, *args, **kwargs):
183
+ """Выполняет задачу."""
184
+ super().process(*args, **kwargs)
185
+
186
+ self._collect_transferred_entities()
187
+ self._collect_managers()
188
+ self._calculate_collecting_managers_logs_period_ended_at()
189
+ self._calculate_exporting_managers_ended_at()
190
+
191
+ task_id = RunningTask.objects.filter(
192
+ pk=self.request.id,
193
+ ).values_list('pk', flat=True).first()
194
+
195
+ model_enum_values = RegionalDataMartEntityEnum.get_entities_model_enums(
196
+ entity_enums=self._transferred_entities,
197
+ is_sorted=True,
198
+ )
199
+
200
+ filtered_model_enum_values = filter(lambda model: model.order_number != DEFAULT_ORDER_NUMBER, model_enum_values)
201
+ for model_enum_value in filtered_model_enum_values:
202
+ self._run_collect_model_data(model_enum_value.key, task_id)
203
+
204
+ for entity_enum_value in sorted(self._transferred_entities, key=lambda entity: entity.order_number):
205
+ self._run_export_entity_data(entity_enum_value.key, task_id)
206
+
207
+ def _run_collect_model_data(self, model: str, task_id: str) -> None:
208
+ """Запускает сбор данных модели РВД."""
209
+ command = self._create_collect_command(model, task_id)
210
+ collect_model_data = self._prepare_collect_model_data_class(command)
211
+ collect_model_data.collect()
212
+
213
+ command.refresh_from_db(fields=['stage_id'])
214
+ save_command_log_link(command, settings.RDM_COLLECT_LOG_DIR)
215
+
216
+ def _run_export_entity_data(self, entity: str, task_id: str) -> None:
217
+ """Запускает экспорт данных сущности РВД."""
218
+ command = self._create_export_command(entity, task_id)
219
+ if command:
220
+ export_entity_data = self._prepare_export_entity_data_class(command)
221
+ export_entity_data.export()
222
+
223
+ command.refresh_from_db(fields=['stage_id'])
224
+ save_command_log_link(command, settings.RDM_EXPORT_LOG_DIR)
225
+
226
+ def _collect_transferred_entities(self) -> None:
227
+ """Собирает сущности РВД, по которым будет произведен сбор и экспорт данных."""
228
+ self._transferred_entities = [
229
+ RegionalDataMartEntityEnum.get_model_enum_value(key=entity)
230
+ for entity in TransferredEntity.objects.values_list('entity', flat=True)
231
+ ]
232
+
233
+ def _collect_managers(self) -> None:
234
+ """Собирает менеджеры Функций для сбора и выгрузки данных."""
235
+ entity_storage = RegionalDataMartEntityStorage()
236
+ entity_storage.prepare()
237
+
238
+ collecting_models_data_managers_map = entity_storage.prepare_entities_manager_map(
239
+ tags={REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA},
240
+ )
241
+ exporting_entities_data_managers_map = entity_storage.prepare_entities_manager_map(
242
+ tags={REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA},
243
+ )
244
+
245
+ for transferred_entity in self._transferred_entities:
246
+ for model in chain((transferred_entity.main_model_enum, ), transferred_entity.additional_model_enums):
247
+ collect_manager_class = collecting_models_data_managers_map.get(model.key)
248
+ if collect_manager_class:
249
+ self._collecting_data_managers[model.key] = collect_manager_class
250
+
251
+ export_manager_class = exporting_entities_data_managers_map.get(transferred_entity.key)
252
+ if export_manager_class:
253
+ self._exporting_data_managers[transferred_entity.key] = export_manager_class
254
+
255
+ def _calculate_collecting_managers_logs_period_ended_at(self) -> None:
256
+ """Определяет дату последнего успешного этапа сбора у менеджеров Функций сбора."""
257
+ self._collecting_data_manager_to_logs_period_end = dict(
258
+ CollectingExportedDataSubStage.objects.annotate(
259
+ manager_id=Cast('stage__manager_id', output_field=CharField()),
260
+ max_logs_period_ended_at=Max('stage__logs_period_ended_at'),
261
+ ).filter(
262
+ status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
263
+ manager_id__in=[manager.uuid for manager in self._collecting_data_managers.values()],
264
+ ).values_list('manager_id', 'max_logs_period_ended_at'),
265
+ )
266
+
267
+ def _calculate_exporting_managers_ended_at(self) -> None:
268
+ """Определяет дату последнего успешного подэтапа экспорта у менеджеров Функций экспорта."""
269
+ self._exporting_data_manager_to_period_end = dict(
270
+ ExportingDataSubStage.objects.annotate(
271
+ manager_id=Cast('stage__manager_id', output_field=CharField()),
272
+ max_ended_at=Max('ended_at'),
273
+ ).filter(
274
+ status_id=ExportingDataSubStageStatus.FINISHED.key,
275
+ manager_id__in=[manager.uuid for manager in self._exporting_data_managers.values()],
276
+ ).values_list('manager_id', 'max_ended_at'),
277
+ )
278
+
279
+ def _create_collect_command(self, model: str, task_id: str) -> EduRdmCollectDataCommandProgress:
280
+ """Создает команду сбора данных моделей РВД."""
281
+ manager = self._collecting_data_managers[model]
282
+ logs_period_started_at = (
283
+ self._collecting_data_manager_to_logs_period_end.get(manager.uuid)
284
+ or get_today_min_datetime()
285
+ )
286
+
287
+ return EduRdmCollectDataCommandProgress.objects.create(
288
+ model_id=model,
289
+ logs_period_started_at=logs_period_started_at,
290
+ logs_period_ended_at=timezone.now(),
291
+ task_id=task_id,
292
+ type=CommandType.AUTO,
293
+ )
294
+
295
+ def _create_export_command(self, entity: str, task_id: str) -> Optional[EduRdmExportDataCommandProgress]:
296
+ """Создает команду экспорта данных сущностей РВД."""
297
+ manager = self._exporting_data_managers[entity]
298
+ period_started_at = self._exporting_data_manager_to_period_end.get(manager.uuid)
299
+
300
+ if period_started_at:
301
+ return EduRdmExportDataCommandProgress.objects.create(
302
+ entity_id=entity,
303
+ period_started_at=period_started_at,
304
+ period_ended_at=timezone.now(),
305
+ task_id=task_id,
306
+ type=CommandType.AUTO,
307
+ )
308
+
309
+ def _prepare_collect_model_data_class(
310
+ self,
311
+ command: EduRdmCollectDataCommandProgress
312
+ ) -> BaseCollectLatestModelsData:
313
+ """Подготавливает объект класса сбора данных моделей РВД."""
314
+ return BaseCollectLatestModelsData(
315
+ models=[command.model_id],
316
+ logs_period_started_at=command.logs_period_started_at,
317
+ logs_period_ended_at=command.logs_period_ended_at,
318
+ command_id=command.id,
319
+ )
320
+
321
+ def _prepare_export_entity_data_class(self, command: EduRdmExportDataCommandProgress) -> ExportLatestEntitiesData:
322
+ """Подготавливает объект класса экспорта данных сущностей РВД.
323
+
324
+ При экспорте данных передаем параметр task_id для обновления поля "Описание"
325
+ наименованиями выгруженных сущностей.
326
+ """
327
+ return ExportLatestEntitiesData(
328
+ entities=[command.entity_id],
329
+ period_started_at=command.period_started_at,
330
+ period_ended_at=command.period_ended_at,
331
+ command_id=command.id,
332
+ task_id=self.request.id,
333
+ )
334
+
335
+
98
336
  celery_app = celery.app.app_or_default()
99
337
  celery_app.register_task(RDMCheckUploadStatus)
100
338
  celery_app.register_task(CheckSuspendedExportedStagePeriodicTask)
339
+ celery_app.register_task(TransferLatestEntitiesDataPeriodicTask)
@@ -17,7 +17,6 @@ from typing import (
17
17
  from django.conf import (
18
18
  settings,
19
19
  )
20
-
21
20
  from edu_rdm_integration.apps import (
22
21
  EduRDMIntegrationConfig,
23
22
  )
@@ -25,6 +24,7 @@ from edu_rdm_integration.consts import (
25
24
  ACADEMIC_YEAR,
26
25
  )
27
26
 
27
+
28
28
  if TYPE_CHECKING:
29
29
  from typing import (
30
30
  NamedTuple,
@@ -34,6 +34,10 @@ if TYPE_CHECKING:
34
34
  Model,
35
35
  )
36
36
 
37
+ from edu_rdm_integration.collect_and_export_data.models import (
38
+ EduRdmCollectDataCommandProgress,
39
+ EduRdmExportDataCommandProgress,
40
+ )
37
41
  from edu_rdm_integration.helpers import (
38
42
  Graph,
39
43
  )
@@ -256,3 +260,27 @@ def build_related_model_graph(
256
260
  graph.add_edge(current_model._meta.model_name, related_model._meta.model_name, field.name)
257
261
 
258
262
  return graph
263
+
264
+
265
+ def get_data_command_progress_attachment_path(
266
+ instance: Union['EduRdmCollectDataCommandProgress', 'EduRdmExportDataCommandProgress'],
267
+ filename: str
268
+ ) -> str:
269
+ """Возвращает путь до файла-вложения при формировании команды сборки или выгрузки данных.
270
+
271
+ Args:
272
+ instance: объект ExportDataCommandProgress или объект CollectDataCommandProgres
273
+ filename: имя загружаемого файла
274
+
275
+ Returns:
276
+ Строковое представление пути
277
+ """
278
+ date_now = datetime.now()
279
+
280
+ return os.path.join(
281
+ settings.UPLOADS,
282
+ date_now.strftime('%Y/%m/%d'),
283
+ instance.__class__.__name__.lower(),
284
+ str(instance.type),
285
+ filename
286
+ )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: edu-rdm-integration
3
- Version: 2.1.0
3
+ Version: 2.2.1
4
4
  Summary: Интеграция с Региональной витриной данных
5
5
  Home-page:
6
6
  Download-URL:
@@ -293,6 +293,27 @@ Requires-Dist: uploader-client <1,>=0.2.1
293
293
 
294
294
  ### Удалено
295
295
 
296
+ ## [2.2.1] - 2024-03-28
297
+ Исправление наследования Meta в моделях EduRdmCollectDataCommandProgress и EduRdmExportDataCommandProgress
298
+
299
+ ### Исправлено
300
+
301
+ - [EDUSCHL-21569](https://jira.bars.group/browse/EDUSCHL-21569)
302
+ MINOR Перенести оставшиеся общие асинхронные задачи из ЭШ в пакет edu_rdm_integration
303
+
304
+ ## [2.2.0] - 2024-03-20
305
+
306
+ Из ЭШ перенесена периодическая задача по сбору и выгрузке данных в РВД.
307
+
308
+ Также из ЭШ перенесены и переименованы модели:
309
+ - CollectDataCommandProgress перименована в EduRdmCollectDataCommandProgress
310
+ - ExportDataCommandProgress перименована в EduRdmExportDataCommandProgress
311
+ - Добавлены миксины CollectCommandMixin и ExportCommandMixin
312
+
313
+ ### Добавлено
314
+
315
+ - [EDUSCHL-21569](https://jira.bars.group/browse/EDUSCHL-21569)
316
+ MINOR Перенести оставшиеся общие асинхронные задачи из ЭШ в пакет edu_rdm_integration
296
317
 
297
318
  ## [2.1.0] - 2024-03-18
298
319
 
@@ -5,14 +5,14 @@ edu_rdm_integration/apps.py,sha256=5OgNdmuqe26fbu4wYb69haQJe-XFO_rDbnU1vPqJU-U,3
5
5
  edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
6
6
  edu_rdm_integration/consts.py,sha256=FFwcMHNsfjP_s9LfkccLAHjJMEMp7ppPmrRlJcgV88k,1104
7
7
  edu_rdm_integration/entities.py,sha256=mAjsYlcIbemo4xT5CSCr4payZubiBHB7Rb3Ow1CVsy0,14552
8
- edu_rdm_integration/enums.py,sha256=6Gv_hpYrC6v75ZtBA_xBrHqvza9NbJKhMa1TdTHkzys,4048
9
- edu_rdm_integration/helpers.py,sha256=akFN1D_hcu8HRgx-LoJKe9u729gHp7-B0eS7CeBIsH0,10373
8
+ edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
9
+ edu_rdm_integration/helpers.py,sha256=wsVd3GiO7qgb_jtZBqEuedBTW8Q9-j2xcfMBi45O0cw,10997
10
10
  edu_rdm_integration/mapping.py,sha256=bwa2fJCbV4YjQcAgRrgT3hgM6dJhr_uBtQgx3L3F2Ck,473
11
11
  edu_rdm_integration/models.py,sha256=Wg8NYj4C6Xqw-15AsQGU0TWNzUjdwBW_YZDzmPqx3GY,29327
12
12
  edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
13
13
  edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
14
- edu_rdm_integration/tasks.py,sha256=EJD4jkIJoWwhX5YZRTZjOP5hLmGv7HEOB8w6NIHDPBw,3591
15
- edu_rdm_integration/utils.py,sha256=EUf1gt7YzlVH3mIX1M70b7buoox7F_ELwLTomoSENwo,9397
14
+ edu_rdm_integration/tasks.py,sha256=8bQZbhVDuK3RYc8lSbWyW1xDyVgIZQCrcsTo8pUeIeE,13716
15
+ edu_rdm_integration/utils.py,sha256=-my8q9fude6Nc9r_qUDww-8QaU2tcv-xOexewYqOPJw,10369
16
16
  edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
17
17
  edu_rdm_integration/adapters/apps.py,sha256=TyJTkSPs2qAHJ11fqbwLGk3Ea7ujtqWwbxqmvYNQxG8,363
18
18
  edu_rdm_integration/adapters/caches.py,sha256=OxSqeXySUN42LxEeHBLtC1ZBt-7aicbRbmP1EJYTvV4,1505
@@ -30,6 +30,12 @@ edu_rdm_integration/adapters/strategies.py,sha256=ztEHvdMYjJzjjkV7J9XtuNhn_uUVoJ
30
30
  edu_rdm_integration/adapters/strings.py,sha256=-k9dex8A7hCpkzUkudVkKRAbNRuuqog2hYl2xmibl8I,181
31
31
  edu_rdm_integration/adapters/tests.py,sha256=MoRY-a75Ow-7EjeQYxkXWunwqTGuBMaUyEkEV2oy05I,59
32
32
  edu_rdm_integration/adapters/validators.py,sha256=LJWnCY8PtXDOj-fm3fBWjQYsHsSLfyKf_D97pqPv73s,496
33
+ edu_rdm_integration/collect_and_export_data/__init__.py,sha256=4glPgPCzAyLreBGUnUrcRPCge45XucJz5bK8VjlQBaE,82
34
+ edu_rdm_integration/collect_and_export_data/apps.py,sha256=fAcctcjxWP4Gd0Qr3YrQkMESrzga4IORWetBTy8wvHo,160
35
+ edu_rdm_integration/collect_and_export_data/models.py,sha256=TGm-hM-1aDhVcFPy0PTC3yNczoZGF4ZmIdr0Y9s8CfU,2038
36
+ edu_rdm_integration/collect_and_export_data/utils.py,sha256=uw85W_uPr7zGa9rA4Bp1nDUrOMnlz0nZ92AZAgNsRlE,3073
37
+ edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py,sha256=UkoaXzh3tokZ8QdCdB09v3rRZfcHhvEwNMuj3mQIB74,4714
38
+ edu_rdm_integration/collect_and_export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
39
  edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
40
  edu_rdm_integration/collect_data/collect.py,sha256=TfjDD25opR6UlV6oWbMXs3T7nQguSg5IT0k2oFasWJI,14543
35
41
  edu_rdm_integration/collect_data/generators.py,sha256=f34AAwdEcQNIokX0ypqYgjRD1XolwBVLER_HYv9ibNw,9075
@@ -156,9 +162,9 @@ edu_rdm_integration/uploader_log/apps.py,sha256=tYJj4-sDlq8fLOSvw18L_yys7SILpTKW
156
162
  edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJLdsGmr3SXwH_U,266
157
163
  edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
158
164
  edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
159
- edu_rdm_integration-2.1.0.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
160
- edu_rdm_integration-2.1.0.dist-info/METADATA,sha256=ji5SEODXhqO13VaW0Pz8k2dwn6RsQEU-z1Czk48DaCQ,57138
161
- edu_rdm_integration-2.1.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
162
- edu_rdm_integration-2.1.0.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
163
- edu_rdm_integration-2.1.0.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
164
- edu_rdm_integration-2.1.0.dist-info/RECORD,,
165
+ edu_rdm_integration-2.2.1.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
166
+ edu_rdm_integration-2.2.1.dist-info/METADATA,sha256=0K4JnLA9jXs0MxuvyJlFL0IaolV9Oil6ppFcTemQhxY,58264
167
+ edu_rdm_integration-2.2.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
168
+ edu_rdm_integration-2.2.1.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
169
+ edu_rdm_integration-2.2.1.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
170
+ edu_rdm_integration-2.2.1.dist-info/RECORD,,