edu-rdm-integration 2.1.0__py3-none-any.whl → 2.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/collect_and_export_data/__init__.py +1 -0
- edu_rdm_integration/collect_and_export_data/apps.py +9 -0
- edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py +67 -0
- edu_rdm_integration/collect_and_export_data/migrations/__init__.py +0 -0
- edu_rdm_integration/collect_and_export_data/models.py +67 -0
- edu_rdm_integration/collect_and_export_data/utils.py +77 -0
- edu_rdm_integration/enums.py +12 -0
- edu_rdm_integration/helpers.py +21 -1
- edu_rdm_integration/tasks.py +239 -0
- edu_rdm_integration/utils.py +29 -1
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/METADATA +22 -1
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/RECORD +16 -10
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/LICENSE +0 -0
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/namespace_packages.txt +0 -0
- {edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
default_app_config = 'edu_rdm_integration.collect_and_export_data.apps.AppConfig'
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# Generated by Django 3.0.14 on 2024-03-11 16:10
|
2
|
+
# flake8: noqa
|
3
|
+
import uuid
|
4
|
+
|
5
|
+
import django.contrib.postgres.fields.jsonb
|
6
|
+
import django.db.models.deletion
|
7
|
+
import django.utils.timezone
|
8
|
+
from django.db import (
|
9
|
+
migrations,
|
10
|
+
models,
|
11
|
+
)
|
12
|
+
|
13
|
+
import educommon.django.db.mixins
|
14
|
+
|
15
|
+
import edu_rdm_integration.utils
|
16
|
+
|
17
|
+
|
18
|
+
class Migration(migrations.Migration):
|
19
|
+
|
20
|
+
initial = True
|
21
|
+
|
22
|
+
dependencies = [
|
23
|
+
('edu_rdm_integration', '0008_transferredentity'),
|
24
|
+
('async_task', '0002_task_type_and_status_data'),
|
25
|
+
]
|
26
|
+
|
27
|
+
operations = [
|
28
|
+
migrations.CreateModel(
|
29
|
+
name='EduRdmExportDataCommandProgress',
|
30
|
+
fields=[
|
31
|
+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
32
|
+
('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
|
33
|
+
('period_started_at', models.DateTimeField(verbose_name='Левая граница периода выборки данных для выгрузки')),
|
34
|
+
('period_ended_at', models.DateTimeField(verbose_name='Правая граница периода выборки данных для выгрузки')),
|
35
|
+
('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
|
36
|
+
('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Файл лога')),
|
37
|
+
('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
|
38
|
+
('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.RegionalDataMartEntityEnum', verbose_name='Сущность РВД')),
|
39
|
+
('stage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration.ExportingDataStage', verbose_name='Этап выгрузки данных')),
|
40
|
+
('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.RunningTask', verbose_name='Асинхронная задача')),
|
41
|
+
],
|
42
|
+
options={
|
43
|
+
'db_table': 'edu_rdm_exporting_data_command_progress',
|
44
|
+
},
|
45
|
+
bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
|
46
|
+
),
|
47
|
+
migrations.CreateModel(
|
48
|
+
name='EduRdmCollectDataCommandProgress',
|
49
|
+
fields=[
|
50
|
+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
51
|
+
('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
|
52
|
+
('logs_period_started_at', models.DateTimeField(verbose_name='Левая граница периода обрабатываемых логов')),
|
53
|
+
('logs_period_ended_at', models.DateTimeField(verbose_name='Правая граница периода обрабатываемых логов')),
|
54
|
+
('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
|
55
|
+
('institute_ids', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, null=True, verbose_name='id учебного заведения')),
|
56
|
+
('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Ссылка на файл логов')),
|
57
|
+
('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
|
58
|
+
('model', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.RegionalDataMartModelEnum', verbose_name='Модель РВД')),
|
59
|
+
('stage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration.CollectingExportedDataStage', verbose_name='Этап формирования данных для выгрузки')),
|
60
|
+
('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.RunningTask', verbose_name='Асинхронная задача')),
|
61
|
+
],
|
62
|
+
options={
|
63
|
+
'db_table': 'edu_rdm_collecting_data_command_progress',
|
64
|
+
},
|
65
|
+
bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
|
66
|
+
),
|
67
|
+
]
|
File without changes
|
@@ -0,0 +1,67 @@
|
|
1
|
+
from django.db.models import (
|
2
|
+
SET_NULL,
|
3
|
+
FileField,
|
4
|
+
ForeignKey,
|
5
|
+
PositiveSmallIntegerField,
|
6
|
+
)
|
7
|
+
|
8
|
+
from edu_rdm_integration.enums import (
|
9
|
+
CommandType,
|
10
|
+
)
|
11
|
+
from edu_rdm_integration.models import (
|
12
|
+
AbstractCollectDataCommandProgress,
|
13
|
+
AbstractExportDataCommandProgress,
|
14
|
+
)
|
15
|
+
from edu_rdm_integration.utils import (
|
16
|
+
get_data_command_progress_attachment_path,
|
17
|
+
)
|
18
|
+
|
19
|
+
|
20
|
+
class EduRdmCollectDataCommandProgress(AbstractCollectDataCommandProgress):
|
21
|
+
"""
|
22
|
+
Модель, хранящая данные для формирования и отслеживания асинхронных задач по сбору данных.
|
23
|
+
"""
|
24
|
+
|
25
|
+
task = ForeignKey(
|
26
|
+
to='async_task.RunningTask',
|
27
|
+
verbose_name='Асинхронная задача',
|
28
|
+
blank=True,
|
29
|
+
null=True,
|
30
|
+
on_delete=SET_NULL,
|
31
|
+
)
|
32
|
+
logs_link = FileField(
|
33
|
+
upload_to=get_data_command_progress_attachment_path,
|
34
|
+
max_length=255,
|
35
|
+
verbose_name='Ссылка на файл логов',
|
36
|
+
)
|
37
|
+
type = PositiveSmallIntegerField( # noqa: A003
|
38
|
+
verbose_name='Тип команды',
|
39
|
+
choices=CommandType.get_choices(),
|
40
|
+
)
|
41
|
+
|
42
|
+
class Meta(AbstractCollectDataCommandProgress.Meta):
|
43
|
+
db_table = 'edu_rdm_collecting_data_command_progress'
|
44
|
+
|
45
|
+
|
46
|
+
class EduRdmExportDataCommandProgress(AbstractExportDataCommandProgress):
|
47
|
+
"""Команда экспорта данных."""
|
48
|
+
|
49
|
+
task = ForeignKey(
|
50
|
+
to='async_task.RunningTask',
|
51
|
+
verbose_name='Асинхронная задача',
|
52
|
+
null=True,
|
53
|
+
blank=True,
|
54
|
+
on_delete=SET_NULL,
|
55
|
+
)
|
56
|
+
logs_link = FileField(
|
57
|
+
upload_to=get_data_command_progress_attachment_path,
|
58
|
+
max_length=255,
|
59
|
+
verbose_name='Файл лога',
|
60
|
+
)
|
61
|
+
type = PositiveSmallIntegerField( # noqa: A003
|
62
|
+
verbose_name='Тип команды',
|
63
|
+
choices=CommandType.get_choices(),
|
64
|
+
)
|
65
|
+
|
66
|
+
class Meta(AbstractExportDataCommandProgress.Meta):
|
67
|
+
db_table = 'edu_rdm_exporting_data_command_progress'
|
@@ -0,0 +1,77 @@
|
|
1
|
+
from edu_rdm_integration.collect_and_export_data.models import (
|
2
|
+
EduRdmCollectDataCommandProgress,
|
3
|
+
EduRdmExportDataCommandProgress,
|
4
|
+
)
|
5
|
+
from edu_rdm_integration.collect_data.collect import (
|
6
|
+
BaseCollectModelsDataByGeneratingLogs,
|
7
|
+
)
|
8
|
+
from edu_rdm_integration.export_data.export import (
|
9
|
+
ExportEntitiesData,
|
10
|
+
)
|
11
|
+
from edu_rdm_integration.helpers import (
|
12
|
+
save_command_log_link,
|
13
|
+
)
|
14
|
+
|
15
|
+
|
16
|
+
class CollectCommandMixin:
|
17
|
+
"""Класс-примесь для запуска команды сборки моделей."""
|
18
|
+
|
19
|
+
def get_collect_command(self, command_id: int) -> EduRdmCollectDataCommandProgress:
|
20
|
+
"""Возвращает экземпляр модели команды запуска."""
|
21
|
+
command = EduRdmCollectDataCommandProgress.objects.get(id=command_id)
|
22
|
+
|
23
|
+
return command
|
24
|
+
|
25
|
+
def get_collect_models_class(self):
|
26
|
+
"""Возвращает класс для сбора данных."""
|
27
|
+
return BaseCollectModelsDataByGeneratingLogs
|
28
|
+
|
29
|
+
def run_collect_command(self, command) -> None:
|
30
|
+
"""Запуск команды сбора."""
|
31
|
+
collect_models_data_class = self.get_collect_models_class()
|
32
|
+
collect_models_data_class(
|
33
|
+
models=(command.model_id,),
|
34
|
+
logs_period_started_at=command.logs_period_started_at,
|
35
|
+
logs_period_ended_at=command.logs_period_ended_at,
|
36
|
+
command_id=command.id,
|
37
|
+
institute_ids=command.institute_ids or (),
|
38
|
+
).collect()
|
39
|
+
|
40
|
+
def save_collect_command_logs(self, command_id: int, log_dir: str):
|
41
|
+
"""Сохранение ссылки на файл логов в команде."""
|
42
|
+
try:
|
43
|
+
command = self.get_collect_command(command_id)
|
44
|
+
except EduRdmCollectDataCommandProgress.DoesNotExist:
|
45
|
+
command = None
|
46
|
+
|
47
|
+
if command:
|
48
|
+
save_command_log_link(command, log_dir)
|
49
|
+
|
50
|
+
|
51
|
+
class ExportCommandMixin:
|
52
|
+
"""Класс-примесь для запуска команды выгрузки сущностей."""
|
53
|
+
|
54
|
+
def get_export_command(self, command_id: int) -> EduRdmExportDataCommandProgress:
|
55
|
+
"""Возвращает экземпляр модели команды запуска."""
|
56
|
+
command = EduRdmExportDataCommandProgress.objects.get(id=command_id)
|
57
|
+
|
58
|
+
return command
|
59
|
+
|
60
|
+
def run_export_command(self, command: EduRdmExportDataCommandProgress) -> None:
|
61
|
+
"""Запуск команды выгрузки."""
|
62
|
+
ExportEntitiesData(
|
63
|
+
entities=(command.entity_id,),
|
64
|
+
period_started_at=command.period_started_at,
|
65
|
+
period_ended_at=command.period_ended_at,
|
66
|
+
command_id=command.id,
|
67
|
+
).export()
|
68
|
+
|
69
|
+
def save_export_command_logs(self, command_id: int, log_dir: str):
|
70
|
+
"""Сохранение ссылки на файл логов в команде."""
|
71
|
+
try:
|
72
|
+
command = self.get_export_command(command_id)
|
73
|
+
except EduRdmExportDataCommandProgress.DoesNotExist:
|
74
|
+
command = None
|
75
|
+
|
76
|
+
if command:
|
77
|
+
save_command_log_link(command, log_dir)
|
edu_rdm_integration/enums.py
CHANGED
@@ -154,3 +154,15 @@ class EntityRetiredStatusEnum(ValueCodeEnumerate):
|
|
154
154
|
RETIRED: 'Прекращена',
|
155
155
|
ENTERED_IN_ERROR: 'Создана по ошибке',
|
156
156
|
}
|
157
|
+
|
158
|
+
|
159
|
+
class CommandType(BaseEnumerate):
|
160
|
+
"""Тип команды."""
|
161
|
+
|
162
|
+
AUTO = 1
|
163
|
+
MANUAL = 2
|
164
|
+
|
165
|
+
values = {
|
166
|
+
AUTO: 'Автоматический',
|
167
|
+
MANUAL: 'Ручной',
|
168
|
+
}
|
edu_rdm_integration/helpers.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
import heapq
|
2
|
+
import os
|
2
3
|
from concurrent.futures import (
|
3
4
|
ThreadPoolExecutor,
|
4
5
|
)
|
@@ -12,8 +13,12 @@ from typing import (
|
|
12
13
|
List,
|
13
14
|
Optional,
|
14
15
|
Tuple,
|
16
|
+
Union,
|
15
17
|
)
|
16
18
|
|
19
|
+
from django.conf import (
|
20
|
+
settings,
|
21
|
+
)
|
17
22
|
from django.db import (
|
18
23
|
transaction,
|
19
24
|
)
|
@@ -28,6 +33,10 @@ from educommon import (
|
|
28
33
|
logger,
|
29
34
|
)
|
30
35
|
|
36
|
+
from edu_rdm_integration.collect_and_export_data.models import (
|
37
|
+
EduRdmCollectDataCommandProgress,
|
38
|
+
EduRdmExportDataCommandProgress,
|
39
|
+
)
|
31
40
|
from edu_rdm_integration.enums import (
|
32
41
|
FileUploadStatusEnum,
|
33
42
|
)
|
@@ -151,7 +160,7 @@ class Graph:
|
|
151
160
|
"""
|
152
161
|
|
153
162
|
def __init__(self):
|
154
|
-
self.vertices: Dict[str, Dict[str,
|
163
|
+
self.vertices: Dict[str, Dict[str, Optional[str]]] = {}
|
155
164
|
"""Словарь для хранения данных графа."""
|
156
165
|
|
157
166
|
def add_vertex(self, vertex: str):
|
@@ -261,3 +270,14 @@ class Graph:
|
|
261
270
|
path.reverse()
|
262
271
|
|
263
272
|
return path
|
273
|
+
|
274
|
+
|
275
|
+
def save_command_log_link(
|
276
|
+
command: Union[EduRdmCollectDataCommandProgress, EduRdmExportDataCommandProgress],
|
277
|
+
log_dir: str
|
278
|
+
) -> None:
|
279
|
+
"""Сохраняет ссылку на лог команды."""
|
280
|
+
log_file = os.path.join(settings.MEDIA_ROOT, log_dir, f'{command.id}.log')
|
281
|
+
if os.path.exists(log_file):
|
282
|
+
command.logs_link = os.path.join(log_dir, f'{command.id}.log')
|
283
|
+
command.save()
|
edu_rdm_integration/tasks.py
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
from itertools import (
|
2
|
+
chain,
|
3
|
+
)
|
4
|
+
from typing import (
|
5
|
+
TYPE_CHECKING,
|
6
|
+
Dict,
|
7
|
+
Optional,
|
8
|
+
Type,
|
9
|
+
)
|
10
|
+
|
1
11
|
import celery
|
2
12
|
from celery.schedules import (
|
3
13
|
crontab,
|
@@ -5,34 +15,84 @@ from celery.schedules import (
|
|
5
15
|
from django.conf import (
|
6
16
|
settings,
|
7
17
|
)
|
18
|
+
from django.db.models import (
|
19
|
+
CharField,
|
20
|
+
Max,
|
21
|
+
)
|
22
|
+
from django.db.models.functions import (
|
23
|
+
Cast,
|
24
|
+
)
|
25
|
+
from django.utils import (
|
26
|
+
timezone,
|
27
|
+
)
|
8
28
|
|
9
29
|
from educommon.async_task.models import (
|
10
30
|
AsyncTaskType,
|
31
|
+
RunningTask,
|
11
32
|
)
|
12
33
|
from educommon.async_task.tasks import (
|
13
34
|
PeriodicAsyncTask,
|
14
35
|
)
|
36
|
+
from educommon.utils.date import (
|
37
|
+
get_today_min_datetime,
|
38
|
+
)
|
39
|
+
from m3_db_utils.consts import (
|
40
|
+
DEFAULT_ORDER_NUMBER,
|
41
|
+
)
|
15
42
|
|
43
|
+
from edu_rdm_integration.collect_and_export_data.models import (
|
44
|
+
EduRdmCollectDataCommandProgress,
|
45
|
+
EduRdmExportDataCommandProgress,
|
46
|
+
)
|
47
|
+
from edu_rdm_integration.collect_data.collect import (
|
48
|
+
BaseCollectLatestModelsData,
|
49
|
+
)
|
16
50
|
from edu_rdm_integration.collect_data.helpers import (
|
17
51
|
set_failed_status_suspended_collecting_data_stages,
|
18
52
|
)
|
19
53
|
from edu_rdm_integration.consts import (
|
54
|
+
REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
|
55
|
+
REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
|
20
56
|
TASK_QUEUE_NAME,
|
21
57
|
)
|
22
58
|
from edu_rdm_integration.enums import (
|
59
|
+
CommandType,
|
23
60
|
FileUploadStatusEnum,
|
24
61
|
)
|
62
|
+
from edu_rdm_integration.export_data.export import (
|
63
|
+
ExportLatestEntitiesData,
|
64
|
+
)
|
25
65
|
from edu_rdm_integration.export_data.helpers import (
|
26
66
|
set_failed_status_suspended_exporting_data_stages,
|
27
67
|
)
|
28
68
|
from edu_rdm_integration.helpers import (
|
29
69
|
UploadStatusHelper,
|
70
|
+
save_command_log_link,
|
30
71
|
)
|
31
72
|
from edu_rdm_integration.models import (
|
73
|
+
CollectingDataSubStageStatus,
|
74
|
+
CollectingExportedDataSubStage,
|
75
|
+
ExportingDataSubStage,
|
76
|
+
ExportingDataSubStageStatus,
|
32
77
|
ExportingDataSubStageUploaderClientLog,
|
78
|
+
RegionalDataMartEntityEnum,
|
79
|
+
TransferredEntity,
|
80
|
+
)
|
81
|
+
from edu_rdm_integration.storages import (
|
82
|
+
RegionalDataMartEntityStorage,
|
33
83
|
)
|
34
84
|
|
35
85
|
|
86
|
+
if TYPE_CHECKING:
|
87
|
+
from datetime import (
|
88
|
+
datetime,
|
89
|
+
)
|
90
|
+
|
91
|
+
from function_tools.managers import (
|
92
|
+
RunnerManager,
|
93
|
+
)
|
94
|
+
|
95
|
+
|
36
96
|
class RDMCheckUploadStatus(PeriodicAsyncTask):
|
37
97
|
"""Периодическая задача для сбора статусов по загрузке файла в витрину."""
|
38
98
|
|
@@ -95,6 +155,185 @@ class CheckSuspendedExportedStagePeriodicTask(PeriodicAsyncTask):
|
|
95
155
|
)
|
96
156
|
|
97
157
|
|
158
|
+
class TransferLatestEntitiesDataPeriodicTask(PeriodicAsyncTask):
|
159
|
+
"""Периодическая задача сбора и выгрузки данных."""
|
160
|
+
|
161
|
+
queue = TASK_QUEUE_NAME
|
162
|
+
routing_key = TASK_QUEUE_NAME
|
163
|
+
description = 'Периодическая задача сбора и экспорта данных РВД'
|
164
|
+
task_type = AsyncTaskType.UNKNOWN
|
165
|
+
run_every = crontab(
|
166
|
+
minute=settings.RDM_TRANSFER_TASK_MINUTE,
|
167
|
+
hour=settings.RDM_TRANSFER_TASK_HOUR,
|
168
|
+
day_of_week=settings.RDM_TRANSFER_TASK_DAY_OF_WEEK,
|
169
|
+
)
|
170
|
+
|
171
|
+
def __init__(self) -> None:
|
172
|
+
super().__init__()
|
173
|
+
|
174
|
+
self._collecting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
175
|
+
self._collecting_data_manager_to_logs_period_end: Dict[str, datetime] = {}
|
176
|
+
|
177
|
+
self._exporting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
178
|
+
self._exporting_data_manager_to_period_end: Dict[str, datetime] = {}
|
179
|
+
|
180
|
+
self._transferred_entities = []
|
181
|
+
|
182
|
+
def process(self, *args, **kwargs):
|
183
|
+
"""Выполняет задачу."""
|
184
|
+
super().process(*args, **kwargs)
|
185
|
+
|
186
|
+
self._collect_transferred_entities()
|
187
|
+
self._collect_managers()
|
188
|
+
self._calculate_collecting_managers_logs_period_ended_at()
|
189
|
+
self._calculate_exporting_managers_ended_at()
|
190
|
+
|
191
|
+
task_id = RunningTask.objects.filter(
|
192
|
+
pk=self.request.id,
|
193
|
+
).values_list('pk', flat=True).first()
|
194
|
+
|
195
|
+
model_enum_values = RegionalDataMartEntityEnum.get_entities_model_enums(
|
196
|
+
entity_enums=self._transferred_entities,
|
197
|
+
is_sorted=True,
|
198
|
+
)
|
199
|
+
|
200
|
+
filtered_model_enum_values = filter(lambda model: model.order_number != DEFAULT_ORDER_NUMBER, model_enum_values)
|
201
|
+
for model_enum_value in filtered_model_enum_values:
|
202
|
+
self._run_collect_model_data(model_enum_value.key, task_id)
|
203
|
+
|
204
|
+
for entity_enum_value in sorted(self._transferred_entities, key=lambda entity: entity.order_number):
|
205
|
+
self._run_export_entity_data(entity_enum_value.key, task_id)
|
206
|
+
|
207
|
+
def _run_collect_model_data(self, model: str, task_id: str) -> None:
|
208
|
+
"""Запускает сбор данных модели РВД."""
|
209
|
+
command = self._create_collect_command(model, task_id)
|
210
|
+
collect_model_data = self._prepare_collect_model_data_class(command)
|
211
|
+
collect_model_data.collect()
|
212
|
+
|
213
|
+
command.refresh_from_db(fields=['stage_id'])
|
214
|
+
save_command_log_link(command, settings.RDM_COLLECT_LOG_DIR)
|
215
|
+
|
216
|
+
def _run_export_entity_data(self, entity: str, task_id: str) -> None:
|
217
|
+
"""Запускает экспорт данных сущности РВД."""
|
218
|
+
command = self._create_export_command(entity, task_id)
|
219
|
+
if command:
|
220
|
+
export_entity_data = self._prepare_export_entity_data_class(command)
|
221
|
+
export_entity_data.export()
|
222
|
+
|
223
|
+
command.refresh_from_db(fields=['stage_id'])
|
224
|
+
save_command_log_link(command, settings.RDM_EXPORT_LOG_DIR)
|
225
|
+
|
226
|
+
def _collect_transferred_entities(self) -> None:
|
227
|
+
"""Собирает сущности РВД, по которым будет произведен сбор и экспорт данных."""
|
228
|
+
self._transferred_entities = [
|
229
|
+
RegionalDataMartEntityEnum.get_model_enum_value(key=entity)
|
230
|
+
for entity in TransferredEntity.objects.values_list('entity', flat=True)
|
231
|
+
]
|
232
|
+
|
233
|
+
def _collect_managers(self) -> None:
|
234
|
+
"""Собирает менеджеры Функций для сбора и выгрузки данных."""
|
235
|
+
entity_storage = RegionalDataMartEntityStorage()
|
236
|
+
entity_storage.prepare()
|
237
|
+
|
238
|
+
collecting_models_data_managers_map = entity_storage.prepare_entities_manager_map(
|
239
|
+
tags={REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA},
|
240
|
+
)
|
241
|
+
exporting_entities_data_managers_map = entity_storage.prepare_entities_manager_map(
|
242
|
+
tags={REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA},
|
243
|
+
)
|
244
|
+
|
245
|
+
for transferred_entity in self._transferred_entities:
|
246
|
+
for model in chain((transferred_entity.main_model_enum, ), transferred_entity.additional_model_enums):
|
247
|
+
collect_manager_class = collecting_models_data_managers_map.get(model.key)
|
248
|
+
if collect_manager_class:
|
249
|
+
self._collecting_data_managers[model.key] = collect_manager_class
|
250
|
+
|
251
|
+
export_manager_class = exporting_entities_data_managers_map.get(transferred_entity.key)
|
252
|
+
if export_manager_class:
|
253
|
+
self._exporting_data_managers[transferred_entity.key] = export_manager_class
|
254
|
+
|
255
|
+
def _calculate_collecting_managers_logs_period_ended_at(self) -> None:
|
256
|
+
"""Определяет дату последнего успешного этапа сбора у менеджеров Функций сбора."""
|
257
|
+
self._collecting_data_manager_to_logs_period_end = dict(
|
258
|
+
CollectingExportedDataSubStage.objects.annotate(
|
259
|
+
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
260
|
+
max_logs_period_ended_at=Max('stage__logs_period_ended_at'),
|
261
|
+
).filter(
|
262
|
+
status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
263
|
+
manager_id__in=[manager.uuid for manager in self._collecting_data_managers.values()],
|
264
|
+
).values_list('manager_id', 'max_logs_period_ended_at'),
|
265
|
+
)
|
266
|
+
|
267
|
+
def _calculate_exporting_managers_ended_at(self) -> None:
|
268
|
+
"""Определяет дату последнего успешного подэтапа экспорта у менеджеров Функций экспорта."""
|
269
|
+
self._exporting_data_manager_to_period_end = dict(
|
270
|
+
ExportingDataSubStage.objects.annotate(
|
271
|
+
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
272
|
+
max_ended_at=Max('ended_at'),
|
273
|
+
).filter(
|
274
|
+
status_id=ExportingDataSubStageStatus.FINISHED.key,
|
275
|
+
manager_id__in=[manager.uuid for manager in self._exporting_data_managers.values()],
|
276
|
+
).values_list('manager_id', 'max_ended_at'),
|
277
|
+
)
|
278
|
+
|
279
|
+
def _create_collect_command(self, model: str, task_id: str) -> EduRdmCollectDataCommandProgress:
|
280
|
+
"""Создает команду сбора данных моделей РВД."""
|
281
|
+
manager = self._collecting_data_managers[model]
|
282
|
+
logs_period_started_at = (
|
283
|
+
self._collecting_data_manager_to_logs_period_end.get(manager.uuid)
|
284
|
+
or get_today_min_datetime()
|
285
|
+
)
|
286
|
+
|
287
|
+
return EduRdmCollectDataCommandProgress.objects.create(
|
288
|
+
model_id=model,
|
289
|
+
logs_period_started_at=logs_period_started_at,
|
290
|
+
logs_period_ended_at=timezone.now(),
|
291
|
+
task_id=task_id,
|
292
|
+
type=CommandType.AUTO,
|
293
|
+
)
|
294
|
+
|
295
|
+
def _create_export_command(self, entity: str, task_id: str) -> Optional[EduRdmExportDataCommandProgress]:
|
296
|
+
"""Создает команду экспорта данных сущностей РВД."""
|
297
|
+
manager = self._exporting_data_managers[entity]
|
298
|
+
period_started_at = self._exporting_data_manager_to_period_end.get(manager.uuid)
|
299
|
+
|
300
|
+
if period_started_at:
|
301
|
+
return EduRdmExportDataCommandProgress.objects.create(
|
302
|
+
entity_id=entity,
|
303
|
+
period_started_at=period_started_at,
|
304
|
+
period_ended_at=timezone.now(),
|
305
|
+
task_id=task_id,
|
306
|
+
type=CommandType.AUTO,
|
307
|
+
)
|
308
|
+
|
309
|
+
def _prepare_collect_model_data_class(
|
310
|
+
self,
|
311
|
+
command: EduRdmCollectDataCommandProgress
|
312
|
+
) -> BaseCollectLatestModelsData:
|
313
|
+
"""Подготавливает объект класса сбора данных моделей РВД."""
|
314
|
+
return BaseCollectLatestModelsData(
|
315
|
+
models=[command.model_id],
|
316
|
+
logs_period_started_at=command.logs_period_started_at,
|
317
|
+
logs_period_ended_at=command.logs_period_ended_at,
|
318
|
+
command_id=command.id,
|
319
|
+
)
|
320
|
+
|
321
|
+
def _prepare_export_entity_data_class(self, command: EduRdmExportDataCommandProgress) -> ExportLatestEntitiesData:
|
322
|
+
"""Подготавливает объект класса экспорта данных сущностей РВД.
|
323
|
+
|
324
|
+
При экспорте данных передаем параметр task_id для обновления поля "Описание"
|
325
|
+
наименованиями выгруженных сущностей.
|
326
|
+
"""
|
327
|
+
return ExportLatestEntitiesData(
|
328
|
+
entities=[command.entity_id],
|
329
|
+
period_started_at=command.period_started_at,
|
330
|
+
period_ended_at=command.period_ended_at,
|
331
|
+
command_id=command.id,
|
332
|
+
task_id=self.request.id,
|
333
|
+
)
|
334
|
+
|
335
|
+
|
98
336
|
celery_app = celery.app.app_or_default()
|
99
337
|
celery_app.register_task(RDMCheckUploadStatus)
|
100
338
|
celery_app.register_task(CheckSuspendedExportedStagePeriodicTask)
|
339
|
+
celery_app.register_task(TransferLatestEntitiesDataPeriodicTask)
|
edu_rdm_integration/utils.py
CHANGED
@@ -17,7 +17,6 @@ from typing import (
|
|
17
17
|
from django.conf import (
|
18
18
|
settings,
|
19
19
|
)
|
20
|
-
|
21
20
|
from edu_rdm_integration.apps import (
|
22
21
|
EduRDMIntegrationConfig,
|
23
22
|
)
|
@@ -25,6 +24,7 @@ from edu_rdm_integration.consts import (
|
|
25
24
|
ACADEMIC_YEAR,
|
26
25
|
)
|
27
26
|
|
27
|
+
|
28
28
|
if TYPE_CHECKING:
|
29
29
|
from typing import (
|
30
30
|
NamedTuple,
|
@@ -34,6 +34,10 @@ if TYPE_CHECKING:
|
|
34
34
|
Model,
|
35
35
|
)
|
36
36
|
|
37
|
+
from edu_rdm_integration.collect_and_export_data.models import (
|
38
|
+
EduRdmCollectDataCommandProgress,
|
39
|
+
EduRdmExportDataCommandProgress,
|
40
|
+
)
|
37
41
|
from edu_rdm_integration.helpers import (
|
38
42
|
Graph,
|
39
43
|
)
|
@@ -256,3 +260,27 @@ def build_related_model_graph(
|
|
256
260
|
graph.add_edge(current_model._meta.model_name, related_model._meta.model_name, field.name)
|
257
261
|
|
258
262
|
return graph
|
263
|
+
|
264
|
+
|
265
|
+
def get_data_command_progress_attachment_path(
|
266
|
+
instance: Union['EduRdmCollectDataCommandProgress', 'EduRdmExportDataCommandProgress'],
|
267
|
+
filename: str
|
268
|
+
) -> str:
|
269
|
+
"""Возвращает путь до файла-вложения при формировании команды сборки или выгрузки данных.
|
270
|
+
|
271
|
+
Args:
|
272
|
+
instance: объект ExportDataCommandProgress или объект CollectDataCommandProgres
|
273
|
+
filename: имя загружаемого файла
|
274
|
+
|
275
|
+
Returns:
|
276
|
+
Строковое представление пути
|
277
|
+
"""
|
278
|
+
date_now = datetime.now()
|
279
|
+
|
280
|
+
return os.path.join(
|
281
|
+
settings.UPLOADS,
|
282
|
+
date_now.strftime('%Y/%m/%d'),
|
283
|
+
instance.__class__.__name__.lower(),
|
284
|
+
str(instance.type),
|
285
|
+
filename
|
286
|
+
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: edu-rdm-integration
|
3
|
-
Version: 2.1
|
3
|
+
Version: 2.2.1
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
5
5
|
Home-page:
|
6
6
|
Download-URL:
|
@@ -293,6 +293,27 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
293
293
|
|
294
294
|
### Удалено
|
295
295
|
|
296
|
+
## [2.2.1] - 2024-03-28
|
297
|
+
Исправление наследования Meta в моделях EduRdmCollectDataCommandProgress и EduRdmExportDataCommandProgress
|
298
|
+
|
299
|
+
### Исправлено
|
300
|
+
|
301
|
+
- [EDUSCHL-21569](https://jira.bars.group/browse/EDUSCHL-21569)
|
302
|
+
MINOR Перенести оставшиеся общие асинхронные задачи из ЭШ в пакет edu_rdm_integration
|
303
|
+
|
304
|
+
## [2.2.0] - 2024-03-20
|
305
|
+
|
306
|
+
Из ЭШ перенесена периодическая задача по сбору и выгрузке данных в РВД.
|
307
|
+
|
308
|
+
Также из ЭШ перенесены и переименованы модели:
|
309
|
+
- CollectDataCommandProgress перименована в EduRdmCollectDataCommandProgress
|
310
|
+
- ExportDataCommandProgress перименована в EduRdmExportDataCommandProgress
|
311
|
+
- Добавлены миксины CollectCommandMixin и ExportCommandMixin
|
312
|
+
|
313
|
+
### Добавлено
|
314
|
+
|
315
|
+
- [EDUSCHL-21569](https://jira.bars.group/browse/EDUSCHL-21569)
|
316
|
+
MINOR Перенести оставшиеся общие асинхронные задачи из ЭШ в пакет edu_rdm_integration
|
296
317
|
|
297
318
|
## [2.1.0] - 2024-03-18
|
298
319
|
|
@@ -5,14 +5,14 @@ edu_rdm_integration/apps.py,sha256=5OgNdmuqe26fbu4wYb69haQJe-XFO_rDbnU1vPqJU-U,3
|
|
5
5
|
edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
|
6
6
|
edu_rdm_integration/consts.py,sha256=FFwcMHNsfjP_s9LfkccLAHjJMEMp7ppPmrRlJcgV88k,1104
|
7
7
|
edu_rdm_integration/entities.py,sha256=mAjsYlcIbemo4xT5CSCr4payZubiBHB7Rb3Ow1CVsy0,14552
|
8
|
-
edu_rdm_integration/enums.py,sha256=
|
9
|
-
edu_rdm_integration/helpers.py,sha256=
|
8
|
+
edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
|
9
|
+
edu_rdm_integration/helpers.py,sha256=wsVd3GiO7qgb_jtZBqEuedBTW8Q9-j2xcfMBi45O0cw,10997
|
10
10
|
edu_rdm_integration/mapping.py,sha256=bwa2fJCbV4YjQcAgRrgT3hgM6dJhr_uBtQgx3L3F2Ck,473
|
11
11
|
edu_rdm_integration/models.py,sha256=Wg8NYj4C6Xqw-15AsQGU0TWNzUjdwBW_YZDzmPqx3GY,29327
|
12
12
|
edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
|
13
13
|
edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
|
14
|
-
edu_rdm_integration/tasks.py,sha256=
|
15
|
-
edu_rdm_integration/utils.py,sha256
|
14
|
+
edu_rdm_integration/tasks.py,sha256=8bQZbhVDuK3RYc8lSbWyW1xDyVgIZQCrcsTo8pUeIeE,13716
|
15
|
+
edu_rdm_integration/utils.py,sha256=-my8q9fude6Nc9r_qUDww-8QaU2tcv-xOexewYqOPJw,10369
|
16
16
|
edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
|
17
17
|
edu_rdm_integration/adapters/apps.py,sha256=TyJTkSPs2qAHJ11fqbwLGk3Ea7ujtqWwbxqmvYNQxG8,363
|
18
18
|
edu_rdm_integration/adapters/caches.py,sha256=OxSqeXySUN42LxEeHBLtC1ZBt-7aicbRbmP1EJYTvV4,1505
|
@@ -30,6 +30,12 @@ edu_rdm_integration/adapters/strategies.py,sha256=ztEHvdMYjJzjjkV7J9XtuNhn_uUVoJ
|
|
30
30
|
edu_rdm_integration/adapters/strings.py,sha256=-k9dex8A7hCpkzUkudVkKRAbNRuuqog2hYl2xmibl8I,181
|
31
31
|
edu_rdm_integration/adapters/tests.py,sha256=MoRY-a75Ow-7EjeQYxkXWunwqTGuBMaUyEkEV2oy05I,59
|
32
32
|
edu_rdm_integration/adapters/validators.py,sha256=LJWnCY8PtXDOj-fm3fBWjQYsHsSLfyKf_D97pqPv73s,496
|
33
|
+
edu_rdm_integration/collect_and_export_data/__init__.py,sha256=4glPgPCzAyLreBGUnUrcRPCge45XucJz5bK8VjlQBaE,82
|
34
|
+
edu_rdm_integration/collect_and_export_data/apps.py,sha256=fAcctcjxWP4Gd0Qr3YrQkMESrzga4IORWetBTy8wvHo,160
|
35
|
+
edu_rdm_integration/collect_and_export_data/models.py,sha256=TGm-hM-1aDhVcFPy0PTC3yNczoZGF4ZmIdr0Y9s8CfU,2038
|
36
|
+
edu_rdm_integration/collect_and_export_data/utils.py,sha256=uw85W_uPr7zGa9rA4Bp1nDUrOMnlz0nZ92AZAgNsRlE,3073
|
37
|
+
edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py,sha256=UkoaXzh3tokZ8QdCdB09v3rRZfcHhvEwNMuj3mQIB74,4714
|
38
|
+
edu_rdm_integration/collect_and_export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
39
|
edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
40
|
edu_rdm_integration/collect_data/collect.py,sha256=TfjDD25opR6UlV6oWbMXs3T7nQguSg5IT0k2oFasWJI,14543
|
35
41
|
edu_rdm_integration/collect_data/generators.py,sha256=f34AAwdEcQNIokX0ypqYgjRD1XolwBVLER_HYv9ibNw,9075
|
@@ -156,9 +162,9 @@ edu_rdm_integration/uploader_log/apps.py,sha256=tYJj4-sDlq8fLOSvw18L_yys7SILpTKW
|
|
156
162
|
edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJLdsGmr3SXwH_U,266
|
157
163
|
edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
|
158
164
|
edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
|
159
|
-
edu_rdm_integration-2.1.
|
160
|
-
edu_rdm_integration-2.1.
|
161
|
-
edu_rdm_integration-2.1.
|
162
|
-
edu_rdm_integration-2.1.
|
163
|
-
edu_rdm_integration-2.1.
|
164
|
-
edu_rdm_integration-2.1.
|
165
|
+
edu_rdm_integration-2.2.1.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
166
|
+
edu_rdm_integration-2.2.1.dist-info/METADATA,sha256=0K4JnLA9jXs0MxuvyJlFL0IaolV9Oil6ppFcTemQhxY,58264
|
167
|
+
edu_rdm_integration-2.2.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
168
|
+
edu_rdm_integration-2.2.1.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
169
|
+
edu_rdm_integration-2.2.1.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
170
|
+
edu_rdm_integration-2.2.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
{edu_rdm_integration-2.1.0.dist-info → edu_rdm_integration-2.2.1.dist-info}/namespace_packages.txt
RENAMED
File without changes
|
File without changes
|