edu-rdm-integration 3.2.0__py3-none-any.whl → 3.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/app_settings.py +0 -1
- edu_rdm_integration/collect_data/collect.py +18 -43
- edu_rdm_integration/export_data/base/functions.py +1 -1
- edu_rdm_integration/export_data/export.py +24 -35
- edu_rdm_integration/helpers.py +75 -4
- edu_rdm_integration/tasks.py +26 -30
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/METADATA +30 -3
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/RECORD +12 -12
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/LICENSE +0 -0
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/namespace_packages.txt +0 -0
- {edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/top_level.txt +0 -0
@@ -37,7 +37,6 @@ RDM_TRANSFER_TASK_MINUTE = '0'
|
|
37
37
|
RDM_TRANSFER_TASK_HOUR = '*/4'
|
38
38
|
RDM_TRANSFER_TASK_DAY_OF_WEEK = '*'
|
39
39
|
RDM_TRANSFER_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 6
|
40
|
-
RDM_TRANSFER_TASK_TIMEDELTA = 3600
|
41
40
|
|
42
41
|
# Настройка запуска периодической задачи статуса загрузки данных в витрину:
|
43
42
|
RDM_UPLOAD_STATUS_TASK_MINUTE = '*/30'
|
@@ -18,14 +18,6 @@ from typing import (
|
|
18
18
|
from django.conf import (
|
19
19
|
settings,
|
20
20
|
)
|
21
|
-
from django.db.models import (
|
22
|
-
F,
|
23
|
-
Max,
|
24
|
-
CharField,
|
25
|
-
)
|
26
|
-
from django.db.models.functions import (
|
27
|
-
Cast,
|
28
|
-
)
|
29
21
|
from django.utils import (
|
30
22
|
timezone,
|
31
23
|
)
|
@@ -33,9 +25,6 @@ from django.utils import (
|
|
33
25
|
from educommon import (
|
34
26
|
logger,
|
35
27
|
)
|
36
|
-
from educommon.audit_log.models import (
|
37
|
-
AuditLog,
|
38
|
-
)
|
39
28
|
from educommon.utils.date import (
|
40
29
|
get_today_min_datetime,
|
41
30
|
)
|
@@ -56,11 +45,12 @@ from edu_rdm_integration.consts import (
|
|
56
45
|
LOGS_SUB_PERIOD_DAYS,
|
57
46
|
REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
|
58
47
|
)
|
48
|
+
from edu_rdm_integration.helpers import (
|
49
|
+
get_collecting_managers_max_period_ended_dates,
|
50
|
+
)
|
59
51
|
from edu_rdm_integration.models import (
|
60
52
|
CollectingDataStageStatus,
|
61
|
-
CollectingDataSubStageStatus,
|
62
53
|
CollectingExportedDataStage,
|
63
|
-
CollectingExportedDataSubStage,
|
64
54
|
RegionalDataMartModelEnum,
|
65
55
|
)
|
66
56
|
from edu_rdm_integration.signals import (
|
@@ -72,6 +62,10 @@ from edu_rdm_integration.storages import (
|
|
72
62
|
|
73
63
|
|
74
64
|
if TYPE_CHECKING:
|
65
|
+
from educommon.audit_log.models import (
|
66
|
+
AuditLog,
|
67
|
+
)
|
68
|
+
|
75
69
|
from edu_rdm_integration.collect_data.generators import (
|
76
70
|
BaseEduLogGenerator,
|
77
71
|
)
|
@@ -141,7 +135,7 @@ class BaseCollectModelsData(BaseOperationData):
|
|
141
135
|
|
142
136
|
logger.info('collecting models data managers finished')
|
143
137
|
|
144
|
-
def _collect_models_data(self, *args, logs: Optional[Dict[str, List[AuditLog]]] = None, **kwargs):
|
138
|
+
def _collect_models_data(self, *args, logs: Optional[Dict[str, List['AuditLog']]] = None, **kwargs):
|
145
139
|
"""
|
146
140
|
Запуск Функций по формированию данных моделей РВД из логов.
|
147
141
|
"""
|
@@ -227,7 +221,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
227
221
|
if temp_logs_period_ended_at > self.logs_period_ended_at:
|
228
222
|
temp_logs_period_ended_at = self.logs_period_ended_at
|
229
223
|
|
230
|
-
temp_logs: Dict[str, List[AuditLog]] = {}
|
224
|
+
temp_logs: Dict[str, List['AuditLog']] = {}
|
231
225
|
|
232
226
|
while temp_logs_period_started_at < temp_logs_period_ended_at <= self.logs_period_ended_at:
|
233
227
|
for model in self.models:
|
@@ -254,7 +248,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
254
248
|
"""
|
255
249
|
Генерация логов за весь период.
|
256
250
|
"""
|
257
|
-
temp_logs: Dict[str, List[AuditLog]] = {}
|
251
|
+
temp_logs: Dict[str, List['AuditLog']] = {}
|
258
252
|
|
259
253
|
for model in self.models:
|
260
254
|
logs = self.log_generator.generate(
|
@@ -268,7 +262,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
268
262
|
|
269
263
|
return [(temp_logs, self.logs_period_started_at, self.logs_period_ended_at)]
|
270
264
|
|
271
|
-
def _generate_logs(self) -> List[Tuple[Dict[str, List[AuditLog]], datetime, datetime]]:
|
265
|
+
def _generate_logs(self) -> List[Tuple[Dict[str, List['AuditLog']], datetime, datetime]]:
|
272
266
|
"""
|
273
267
|
Генерация логов.
|
274
268
|
|
@@ -318,47 +312,28 @@ class BaseCollectLatestModelsData(BaseCollectModelsData):
|
|
318
312
|
|
319
313
|
return stage_created_or_in_progress
|
320
314
|
|
321
|
-
def
|
322
|
-
"""
|
323
|
-
Возвращает словарь с uuid менеджера и крайней датой из переданного поля по указанным моделям.
|
324
|
-
"""
|
325
|
-
manager_to_start_date = dict(
|
326
|
-
CollectingExportedDataSubStage.objects.annotate(
|
327
|
-
max_date=Max(f'stage__{date_field}'),
|
328
|
-
).values(
|
329
|
-
'stage__manager_id', 'max_date',
|
330
|
-
).annotate(
|
331
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
332
|
-
date=F('max_date'),
|
333
|
-
).filter(
|
334
|
-
status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
335
|
-
manager_id__in=(m.uuid for m in
|
336
|
-
self._collecting_data_managers.values()),
|
337
|
-
).values_list('manager_id', 'date')
|
338
|
-
)
|
339
|
-
|
340
|
-
return manager_to_start_date
|
341
|
-
|
342
|
-
def _collect_models_data(self, *args, logs: Optional[Dict[str, List[AuditLog]]] = None, **kwargs) -> None:
|
315
|
+
def _collect_models_data(self, *args, logs: Optional[Dict[str, List['AuditLog']]] = None, **kwargs) -> None:
|
343
316
|
"""
|
344
317
|
Запуск Функций по формированию данных из логов для дальнейшей выгрузки.
|
345
318
|
"""
|
346
319
|
logger.info('collect models data..')
|
347
320
|
|
348
|
-
|
349
|
-
|
321
|
+
managers_max_period_ended_dates = get_collecting_managers_max_period_ended_dates(
|
322
|
+
self._collecting_data_managers.values()
|
323
|
+
)
|
350
324
|
|
351
325
|
for model_key, manager_class in self._collecting_data_managers.items():
|
352
326
|
model_logs = logs.get(model_key) if logs else None
|
327
|
+
manager_max_dates = managers_max_period_ended_dates.get(manager_class.uuid, {})
|
353
328
|
|
354
329
|
kwargs['logs_period_started_at'] = (
|
355
|
-
|
330
|
+
manager_max_dates.get('max_period_ended_at')
|
356
331
|
or get_today_min_datetime()
|
357
332
|
)
|
358
333
|
kwargs['logs_period_ended_at'] = timezone.now()
|
359
334
|
|
360
335
|
if kwargs['logs_period_started_at'] > kwargs['logs_period_ended_at']:
|
361
|
-
kwargs['logs_period_started_at'] =
|
336
|
+
kwargs['logs_period_started_at'] = manager_max_dates['max_started_at']
|
362
337
|
|
363
338
|
manager = manager_class(*args, logs=model_logs, **kwargs)
|
364
339
|
|
@@ -9,6 +9,7 @@ from datetime import (
|
|
9
9
|
timedelta,
|
10
10
|
)
|
11
11
|
from typing import (
|
12
|
+
TYPE_CHECKING,
|
12
13
|
Dict,
|
13
14
|
Iterable,
|
14
15
|
List,
|
@@ -21,7 +22,6 @@ from django.conf import (
|
|
21
22
|
)
|
22
23
|
from django.db.models import (
|
23
24
|
F,
|
24
|
-
Max,
|
25
25
|
Model,
|
26
26
|
Value,
|
27
27
|
)
|
@@ -53,9 +53,6 @@ from educommon.utils.date import (
|
|
53
53
|
from educommon.utils.seqtools import (
|
54
54
|
make_chunks,
|
55
55
|
)
|
56
|
-
from function_tools.managers import (
|
57
|
-
RunnerManager,
|
58
|
-
)
|
59
56
|
from m3_db_utils.consts import (
|
60
57
|
DEFAULT_ORDER_NUMBER,
|
61
58
|
)
|
@@ -72,9 +69,11 @@ from edu_rdm_integration.consts import (
|
|
72
69
|
from edu_rdm_integration.enums import (
|
73
70
|
FileUploadStatusEnum,
|
74
71
|
)
|
72
|
+
from edu_rdm_integration.helpers import (
|
73
|
+
get_exporting_managers_max_period_ended_dates,
|
74
|
+
)
|
75
75
|
from edu_rdm_integration.models import (
|
76
76
|
ExportingDataStage,
|
77
|
-
ExportingDataSubStage,
|
78
77
|
ExportingDataSubStageStatus,
|
79
78
|
RegionalDataMartEntityEnum,
|
80
79
|
)
|
@@ -86,6 +85,12 @@ from edu_rdm_integration.storages import (
|
|
86
85
|
)
|
87
86
|
|
88
87
|
|
88
|
+
if TYPE_CHECKING:
|
89
|
+
from function_tools.managers import (
|
90
|
+
RunnerManager,
|
91
|
+
)
|
92
|
+
|
93
|
+
|
89
94
|
class BaseExportEntitiesData(BaseOperationData):
|
90
95
|
"""Базовый класс экспорта сущностей РВД за указанных период."""
|
91
96
|
|
@@ -108,7 +113,7 @@ class BaseExportEntitiesData(BaseOperationData):
|
|
108
113
|
self.period_ended_at = period_ended_at
|
109
114
|
|
110
115
|
# Классы менеджеров Функций, которые должны быть запущены для выгрузки данных
|
111
|
-
self._exporting_data_managers: Set[Type[RunnerManager]] = set()
|
116
|
+
self._exporting_data_managers: Set[Type['RunnerManager']] = set()
|
112
117
|
|
113
118
|
# Результаты работы Функций выгрузки данных
|
114
119
|
self._exporting_data_results = []
|
@@ -277,7 +282,7 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
277
282
|
):
|
278
283
|
super().__init__(entities, period_started_at, period_ended_at, **kwargs)
|
279
284
|
|
280
|
-
self._exporting_data_managers: Set[Type[RunnerManager]] = OrderedSet()
|
285
|
+
self._exporting_data_managers: Set[Type['RunnerManager']] = OrderedSet()
|
281
286
|
|
282
287
|
# Словарь данных с классами менеджеров и их сущностями
|
283
288
|
self._manager_entities_map: Dict[Type[object], List[str]] = defaultdict(set)
|
@@ -353,20 +358,6 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
353
358
|
):
|
354
359
|
queryset.filter(id__in=model_ids).update(modified=now)
|
355
360
|
|
356
|
-
def _calculate_last_finished_entity_export(self) -> Dict[str, datetime]:
|
357
|
-
"""Находит левую границу экспорта данных для сущностей."""
|
358
|
-
manager_to_last_date = ExportingDataSubStage.objects.values(
|
359
|
-
'stage__manager_id',
|
360
|
-
).annotate(
|
361
|
-
max_ended_at=Max('ended_at'),
|
362
|
-
entity_uuid=F('stage__manager_id'),
|
363
|
-
).filter(
|
364
|
-
status_id=ExportingDataSubStageStatus.FINISHED.key,
|
365
|
-
entity_uuid__in=[m.uuid for m in self._exporting_data_managers],
|
366
|
-
).values('entity_uuid', 'max_ended_at')
|
367
|
-
|
368
|
-
return {str(m['entity_uuid']): m['max_ended_at'] for m in manager_to_last_date}
|
369
|
-
|
370
361
|
def _export_entities_data(self, *args, **kwargs) -> None:
|
371
362
|
"""
|
372
363
|
Запуск Функций по для экспорта данных.
|
@@ -376,26 +367,24 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
376
367
|
# Массив с выгружаемыми сущностями для поля "Описание" в асинхронной задаче
|
377
368
|
exported_entities = []
|
378
369
|
|
379
|
-
|
370
|
+
managers_max_period_ended_dates = get_exporting_managers_max_period_ended_dates(
|
371
|
+
self._exporting_data_managers
|
372
|
+
)
|
380
373
|
|
381
374
|
for manager_class in self._exporting_data_managers:
|
382
|
-
|
383
|
-
manager_class.uuid,
|
384
|
-
timezone.now(),
|
385
|
-
)
|
386
|
-
# Обновить поля modified у модели сущности:
|
387
|
-
self._update_model_modified_field(
|
388
|
-
manager_id=manager_class.uuid,
|
389
|
-
last_finished_export_data=last_finished_export_data,
|
390
|
-
)
|
375
|
+
manager_max_dates = managers_max_period_ended_dates.get(manager_class.uuid, {})
|
391
376
|
|
392
|
-
kwargs['period_started_at'] =
|
377
|
+
kwargs['period_started_at'] = manager_max_dates.get('max_period_ended_at') or timezone.now()
|
393
378
|
kwargs['period_ended_at'] = get_today_max_datetime()
|
394
379
|
|
395
380
|
if kwargs['period_started_at'] > kwargs['period_ended_at']:
|
396
|
-
kwargs['period_started_at'] =
|
397
|
-
|
398
|
-
|
381
|
+
kwargs['period_started_at'] = manager_max_dates['max_started_at']
|
382
|
+
|
383
|
+
# Обновить поля modified у модели сущности:
|
384
|
+
self._update_model_modified_field(
|
385
|
+
manager_id=manager_class.uuid,
|
386
|
+
last_finished_export_data=kwargs['period_started_at'],
|
387
|
+
)
|
399
388
|
|
400
389
|
manager = manager_class(*args, **kwargs)
|
401
390
|
|
edu_rdm_integration/helpers.py
CHANGED
@@ -11,6 +11,7 @@ from typing import (
|
|
11
11
|
Any,
|
12
12
|
Dict,
|
13
13
|
List,
|
14
|
+
Iterable,
|
14
15
|
Optional,
|
15
16
|
Tuple,
|
16
17
|
Union,
|
@@ -23,15 +24,20 @@ from django.db import (
|
|
23
24
|
transaction,
|
24
25
|
)
|
25
26
|
from django.db.models import (
|
27
|
+
CharField,
|
28
|
+
Max,
|
26
29
|
QuerySet,
|
27
30
|
)
|
28
|
-
from
|
29
|
-
|
31
|
+
from django.db.models.functions import (
|
32
|
+
Cast,
|
30
33
|
)
|
31
34
|
|
32
35
|
from educommon import (
|
33
36
|
logger,
|
34
37
|
)
|
38
|
+
from uploader_client.adapters import (
|
39
|
+
adapter,
|
40
|
+
)
|
35
41
|
|
36
42
|
from edu_rdm_integration.collect_and_export_data.models import (
|
37
43
|
EduRdmCollectDataCommandProgress,
|
@@ -45,12 +51,25 @@ from edu_rdm_integration.export_data.base.requests import (
|
|
45
51
|
)
|
46
52
|
from edu_rdm_integration.models import (
|
47
53
|
DataMartRequestStatus,
|
54
|
+
ExportingDataStage,
|
55
|
+
CollectingDataStageStatus,
|
56
|
+
CollectingExportedDataStage,
|
48
57
|
ExportingDataSubStageUploaderClientLog,
|
49
58
|
UploadStatusRequestLog,
|
59
|
+
ExportingDataStageStatus,
|
50
60
|
)
|
51
61
|
|
52
|
-
|
53
62
|
if TYPE_CHECKING:
|
63
|
+
from datetime import (
|
64
|
+
datetime,
|
65
|
+
)
|
66
|
+
|
67
|
+
from edu_rdm_integration.collect_data.non_calculated.base.managers import (
|
68
|
+
BaseCollectingExportedDataRunnerManager,
|
69
|
+
)
|
70
|
+
from edu_rdm_integration.export_data.base.managers import (
|
71
|
+
BaseExportDataRunnerManager,
|
72
|
+
)
|
54
73
|
from uploader_client.models import (
|
55
74
|
Entry,
|
56
75
|
)
|
@@ -88,7 +107,7 @@ class UploadStatusHelper:
|
|
88
107
|
response = None
|
89
108
|
|
90
109
|
if result.error:
|
91
|
-
logger.
|
110
|
+
logger.warning(
|
92
111
|
f'Ошибка при получении статуса загрузки данных в витрину. Идентификатор загрузки: {request_id}. '
|
93
112
|
f'Ошибка: {result.error}, запрос: {result.log.request}, ответ: {result.log.response}',
|
94
113
|
)
|
@@ -281,3 +300,55 @@ def save_command_log_link(
|
|
281
300
|
if os.path.exists(log_file):
|
282
301
|
command.logs_link = os.path.join(log_dir, f'{command.id}.log')
|
283
302
|
command.save()
|
303
|
+
|
304
|
+
|
305
|
+
def get_collecting_managers_max_period_ended_dates(
|
306
|
+
collecting_managers: Iterable['BaseCollectingExportedDataRunnerManager'],
|
307
|
+
) -> Dict[str, Dict[str, 'datetime']]:
|
308
|
+
"""Возвращает даты и времени последнего успешного этапа сбора для менеджеров Функций сбора."""
|
309
|
+
managers_max_period_ended_dates = CollectingExportedDataStage.objects.filter(
|
310
|
+
status_id=CollectingDataStageStatus.FINISHED.key,
|
311
|
+
manager_id__in=[manager.uuid for manager in collecting_managers],
|
312
|
+
).values(
|
313
|
+
'manager_id',
|
314
|
+
).annotate(
|
315
|
+
str_manager_id=Cast('manager_id', output_field=CharField()),
|
316
|
+
max_period_ended_at=Max('logs_period_ended_at'),
|
317
|
+
max_started_at=Max('started_at'),
|
318
|
+
).values_list(
|
319
|
+
'str_manager_id', 'max_period_ended_at', 'max_started_at',
|
320
|
+
)
|
321
|
+
|
322
|
+
return {
|
323
|
+
manager_id: {
|
324
|
+
'max_period_ended_at': max_period_ended_at,
|
325
|
+
'max_started_at': max_started_at,
|
326
|
+
}
|
327
|
+
for manager_id, max_period_ended_at, max_started_at in managers_max_period_ended_dates
|
328
|
+
}
|
329
|
+
|
330
|
+
|
331
|
+
def get_exporting_managers_max_period_ended_dates(
|
332
|
+
exporting_managers: Iterable['BaseExportDataRunnerManager'],
|
333
|
+
) -> Dict[str, Dict[str, 'datetime']]:
|
334
|
+
"""Возвращает даты и времени последнего успешного этапа экспорта для менеджеров Функций экспорта."""
|
335
|
+
managers_max_period_ended_dates = ExportingDataStage.objects.filter(
|
336
|
+
status_id=ExportingDataStageStatus.FINISHED.key,
|
337
|
+
manager_id__in=[manager.uuid for manager in exporting_managers],
|
338
|
+
).values(
|
339
|
+
'manager_id',
|
340
|
+
).annotate(
|
341
|
+
str_manager_id=Cast('manager_id', output_field=CharField()),
|
342
|
+
max_period_ended_at=Max('period_ended_at'),
|
343
|
+
max_started_at=Max('started_at'),
|
344
|
+
).values_list(
|
345
|
+
'str_manager_id', 'max_period_ended_at', 'max_started_at',
|
346
|
+
)
|
347
|
+
|
348
|
+
return {
|
349
|
+
manager_id: {
|
350
|
+
'max_period_ended_at': max_period_ended_at,
|
351
|
+
'max_started_at': max_started_at,
|
352
|
+
}
|
353
|
+
for manager_id, max_period_ended_at, max_started_at in managers_max_period_ended_dates
|
354
|
+
}
|
edu_rdm_integration/tasks.py
CHANGED
@@ -28,7 +28,6 @@ from django.db.models.functions import (
|
|
28
28
|
from django.utils import (
|
29
29
|
timezone,
|
30
30
|
)
|
31
|
-
|
32
31
|
from educommon.async_task.locker import (
|
33
32
|
TaskLocker,
|
34
33
|
)
|
@@ -72,14 +71,12 @@ from edu_rdm_integration.export_data.helpers import (
|
|
72
71
|
set_failed_status_suspended_exporting_data_stages,
|
73
72
|
)
|
74
73
|
from edu_rdm_integration.helpers import (
|
74
|
+
get_collecting_managers_max_period_ended_dates,
|
75
|
+
get_exporting_managers_max_period_ended_dates,
|
75
76
|
UploadStatusHelper,
|
76
77
|
save_command_log_link,
|
77
78
|
)
|
78
79
|
from edu_rdm_integration.models import (
|
79
|
-
CollectingDataSubStageStatus,
|
80
|
-
CollectingExportedDataSubStage,
|
81
|
-
ExportingDataSubStage,
|
82
|
-
ExportingDataSubStageStatus,
|
83
80
|
ExportingDataSubStageUploaderClientLog,
|
84
81
|
RegionalDataMartEntityEnum,
|
85
82
|
TransferredEntity,
|
@@ -88,7 +85,6 @@ from edu_rdm_integration.storages import (
|
|
88
85
|
RegionalDataMartEntityStorage,
|
89
86
|
)
|
90
87
|
|
91
|
-
|
92
88
|
if TYPE_CHECKING:
|
93
89
|
from datetime import (
|
94
90
|
datetime,
|
@@ -214,10 +210,10 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
214
210
|
super().__init__()
|
215
211
|
|
216
212
|
self._collecting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
217
|
-
self._collecting_data_manager_to_logs_period_end: Dict[str, datetime] = {}
|
213
|
+
self._collecting_data_manager_to_logs_period_end: Dict[str, 'datetime'] = {}
|
218
214
|
|
219
215
|
self._exporting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
220
|
-
self._exporting_data_manager_to_period_end: Dict[str, datetime] = {}
|
216
|
+
self._exporting_data_manager_to_period_end: Dict[str, 'datetime'] = {}
|
221
217
|
|
222
218
|
self._transferred_entities = []
|
223
219
|
|
@@ -296,40 +292,34 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
296
292
|
|
297
293
|
def _calculate_collecting_managers_logs_period_ended_at(self) -> None:
|
298
294
|
"""Определяет дату последнего успешного этапа сбора у менеджеров Функций сбора."""
|
299
|
-
self._collecting_data_manager_to_logs_period_end =
|
300
|
-
|
301
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
302
|
-
max_logs_period_ended_at=Max('stage__logs_period_ended_at'),
|
303
|
-
).filter(
|
304
|
-
status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
305
|
-
manager_id__in=[manager.uuid for manager in self._collecting_data_managers.values()],
|
306
|
-
).values_list('manager_id', 'max_logs_period_ended_at'),
|
295
|
+
self._collecting_data_manager_to_logs_period_end = get_collecting_managers_max_period_ended_dates(
|
296
|
+
self._collecting_data_managers.values()
|
307
297
|
)
|
308
298
|
|
309
299
|
def _calculate_exporting_managers_ended_at(self) -> None:
|
310
300
|
"""Определяет дату последнего успешного подэтапа экспорта у менеджеров Функций экспорта."""
|
311
|
-
self._exporting_data_manager_to_period_end =
|
312
|
-
|
313
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
314
|
-
max_ended_at=Max('ended_at'),
|
315
|
-
).filter(
|
316
|
-
status_id=ExportingDataSubStageStatus.FINISHED.key,
|
317
|
-
manager_id__in=[manager.uuid for manager in self._exporting_data_managers.values()],
|
318
|
-
).values_list('manager_id', 'max_ended_at'),
|
301
|
+
self._exporting_data_manager_to_period_end = get_exporting_managers_max_period_ended_dates(
|
302
|
+
self._exporting_data_managers.values()
|
319
303
|
)
|
320
304
|
|
321
305
|
def _create_collect_command(self, model: str, task_id: str) -> EduRdmCollectDataCommandProgress:
|
322
306
|
"""Создает команду сбора данных моделей РВД."""
|
323
307
|
manager = self._collecting_data_managers[model]
|
324
|
-
|
308
|
+
manager_last_collected = (
|
325
309
|
self._collecting_data_manager_to_logs_period_end.get(manager.uuid)
|
326
310
|
or get_today_min_datetime()
|
327
311
|
)
|
328
312
|
|
313
|
+
period_started_at = manager_last_collected['max_period_ended_at']
|
314
|
+
period_ended_at = timezone.now()
|
315
|
+
|
316
|
+
if period_started_at >= period_ended_at:
|
317
|
+
period_started_at = manager_last_collected['max_started_at']
|
318
|
+
|
329
319
|
return EduRdmCollectDataCommandProgress.objects.create(
|
330
320
|
model_id=model,
|
331
|
-
logs_period_started_at=
|
332
|
-
logs_period_ended_at=
|
321
|
+
logs_period_started_at=period_started_at,
|
322
|
+
logs_period_ended_at=period_ended_at,
|
333
323
|
task_id=task_id,
|
334
324
|
type=CommandType.AUTO,
|
335
325
|
)
|
@@ -337,13 +327,19 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
337
327
|
def _create_export_command(self, entity: str, task_id: str) -> Optional[EduRdmExportDataCommandProgress]:
|
338
328
|
"""Создает команду экспорта данных сущностей РВД."""
|
339
329
|
manager = self._exporting_data_managers[entity]
|
340
|
-
|
330
|
+
manager_last_exported = self._exporting_data_manager_to_period_end.get(manager.uuid)
|
331
|
+
|
332
|
+
if manager_last_exported:
|
333
|
+
period_started_at = manager_last_exported['max_period_ended_at']
|
334
|
+
period_ended_at = timezone.now()
|
335
|
+
|
336
|
+
if period_started_at >= period_ended_at:
|
337
|
+
period_started_at = manager_last_exported['max_started_at']
|
341
338
|
|
342
|
-
if period_started_at:
|
343
339
|
return EduRdmExportDataCommandProgress.objects.create(
|
344
340
|
entity_id=entity,
|
345
341
|
period_started_at=period_started_at,
|
346
|
-
period_ended_at=
|
342
|
+
period_ended_at=period_ended_at,
|
347
343
|
task_id=task_id,
|
348
344
|
type=CommandType.AUTO,
|
349
345
|
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: edu-rdm-integration
|
3
|
-
Version: 3.2.
|
3
|
+
Version: 3.2.2
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
5
5
|
Home-page:
|
6
6
|
Download-URL:
|
@@ -148,7 +148,6 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
148
148
|
RDM_TRANSFER_TASK_HOUR = conf.get('rdm_transfer_task', 'HOUR')
|
149
149
|
RDM_TRANSFER_TASK_DAY_OF_WEEK = conf.get('rdm_transfer_task', 'DAY_OF_WEEK')
|
150
150
|
RDM_TRANSFER_TASK_EXPIRE_SECOND = conf.get('rdm_transfer_task', 'LOCK_EXPIRE_SECONDS')
|
151
|
-
RDM_TRANSFER_TASK_TIMEDELTA = conf.get_int('rdm_transfer_task', 'TIMEDELTA')
|
152
151
|
|
153
152
|
# Настройка запуска периодической задачи статуса загрузки данных в витрину:
|
154
153
|
RDM_UPLOAD_STATUS_TASK_MINUTE = conf.get('rdm_upload_status_task', 'MINUTE')
|
@@ -198,7 +197,6 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
198
197
|
| RDM_TRANSFER_TASK_HOUR | Настройка запуска периодической задачи выгрузки данных. Час | '*/4' |
|
199
198
|
| RDM_TRANSFER_TASK_DAY_OF_WEEK | Настройка запуска периодической задачи выгрузки данных. День недели | '*' |
|
200
199
|
| RDM_TRANSFER_TASK_LOCK_EXPIRE_SECONDS | Время по истечении которого, блокировка может быть снята (в секунадх) | 21600 |
|
201
|
-
| RDM_TRANSFER_TASK_TIMEDELTA | Дельта между предыдущим и следующим запуском периодической задачи в секундах | 3600 |
|
202
200
|
| RDM_UPLOAD_STATUS_TASK_MINUTE | Настройка запуска периодической задачи статуса загрузки данных в витрину. Минута | '*/30' |
|
203
201
|
| RDM_UPLOAD_STATUS_TASK_HOUR | Настройка запуска периодической задачи статуса загрузки данных в витрину. Час | '*' |
|
204
202
|
| RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK | Настройка запуска периодической задачи статуса загрузки данных в витрину. День недели | '*' |
|
@@ -303,6 +301,27 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
303
301
|
|
304
302
|
### Удалено
|
305
303
|
|
304
|
+
## 3.2.2 - 2024-05-21
|
305
|
+
Изменения в переодической задаче TransferLatestEntitiesDataPeriodicTask,
|
306
|
+
Изменено определение даты последнего успешного этапа сбора у менеджеров Функций сбора и экспорта.
|
307
|
+
Изменения получения крайней даты из переданного поля по указанным моделям в BaseCollectLatestModelsData.
|
308
|
+
Удалён параметр RDM_TRANSFER_TASK_TIMEDELTA.
|
309
|
+
|
310
|
+
### Изменено
|
311
|
+
- [EDUSCHL-21804](https://jira.bars.group/browse/EDUSCHL-21804)
|
312
|
+
PATCH Изменение определения даты последнего успешного этапа сбора у менеджеров Функций сбора и экспорта
|
313
|
+
Изменения получения крайней даты из переданного поля по указанным моделям в BaseCollectLatestModelsData.
|
314
|
+
Реализованы функции get_collecting_managers_max_period_ended_dates и get_exporting_managers_max_period_ended_dates
|
315
|
+
для получения даты времени успешного этапа сбора/экспорта.
|
316
|
+
|
317
|
+
|
318
|
+
## 3.2.1 - 2024-05-03
|
319
|
+
Понижен уровень логирования при возникновении ошибки отправки запросов в РВД с ERROR до WARNING
|
320
|
+
|
321
|
+
### Изменено
|
322
|
+
- [EDUSCHL-21903](https://jira.bars.group/browse/EDUSCHL-21903)
|
323
|
+
PATCH Понижен уровень логирования при возникновении ошибки отправки запросов в РВД
|
324
|
+
|
306
325
|
|
307
326
|
## 3.2.0 - 2024-05-02
|
308
327
|
Периодические задачи RDMCheckUploadStatus, CheckSuspendedExportedStagePeriodicTask,
|
@@ -315,6 +334,14 @@ TransferLatestEntitiesDataPeriodicTask сделаны уникальными.
|
|
315
334
|
поставить новую подобную задачу.
|
316
335
|
|
317
336
|
|
337
|
+
## 3.1.1 - 2024-05-03
|
338
|
+
Понижен уровень логирования при возникновении ошибки отправки запросов в РВД с ERROR до WARNING
|
339
|
+
|
340
|
+
### Изменено
|
341
|
+
- [EDUSCHL-21903](https://jira.bars.group/browse/EDUSCHL-21903)
|
342
|
+
PATCH Понижен уровень логирования при возникновении ошибки отправки запросов в РВД
|
343
|
+
|
344
|
+
|
318
345
|
## 3.1.0 - 2024-04-23
|
319
346
|
Добавлена поддержка setuptools 69.*.
|
320
347
|
Поднята минимальная версия пакета pip 23.2.1
|
@@ -1,17 +1,17 @@
|
|
1
1
|
edu_rdm_integration/__init__.py,sha256=fVCvQ7QGI_iCyAeE8dMapyY8gOM617ye5GQqAVGPlZI,72
|
2
2
|
edu_rdm_integration/app_meta.py,sha256=v5IU69yaeLbyHF0Ln6iPN_IfizbtF3rCWrz2n71m8dU,337
|
3
|
-
edu_rdm_integration/app_settings.py,sha256=
|
3
|
+
edu_rdm_integration/app_settings.py,sha256=b8lkNz0ht7hTIKB3ZOrbF3Pp0XAJeNEs5z1WXFxHKEc,2130
|
4
4
|
edu_rdm_integration/apps.py,sha256=5OgNdmuqe26fbu4wYb69haQJe-XFO_rDbnU1vPqJU-U,3571
|
5
5
|
edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
|
6
6
|
edu_rdm_integration/consts.py,sha256=FFwcMHNsfjP_s9LfkccLAHjJMEMp7ppPmrRlJcgV88k,1104
|
7
7
|
edu_rdm_integration/entities.py,sha256=qNVWUhjwvX298Ak86_AKmqBZioP0czGwBcAz_4dtUUE,14552
|
8
8
|
edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
|
9
|
-
edu_rdm_integration/helpers.py,sha256=
|
9
|
+
edu_rdm_integration/helpers.py,sha256=pGgefbVCsi4-qb4KTy8ONV3gVNKx-FiJibFkK6hKzMM,13725
|
10
10
|
edu_rdm_integration/mapping.py,sha256=bwa2fJCbV4YjQcAgRrgT3hgM6dJhr_uBtQgx3L3F2Ck,473
|
11
11
|
edu_rdm_integration/models.py,sha256=Wg8NYj4C6Xqw-15AsQGU0TWNzUjdwBW_YZDzmPqx3GY,29327
|
12
12
|
edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
|
13
13
|
edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
|
14
|
-
edu_rdm_integration/tasks.py,sha256=
|
14
|
+
edu_rdm_integration/tasks.py,sha256=LPbEbfRtEwYlfneLrMvE3EnyN0B52qwnwMUEAIFgQDs,14998
|
15
15
|
edu_rdm_integration/utils.py,sha256=-my8q9fude6Nc9r_qUDww-8QaU2tcv-xOexewYqOPJw,10369
|
16
16
|
edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
|
17
17
|
edu_rdm_integration/adapters/apps.py,sha256=TyJTkSPs2qAHJ11fqbwLGk3Ea7ujtqWwbxqmvYNQxG8,363
|
@@ -37,7 +37,7 @@ edu_rdm_integration/collect_and_export_data/utils.py,sha256=lBWqqhLgMyz2IQN7r9zu
|
|
37
37
|
edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py,sha256=UkoaXzh3tokZ8QdCdB09v3rRZfcHhvEwNMuj3mQIB74,4714
|
38
38
|
edu_rdm_integration/collect_and_export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
39
39
|
edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
40
|
-
edu_rdm_integration/collect_data/collect.py,sha256=
|
40
|
+
edu_rdm_integration/collect_data/collect.py,sha256=i5kf__jrlBOIZC2rRD-CHXn2HeAKWpQXrqxQg-fk94M,13596
|
41
41
|
edu_rdm_integration/collect_data/generators.py,sha256=f34AAwdEcQNIokX0ypqYgjRD1XolwBVLER_HYv9ibNw,9075
|
42
42
|
edu_rdm_integration/collect_data/helpers.py,sha256=gAFZAm9YhMtQhvlwSF3LMthPPa8LsqG_zbVe7vnW_Ag,2995
|
43
43
|
edu_rdm_integration/collect_data/tests.py,sha256=-UNX3-GL0H9i89GXRfICT0ABltn_9aN_I_cmn1gQcDA,5367
|
@@ -85,7 +85,7 @@ edu_rdm_integration/enum_register/mixins.py,sha256=nmHue9-9bqKZn9_MFiL0E-5wBUBIZ
|
|
85
85
|
edu_rdm_integration/enum_register/register.py,sha256=5OWOjK-M0Erd_5CENpBaXhVtfL0pEaDl3Bev5QKNDJc,2218
|
86
86
|
edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
87
87
|
edu_rdm_integration/export_data/consts.py,sha256=Z6Uho8oViMc0t1tyQFZfrPFBwPx6KnhUf_iZ0qgob8o,16
|
88
|
-
edu_rdm_integration/export_data/export.py,sha256=
|
88
|
+
edu_rdm_integration/export_data/export.py,sha256=AAN_DqOAPENKEtlbnMF2j4CPmkoy57vJQ8t2yQRZtrM,16595
|
89
89
|
edu_rdm_integration/export_data/generators.py,sha256=yLDOcHB1PoilJwXtKGxZQhDjpeKBzEWoosahbJJ4Ba4,4020
|
90
90
|
edu_rdm_integration/export_data/helpers.py,sha256=hU346RmQ17Ra2etFvxXI7JQlLyp_0KxH1jm-eeCqejc,2933
|
91
91
|
edu_rdm_integration/export_data/strategies.py,sha256=ocHskG-x54U-ESX56OlZPgOibkGkuv7VKmK7hVmqj_0,6725
|
@@ -94,7 +94,7 @@ edu_rdm_integration/export_data/base/caches.py,sha256=dj8LGB7TMtX3cSfp-1UB3dnzcT
|
|
94
94
|
edu_rdm_integration/export_data/base/consts.py,sha256=jLuy2y7YQ921OrEuzDld3F0k6YxgCnr5yorWBQJiLLw,519
|
95
95
|
edu_rdm_integration/export_data/base/enums.py,sha256=BSmwrkzYwEQhz9NbZCJsldY532PqgZJzxzsVk6ue0bM,93
|
96
96
|
edu_rdm_integration/export_data/base/errors.py,sha256=SaxXgKWNGG8RZQ8soFlzbX8d2_j58gWVmikzNZG1s74,291
|
97
|
-
edu_rdm_integration/export_data/base/functions.py,sha256=
|
97
|
+
edu_rdm_integration/export_data/base/functions.py,sha256=ZCf3j0HA3rFrQiYHB-fG2xAW-pKyXDuDUm0yJrHyssg,12786
|
98
98
|
edu_rdm_integration/export_data/base/helpers.py,sha256=h0S_fItbaygZfNZb9oUUPNF4dmeOojNx2nHCv3m3TWU,4609
|
99
99
|
edu_rdm_integration/export_data/base/managers.py,sha256=AJBBOmQbW-cnVEa3d2DCpyoNbnRxXmgpAcUCBu1rDiI,6075
|
100
100
|
edu_rdm_integration/export_data/base/presenters.py,sha256=JXh8nwyezaC-2h_TYfPPYUZRitFr9Y9Ifp-j7CaUCL0,403
|
@@ -162,9 +162,9 @@ edu_rdm_integration/uploader_log/apps.py,sha256=tYJj4-sDlq8fLOSvw18L_yys7SILpTKW
|
|
162
162
|
edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJLdsGmr3SXwH_U,266
|
163
163
|
edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
|
164
164
|
edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
|
165
|
-
edu_rdm_integration-3.2.
|
166
|
-
edu_rdm_integration-3.2.
|
167
|
-
edu_rdm_integration-3.2.
|
168
|
-
edu_rdm_integration-3.2.
|
169
|
-
edu_rdm_integration-3.2.
|
170
|
-
edu_rdm_integration-3.2.
|
165
|
+
edu_rdm_integration-3.2.2.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
166
|
+
edu_rdm_integration-3.2.2.dist-info/METADATA,sha256=pf2mWm6W814a7FE1IGxpN0bhvun_2q7Zm0dmi1k6SHQ,65013
|
167
|
+
edu_rdm_integration-3.2.2.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
168
|
+
edu_rdm_integration-3.2.2.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
169
|
+
edu_rdm_integration-3.2.2.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
170
|
+
edu_rdm_integration-3.2.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|
{edu_rdm_integration-3.2.0.dist-info → edu_rdm_integration-3.2.2.dist-info}/namespace_packages.txt
RENAMED
File without changes
|
File without changes
|