edu-rdm-integration 3.2.1__py3-none-any.whl → 3.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/app_settings.py +0 -1
- edu_rdm_integration/collect_data/collect.py +17 -48
- edu_rdm_integration/export_data/export.py +21 -35
- edu_rdm_integration/helpers.py +74 -3
- edu_rdm_integration/tasks.py +20 -30
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/METADATA +32 -3
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/RECORD +11 -11
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/LICENSE +0 -0
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/namespace_packages.txt +0 -0
- {edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/top_level.txt +0 -0
@@ -37,7 +37,6 @@ RDM_TRANSFER_TASK_MINUTE = '0'
|
|
37
37
|
RDM_TRANSFER_TASK_HOUR = '*/4'
|
38
38
|
RDM_TRANSFER_TASK_DAY_OF_WEEK = '*'
|
39
39
|
RDM_TRANSFER_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 6
|
40
|
-
RDM_TRANSFER_TASK_TIMEDELTA = 3600
|
41
40
|
|
42
41
|
# Настройка запуска периодической задачи статуса загрузки данных в витрину:
|
43
42
|
RDM_UPLOAD_STATUS_TASK_MINUTE = '*/30'
|
@@ -18,14 +18,6 @@ from typing import (
|
|
18
18
|
from django.conf import (
|
19
19
|
settings,
|
20
20
|
)
|
21
|
-
from django.db.models import (
|
22
|
-
F,
|
23
|
-
Max,
|
24
|
-
CharField,
|
25
|
-
)
|
26
|
-
from django.db.models.functions import (
|
27
|
-
Cast,
|
28
|
-
)
|
29
21
|
from django.utils import (
|
30
22
|
timezone,
|
31
23
|
)
|
@@ -33,9 +25,6 @@ from django.utils import (
|
|
33
25
|
from educommon import (
|
34
26
|
logger,
|
35
27
|
)
|
36
|
-
from educommon.audit_log.models import (
|
37
|
-
AuditLog,
|
38
|
-
)
|
39
28
|
from educommon.utils.date import (
|
40
29
|
get_today_min_datetime,
|
41
30
|
)
|
@@ -56,11 +45,12 @@ from edu_rdm_integration.consts import (
|
|
56
45
|
LOGS_SUB_PERIOD_DAYS,
|
57
46
|
REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
|
58
47
|
)
|
48
|
+
from edu_rdm_integration.helpers import (
|
49
|
+
get_collecting_managers_max_period_ended_dates,
|
50
|
+
)
|
59
51
|
from edu_rdm_integration.models import (
|
60
52
|
CollectingDataStageStatus,
|
61
|
-
CollectingDataSubStageStatus,
|
62
53
|
CollectingExportedDataStage,
|
63
|
-
CollectingExportedDataSubStage,
|
64
54
|
RegionalDataMartModelEnum,
|
65
55
|
)
|
66
56
|
from edu_rdm_integration.signals import (
|
@@ -72,6 +62,10 @@ from edu_rdm_integration.storages import (
|
|
72
62
|
|
73
63
|
|
74
64
|
if TYPE_CHECKING:
|
65
|
+
from educommon.audit_log.models import (
|
66
|
+
AuditLog,
|
67
|
+
)
|
68
|
+
|
75
69
|
from edu_rdm_integration.collect_data.generators import (
|
76
70
|
BaseEduLogGenerator,
|
77
71
|
)
|
@@ -141,7 +135,7 @@ class BaseCollectModelsData(BaseOperationData):
|
|
141
135
|
|
142
136
|
logger.info('collecting models data managers finished')
|
143
137
|
|
144
|
-
def _collect_models_data(self, *args, logs: Optional[Dict[str, List[AuditLog]]] = None, **kwargs):
|
138
|
+
def _collect_models_data(self, *args, logs: Optional[Dict[str, List['AuditLog']]] = None, **kwargs):
|
145
139
|
"""
|
146
140
|
Запуск Функций по формированию данных моделей РВД из логов.
|
147
141
|
"""
|
@@ -227,7 +221,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
227
221
|
if temp_logs_period_ended_at > self.logs_period_ended_at:
|
228
222
|
temp_logs_period_ended_at = self.logs_period_ended_at
|
229
223
|
|
230
|
-
temp_logs: Dict[str, List[AuditLog]] = {}
|
224
|
+
temp_logs: Dict[str, List['AuditLog']] = {}
|
231
225
|
|
232
226
|
while temp_logs_period_started_at < temp_logs_period_ended_at <= self.logs_period_ended_at:
|
233
227
|
for model in self.models:
|
@@ -254,7 +248,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
254
248
|
"""
|
255
249
|
Генерация логов за весь период.
|
256
250
|
"""
|
257
|
-
temp_logs: Dict[str, List[AuditLog]] = {}
|
251
|
+
temp_logs: Dict[str, List['AuditLog']] = {}
|
258
252
|
|
259
253
|
for model in self.models:
|
260
254
|
logs = self.log_generator.generate(
|
@@ -268,7 +262,7 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
|
|
268
262
|
|
269
263
|
return [(temp_logs, self.logs_period_started_at, self.logs_period_ended_at)]
|
270
264
|
|
271
|
-
def _generate_logs(self) -> List[Tuple[Dict[str, List[AuditLog]], datetime, datetime]]:
|
265
|
+
def _generate_logs(self) -> List[Tuple[Dict[str, List['AuditLog']], datetime, datetime]]:
|
272
266
|
"""
|
273
267
|
Генерация логов.
|
274
268
|
|
@@ -318,48 +312,23 @@ class BaseCollectLatestModelsData(BaseCollectModelsData):
|
|
318
312
|
|
319
313
|
return stage_created_or_in_progress
|
320
314
|
|
321
|
-
def
|
322
|
-
"""
|
323
|
-
Возвращает словарь с uuid менеджера и крайней датой из переданного поля по указанным моделям.
|
324
|
-
"""
|
325
|
-
manager_to_start_date = dict(
|
326
|
-
CollectingExportedDataSubStage.objects.annotate(
|
327
|
-
max_date=Max(f'stage__{date_field}'),
|
328
|
-
).values(
|
329
|
-
'stage__manager_id', 'max_date',
|
330
|
-
).annotate(
|
331
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
332
|
-
date=F('max_date'),
|
333
|
-
).filter(
|
334
|
-
status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
335
|
-
manager_id__in=(m.uuid for m in
|
336
|
-
self._collecting_data_managers.values()),
|
337
|
-
).values_list('manager_id', 'date')
|
338
|
-
)
|
339
|
-
|
340
|
-
return manager_to_start_date
|
341
|
-
|
342
|
-
def _collect_models_data(self, *args, logs: Optional[Dict[str, List[AuditLog]]] = None, **kwargs) -> None:
|
315
|
+
def _collect_models_data(self, *args, logs: Optional[Dict[str, List['AuditLog']]] = None, **kwargs) -> None:
|
343
316
|
"""
|
344
317
|
Запуск Функций по формированию данных из логов для дальнейшей выгрузки.
|
345
318
|
"""
|
346
319
|
logger.info('collect models data..')
|
347
320
|
|
348
|
-
|
349
|
-
|
321
|
+
managers_last_period_ended_at = get_collecting_managers_max_period_ended_dates(
|
322
|
+
self._collecting_data_managers.values()
|
323
|
+
)
|
350
324
|
|
351
325
|
for model_key, manager_class in self._collecting_data_managers.items():
|
352
326
|
model_logs = logs.get(model_key) if logs else None
|
327
|
+
manager_last_period_ended_at = managers_last_period_ended_at.get(manager_class.uuid)
|
353
328
|
|
354
|
-
kwargs['logs_period_started_at'] = (
|
355
|
-
last_finished_model_upload.get(manager_class.uuid)
|
356
|
-
or get_today_min_datetime()
|
357
|
-
)
|
329
|
+
kwargs['logs_period_started_at'] = manager_last_period_ended_at or get_today_min_datetime()
|
358
330
|
kwargs['logs_period_ended_at'] = timezone.now()
|
359
331
|
|
360
|
-
if kwargs['logs_period_started_at'] > kwargs['logs_period_ended_at']:
|
361
|
-
kwargs['logs_period_started_at'] = last_model_start.get(manager_class.uuid)
|
362
|
-
|
363
332
|
manager = manager_class(*args, logs=model_logs, **kwargs)
|
364
333
|
|
365
334
|
if self.command_id:
|
@@ -9,6 +9,7 @@ from datetime import (
|
|
9
9
|
timedelta,
|
10
10
|
)
|
11
11
|
from typing import (
|
12
|
+
TYPE_CHECKING,
|
12
13
|
Dict,
|
13
14
|
Iterable,
|
14
15
|
List,
|
@@ -21,7 +22,6 @@ from django.conf import (
|
|
21
22
|
)
|
22
23
|
from django.db.models import (
|
23
24
|
F,
|
24
|
-
Max,
|
25
25
|
Model,
|
26
26
|
Value,
|
27
27
|
)
|
@@ -53,9 +53,6 @@ from educommon.utils.date import (
|
|
53
53
|
from educommon.utils.seqtools import (
|
54
54
|
make_chunks,
|
55
55
|
)
|
56
|
-
from function_tools.managers import (
|
57
|
-
RunnerManager,
|
58
|
-
)
|
59
56
|
from m3_db_utils.consts import (
|
60
57
|
DEFAULT_ORDER_NUMBER,
|
61
58
|
)
|
@@ -72,9 +69,11 @@ from edu_rdm_integration.consts import (
|
|
72
69
|
from edu_rdm_integration.enums import (
|
73
70
|
FileUploadStatusEnum,
|
74
71
|
)
|
72
|
+
from edu_rdm_integration.helpers import (
|
73
|
+
get_exporting_managers_max_period_ended_dates,
|
74
|
+
)
|
75
75
|
from edu_rdm_integration.models import (
|
76
76
|
ExportingDataStage,
|
77
|
-
ExportingDataSubStage,
|
78
77
|
ExportingDataSubStageStatus,
|
79
78
|
RegionalDataMartEntityEnum,
|
80
79
|
)
|
@@ -86,6 +85,12 @@ from edu_rdm_integration.storages import (
|
|
86
85
|
)
|
87
86
|
|
88
87
|
|
88
|
+
if TYPE_CHECKING:
|
89
|
+
from function_tools.managers import (
|
90
|
+
RunnerManager,
|
91
|
+
)
|
92
|
+
|
93
|
+
|
89
94
|
class BaseExportEntitiesData(BaseOperationData):
|
90
95
|
"""Базовый класс экспорта сущностей РВД за указанных период."""
|
91
96
|
|
@@ -108,7 +113,7 @@ class BaseExportEntitiesData(BaseOperationData):
|
|
108
113
|
self.period_ended_at = period_ended_at
|
109
114
|
|
110
115
|
# Классы менеджеров Функций, которые должны быть запущены для выгрузки данных
|
111
|
-
self._exporting_data_managers: Set[Type[RunnerManager]] = set()
|
116
|
+
self._exporting_data_managers: Set[Type['RunnerManager']] = set()
|
112
117
|
|
113
118
|
# Результаты работы Функций выгрузки данных
|
114
119
|
self._exporting_data_results = []
|
@@ -277,7 +282,7 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
277
282
|
):
|
278
283
|
super().__init__(entities, period_started_at, period_ended_at, **kwargs)
|
279
284
|
|
280
|
-
self._exporting_data_managers: Set[Type[RunnerManager]] = OrderedSet()
|
285
|
+
self._exporting_data_managers: Set[Type['RunnerManager']] = OrderedSet()
|
281
286
|
|
282
287
|
# Словарь данных с классами менеджеров и их сущностями
|
283
288
|
self._manager_entities_map: Dict[Type[object], List[str]] = defaultdict(set)
|
@@ -353,20 +358,6 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
353
358
|
):
|
354
359
|
queryset.filter(id__in=model_ids).update(modified=now)
|
355
360
|
|
356
|
-
def _calculate_last_finished_entity_export(self) -> Dict[str, datetime]:
|
357
|
-
"""Находит левую границу экспорта данных для сущностей."""
|
358
|
-
manager_to_last_date = ExportingDataSubStage.objects.values(
|
359
|
-
'stage__manager_id',
|
360
|
-
).annotate(
|
361
|
-
max_ended_at=Max('ended_at'),
|
362
|
-
entity_uuid=F('stage__manager_id'),
|
363
|
-
).filter(
|
364
|
-
status_id=ExportingDataSubStageStatus.FINISHED.key,
|
365
|
-
entity_uuid__in=[m.uuid for m in self._exporting_data_managers],
|
366
|
-
).values('entity_uuid', 'max_ended_at')
|
367
|
-
|
368
|
-
return {str(m['entity_uuid']): m['max_ended_at'] for m in manager_to_last_date}
|
369
|
-
|
370
361
|
def _export_entities_data(self, *args, **kwargs) -> None:
|
371
362
|
"""
|
372
363
|
Запуск Функций по для экспорта данных.
|
@@ -376,27 +367,22 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
|
|
376
367
|
# Массив с выгружаемыми сущностями для поля "Описание" в асинхронной задаче
|
377
368
|
exported_entities = []
|
378
369
|
|
379
|
-
|
370
|
+
managers_max_period_ended = get_exporting_managers_max_period_ended_dates(
|
371
|
+
self._exporting_data_managers
|
372
|
+
)
|
380
373
|
|
381
374
|
for manager_class in self._exporting_data_managers:
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
)
|
375
|
+
manager_last_exported = managers_max_period_ended.get(manager_class.uuid)
|
376
|
+
|
377
|
+
kwargs['period_started_at'] = manager_last_exported or timezone.now()
|
378
|
+
kwargs['period_ended_at'] = get_today_max_datetime()
|
379
|
+
|
386
380
|
# Обновить поля modified у модели сущности:
|
387
381
|
self._update_model_modified_field(
|
388
382
|
manager_id=manager_class.uuid,
|
389
|
-
last_finished_export_data=
|
383
|
+
last_finished_export_data=kwargs['period_started_at'],
|
390
384
|
)
|
391
385
|
|
392
|
-
kwargs['period_started_at'] = last_finished_export_data
|
393
|
-
kwargs['period_ended_at'] = get_today_max_datetime()
|
394
|
-
|
395
|
-
if kwargs['period_started_at'] > kwargs['period_ended_at']:
|
396
|
-
kwargs['period_started_at'] = kwargs['period_ended_at'] - timedelta(
|
397
|
-
seconds=settings.RDM_TRANSFER_TASK_TIMEDELTA
|
398
|
-
)
|
399
|
-
|
400
386
|
manager = manager_class(*args, **kwargs)
|
401
387
|
|
402
388
|
if self.command_id:
|
edu_rdm_integration/helpers.py
CHANGED
@@ -11,6 +11,7 @@ from typing import (
|
|
11
11
|
Any,
|
12
12
|
Dict,
|
13
13
|
List,
|
14
|
+
Iterable,
|
14
15
|
Optional,
|
15
16
|
Tuple,
|
16
17
|
Union,
|
@@ -23,15 +24,22 @@ from django.db import (
|
|
23
24
|
transaction,
|
24
25
|
)
|
25
26
|
from django.db.models import (
|
27
|
+
CharField,
|
28
|
+
OuterRef,
|
26
29
|
QuerySet,
|
30
|
+
Subquery,
|
27
31
|
)
|
28
|
-
from
|
29
|
-
|
32
|
+
from django.db.models.functions import (
|
33
|
+
Cast,
|
34
|
+
Least,
|
30
35
|
)
|
31
36
|
|
32
37
|
from educommon import (
|
33
38
|
logger,
|
34
39
|
)
|
40
|
+
from uploader_client.adapters import (
|
41
|
+
adapter,
|
42
|
+
)
|
35
43
|
|
36
44
|
from edu_rdm_integration.collect_and_export_data.models import (
|
37
45
|
EduRdmCollectDataCommandProgress,
|
@@ -45,12 +53,25 @@ from edu_rdm_integration.export_data.base.requests import (
|
|
45
53
|
)
|
46
54
|
from edu_rdm_integration.models import (
|
47
55
|
DataMartRequestStatus,
|
56
|
+
ExportingDataStage,
|
57
|
+
CollectingDataStageStatus,
|
58
|
+
CollectingExportedDataStage,
|
48
59
|
ExportingDataSubStageUploaderClientLog,
|
49
60
|
UploadStatusRequestLog,
|
61
|
+
ExportingDataStageStatus,
|
50
62
|
)
|
51
63
|
|
52
|
-
|
53
64
|
if TYPE_CHECKING:
|
65
|
+
from datetime import (
|
66
|
+
datetime,
|
67
|
+
)
|
68
|
+
|
69
|
+
from edu_rdm_integration.collect_data.non_calculated.base.managers import (
|
70
|
+
BaseCollectingExportedDataRunnerManager,
|
71
|
+
)
|
72
|
+
from edu_rdm_integration.export_data.base.managers import (
|
73
|
+
BaseExportDataRunnerManager,
|
74
|
+
)
|
54
75
|
from uploader_client.models import (
|
55
76
|
Entry,
|
56
77
|
)
|
@@ -281,3 +302,53 @@ def save_command_log_link(
|
|
281
302
|
if os.path.exists(log_file):
|
282
303
|
command.logs_link = os.path.join(log_dir, f'{command.id}.log')
|
283
304
|
command.save()
|
305
|
+
|
306
|
+
|
307
|
+
def get_collecting_managers_max_period_ended_dates(
|
308
|
+
collecting_managers: Iterable['BaseCollectingExportedDataRunnerManager'],
|
309
|
+
) -> Dict[str, 'datetime']:
|
310
|
+
"""Возвращает дату и время завершения последнего успешного этапа сбора для менеджеров Функций сбора."""
|
311
|
+
managers_last_period_ended = CollectingExportedDataStage.objects.filter(
|
312
|
+
status_id=CollectingDataStageStatus.FINISHED.key,
|
313
|
+
manager_id__in=[manager.uuid for manager in collecting_managers],
|
314
|
+
id=Subquery(
|
315
|
+
CollectingExportedDataStage.objects.filter(
|
316
|
+
manager_id=OuterRef('manager_id')
|
317
|
+
).order_by('-id').values('id')[:1]
|
318
|
+
),
|
319
|
+
).annotate(
|
320
|
+
str_manager_id=Cast('manager_id', output_field=CharField()),
|
321
|
+
last_period_ended_at=Least('logs_period_ended_at', 'started_at'),
|
322
|
+
).values_list(
|
323
|
+
'str_manager_id', 'last_period_ended_at',
|
324
|
+
)
|
325
|
+
|
326
|
+
return {
|
327
|
+
manager_id: last_period_ended_at
|
328
|
+
for manager_id, last_period_ended_at in managers_last_period_ended
|
329
|
+
}
|
330
|
+
|
331
|
+
|
332
|
+
def get_exporting_managers_max_period_ended_dates(
|
333
|
+
exporting_managers: Iterable['BaseExportDataRunnerManager'],
|
334
|
+
) -> Dict[str, 'datetime']:
|
335
|
+
"""Возвращает дату и время последнего успешного этапа экспорта для менеджеров Функций экспорта."""
|
336
|
+
managers_last_period_ended = ExportingDataStage.objects.filter(
|
337
|
+
status_id=ExportingDataStageStatus.FINISHED.key,
|
338
|
+
manager_id__in=[manager.uuid for manager in exporting_managers],
|
339
|
+
id=Subquery(
|
340
|
+
ExportingDataStage.objects.filter(
|
341
|
+
manager_id=OuterRef('manager_id')
|
342
|
+
).order_by('-id').values('id')[:1]
|
343
|
+
),
|
344
|
+
).annotate(
|
345
|
+
str_manager_id=Cast('manager_id', output_field=CharField()),
|
346
|
+
last_period_ended_at=Least('period_ended_at', 'started_at'),
|
347
|
+
).values_list(
|
348
|
+
'str_manager_id', 'last_period_ended_at',
|
349
|
+
)
|
350
|
+
|
351
|
+
return {
|
352
|
+
manager_id: last_period_ended_at
|
353
|
+
for manager_id, last_period_ended_at in managers_last_period_ended
|
354
|
+
}
|
edu_rdm_integration/tasks.py
CHANGED
@@ -28,7 +28,6 @@ from django.db.models.functions import (
|
|
28
28
|
from django.utils import (
|
29
29
|
timezone,
|
30
30
|
)
|
31
|
-
|
32
31
|
from educommon.async_task.locker import (
|
33
32
|
TaskLocker,
|
34
33
|
)
|
@@ -72,14 +71,12 @@ from edu_rdm_integration.export_data.helpers import (
|
|
72
71
|
set_failed_status_suspended_exporting_data_stages,
|
73
72
|
)
|
74
73
|
from edu_rdm_integration.helpers import (
|
74
|
+
get_collecting_managers_max_period_ended_dates,
|
75
|
+
get_exporting_managers_max_period_ended_dates,
|
75
76
|
UploadStatusHelper,
|
76
77
|
save_command_log_link,
|
77
78
|
)
|
78
79
|
from edu_rdm_integration.models import (
|
79
|
-
CollectingDataSubStageStatus,
|
80
|
-
CollectingExportedDataSubStage,
|
81
|
-
ExportingDataSubStage,
|
82
|
-
ExportingDataSubStageStatus,
|
83
80
|
ExportingDataSubStageUploaderClientLog,
|
84
81
|
RegionalDataMartEntityEnum,
|
85
82
|
TransferredEntity,
|
@@ -88,7 +85,6 @@ from edu_rdm_integration.storages import (
|
|
88
85
|
RegionalDataMartEntityStorage,
|
89
86
|
)
|
90
87
|
|
91
|
-
|
92
88
|
if TYPE_CHECKING:
|
93
89
|
from datetime import (
|
94
90
|
datetime,
|
@@ -214,10 +210,10 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
214
210
|
super().__init__()
|
215
211
|
|
216
212
|
self._collecting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
217
|
-
self._collecting_data_manager_to_logs_period_end: Dict[str, datetime] = {}
|
213
|
+
self._collecting_data_manager_to_logs_period_end: Dict[str, 'datetime'] = {}
|
218
214
|
|
219
215
|
self._exporting_data_managers: Dict[str, Type['RunnerManager']] = {}
|
220
|
-
self._exporting_data_manager_to_period_end: Dict[str, datetime] = {}
|
216
|
+
self._exporting_data_manager_to_period_end: Dict[str, 'datetime'] = {}
|
221
217
|
|
222
218
|
self._transferred_entities = []
|
223
219
|
|
@@ -296,40 +292,31 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
296
292
|
|
297
293
|
def _calculate_collecting_managers_logs_period_ended_at(self) -> None:
|
298
294
|
"""Определяет дату последнего успешного этапа сбора у менеджеров Функций сбора."""
|
299
|
-
self._collecting_data_manager_to_logs_period_end =
|
300
|
-
|
301
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
302
|
-
max_logs_period_ended_at=Max('stage__logs_period_ended_at'),
|
303
|
-
).filter(
|
304
|
-
status_id=CollectingDataSubStageStatus.READY_TO_EXPORT.key,
|
305
|
-
manager_id__in=[manager.uuid for manager in self._collecting_data_managers.values()],
|
306
|
-
).values_list('manager_id', 'max_logs_period_ended_at'),
|
295
|
+
self._collecting_data_manager_to_logs_period_end = get_collecting_managers_max_period_ended_dates(
|
296
|
+
self._collecting_data_managers.values()
|
307
297
|
)
|
308
298
|
|
309
299
|
def _calculate_exporting_managers_ended_at(self) -> None:
|
310
300
|
"""Определяет дату последнего успешного подэтапа экспорта у менеджеров Функций экспорта."""
|
311
|
-
self._exporting_data_manager_to_period_end =
|
312
|
-
|
313
|
-
manager_id=Cast('stage__manager_id', output_field=CharField()),
|
314
|
-
max_ended_at=Max('ended_at'),
|
315
|
-
).filter(
|
316
|
-
status_id=ExportingDataSubStageStatus.FINISHED.key,
|
317
|
-
manager_id__in=[manager.uuid for manager in self._exporting_data_managers.values()],
|
318
|
-
).values_list('manager_id', 'max_ended_at'),
|
301
|
+
self._exporting_data_manager_to_period_end = get_exporting_managers_max_period_ended_dates(
|
302
|
+
self._exporting_data_managers.values()
|
319
303
|
)
|
320
304
|
|
321
305
|
def _create_collect_command(self, model: str, task_id: str) -> EduRdmCollectDataCommandProgress:
|
322
306
|
"""Создает команду сбора данных моделей РВД."""
|
323
307
|
manager = self._collecting_data_managers[model]
|
324
|
-
|
308
|
+
manager_last_collected = (
|
325
309
|
self._collecting_data_manager_to_logs_period_end.get(manager.uuid)
|
326
310
|
or get_today_min_datetime()
|
327
311
|
)
|
328
312
|
|
313
|
+
period_started_at = manager_last_collected
|
314
|
+
period_ended_at = timezone.now()
|
315
|
+
|
329
316
|
return EduRdmCollectDataCommandProgress.objects.create(
|
330
317
|
model_id=model,
|
331
|
-
logs_period_started_at=
|
332
|
-
logs_period_ended_at=
|
318
|
+
logs_period_started_at=period_started_at,
|
319
|
+
logs_period_ended_at=period_ended_at,
|
333
320
|
task_id=task_id,
|
334
321
|
type=CommandType.AUTO,
|
335
322
|
)
|
@@ -337,13 +324,16 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
|
|
337
324
|
def _create_export_command(self, entity: str, task_id: str) -> Optional[EduRdmExportDataCommandProgress]:
|
338
325
|
"""Создает команду экспорта данных сущностей РВД."""
|
339
326
|
manager = self._exporting_data_managers[entity]
|
340
|
-
|
327
|
+
manager_last_exported = self._exporting_data_manager_to_period_end.get(manager.uuid)
|
328
|
+
|
329
|
+
if manager_last_exported:
|
330
|
+
period_started_at = manager_last_exported
|
331
|
+
period_ended_at = timezone.now()
|
341
332
|
|
342
|
-
if period_started_at:
|
343
333
|
return EduRdmExportDataCommandProgress.objects.create(
|
344
334
|
entity_id=entity,
|
345
335
|
period_started_at=period_started_at,
|
346
|
-
period_ended_at=
|
336
|
+
period_ended_at=period_ended_at,
|
347
337
|
task_id=task_id,
|
348
338
|
type=CommandType.AUTO,
|
349
339
|
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: edu-rdm-integration
|
3
|
-
Version: 3.2.
|
3
|
+
Version: 3.2.3
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
5
5
|
Home-page:
|
6
6
|
Download-URL:
|
@@ -148,7 +148,6 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
148
148
|
RDM_TRANSFER_TASK_HOUR = conf.get('rdm_transfer_task', 'HOUR')
|
149
149
|
RDM_TRANSFER_TASK_DAY_OF_WEEK = conf.get('rdm_transfer_task', 'DAY_OF_WEEK')
|
150
150
|
RDM_TRANSFER_TASK_EXPIRE_SECOND = conf.get('rdm_transfer_task', 'LOCK_EXPIRE_SECONDS')
|
151
|
-
RDM_TRANSFER_TASK_TIMEDELTA = conf.get_int('rdm_transfer_task', 'TIMEDELTA')
|
152
151
|
|
153
152
|
# Настройка запуска периодической задачи статуса загрузки данных в витрину:
|
154
153
|
RDM_UPLOAD_STATUS_TASK_MINUTE = conf.get('rdm_upload_status_task', 'MINUTE')
|
@@ -198,7 +197,6 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
198
197
|
| RDM_TRANSFER_TASK_HOUR | Настройка запуска периодической задачи выгрузки данных. Час | '*/4' |
|
199
198
|
| RDM_TRANSFER_TASK_DAY_OF_WEEK | Настройка запуска периодической задачи выгрузки данных. День недели | '*' |
|
200
199
|
| RDM_TRANSFER_TASK_LOCK_EXPIRE_SECONDS | Время по истечении которого, блокировка может быть снята (в секунадх) | 21600 |
|
201
|
-
| RDM_TRANSFER_TASK_TIMEDELTA | Дельта между предыдущим и следующим запуском периодической задачи в секундах | 3600 |
|
202
200
|
| RDM_UPLOAD_STATUS_TASK_MINUTE | Настройка запуска периодической задачи статуса загрузки данных в витрину. Минута | '*/30' |
|
203
201
|
| RDM_UPLOAD_STATUS_TASK_HOUR | Настройка запуска периодической задачи статуса загрузки данных в витрину. Час | '*' |
|
204
202
|
| RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK | Настройка запуска периодической задачи статуса загрузки данных в витрину. День недели | '*' |
|
@@ -304,6 +302,29 @@ Requires-Dist: uploader-client <1,>=0.2.1
|
|
304
302
|
### Удалено
|
305
303
|
|
306
304
|
|
305
|
+
## 3.2.3 - 2024-06-06
|
306
|
+
Внесены корректировки в запросы используемые в функциях get_collecting_managers_max_period_ended_dates и get_exporting_managers_max_period_ended_dates.
|
307
|
+
|
308
|
+
### Изменено
|
309
|
+
- [EDUSCHL-21804](https://jira.bars.group/browse/EDUSCHL-21804)
|
310
|
+
PATCH Внесены корректировки в запросы используемые в функциях get_collecting_managers_max_period_ended_dates и get_exporting_managers_max_period_ended_dates.
|
311
|
+
Дата и время окончания предыдущего сбора/экспорта могла быть не корректно определена.
|
312
|
+
|
313
|
+
|
314
|
+
## 3.2.2 - 2024-05-21
|
315
|
+
Изменения в переодической задаче TransferLatestEntitiesDataPeriodicTask,
|
316
|
+
Изменено определение даты последнего успешного этапа сбора у менеджеров Функций сбора и экспорта.
|
317
|
+
Изменения получения крайней даты из переданного поля по указанным моделям в BaseCollectLatestModelsData.
|
318
|
+
Удалён параметр RDM_TRANSFER_TASK_TIMEDELTA.
|
319
|
+
|
320
|
+
### Изменено
|
321
|
+
- [EDUSCHL-21804](https://jira.bars.group/browse/EDUSCHL-21804)
|
322
|
+
PATCH Изменение определения даты последнего успешного этапа сбора у менеджеров Функций сбора и экспорта
|
323
|
+
Изменения получения крайней даты из переданного поля по указанным моделям в BaseCollectLatestModelsData.
|
324
|
+
Реализованы функции get_collecting_managers_max_period_ended_dates и get_exporting_managers_max_period_ended_dates
|
325
|
+
для получения даты времени успешного этапа сбора/экспорта.
|
326
|
+
|
327
|
+
|
307
328
|
## 3.2.1 - 2024-05-03
|
308
329
|
Понижен уровень логирования при возникновении ошибки отправки запросов в РВД с ERROR до WARNING
|
309
330
|
|
@@ -323,6 +344,14 @@ TransferLatestEntitiesDataPeriodicTask сделаны уникальными.
|
|
323
344
|
поставить новую подобную задачу.
|
324
345
|
|
325
346
|
|
347
|
+
## 3.1.1 - 2024-05-03
|
348
|
+
Понижен уровень логирования при возникновении ошибки отправки запросов в РВД с ERROR до WARNING
|
349
|
+
|
350
|
+
### Изменено
|
351
|
+
- [EDUSCHL-21903](https://jira.bars.group/browse/EDUSCHL-21903)
|
352
|
+
PATCH Понижен уровень логирования при возникновении ошибки отправки запросов в РВД
|
353
|
+
|
354
|
+
|
326
355
|
## 3.1.0 - 2024-04-23
|
327
356
|
Добавлена поддержка setuptools 69.*.
|
328
357
|
Поднята минимальная версия пакета pip 23.2.1
|
@@ -1,17 +1,17 @@
|
|
1
1
|
edu_rdm_integration/__init__.py,sha256=fVCvQ7QGI_iCyAeE8dMapyY8gOM617ye5GQqAVGPlZI,72
|
2
2
|
edu_rdm_integration/app_meta.py,sha256=v5IU69yaeLbyHF0Ln6iPN_IfizbtF3rCWrz2n71m8dU,337
|
3
|
-
edu_rdm_integration/app_settings.py,sha256=
|
3
|
+
edu_rdm_integration/app_settings.py,sha256=b8lkNz0ht7hTIKB3ZOrbF3Pp0XAJeNEs5z1WXFxHKEc,2130
|
4
4
|
edu_rdm_integration/apps.py,sha256=5OgNdmuqe26fbu4wYb69haQJe-XFO_rDbnU1vPqJU-U,3571
|
5
5
|
edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
|
6
6
|
edu_rdm_integration/consts.py,sha256=FFwcMHNsfjP_s9LfkccLAHjJMEMp7ppPmrRlJcgV88k,1104
|
7
7
|
edu_rdm_integration/entities.py,sha256=qNVWUhjwvX298Ak86_AKmqBZioP0czGwBcAz_4dtUUE,14552
|
8
8
|
edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
|
9
|
-
edu_rdm_integration/helpers.py,sha256=
|
9
|
+
edu_rdm_integration/helpers.py,sha256=lUJv8X8gCY-w2SfcyOShNRtydOTJe0dGHN4h2Y6elqk,13715
|
10
10
|
edu_rdm_integration/mapping.py,sha256=bwa2fJCbV4YjQcAgRrgT3hgM6dJhr_uBtQgx3L3F2Ck,473
|
11
11
|
edu_rdm_integration/models.py,sha256=Wg8NYj4C6Xqw-15AsQGU0TWNzUjdwBW_YZDzmPqx3GY,29327
|
12
12
|
edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
|
13
13
|
edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
|
14
|
-
edu_rdm_integration/tasks.py,sha256=
|
14
|
+
edu_rdm_integration/tasks.py,sha256=PEqJeuGaWA_GgE4N6RU57dj-p2hRzGqYmKLRdSg2otQ,14699
|
15
15
|
edu_rdm_integration/utils.py,sha256=-my8q9fude6Nc9r_qUDww-8QaU2tcv-xOexewYqOPJw,10369
|
16
16
|
edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
|
17
17
|
edu_rdm_integration/adapters/apps.py,sha256=TyJTkSPs2qAHJ11fqbwLGk3Ea7ujtqWwbxqmvYNQxG8,363
|
@@ -37,7 +37,7 @@ edu_rdm_integration/collect_and_export_data/utils.py,sha256=lBWqqhLgMyz2IQN7r9zu
|
|
37
37
|
edu_rdm_integration/collect_and_export_data/migrations/0001_initial.py,sha256=UkoaXzh3tokZ8QdCdB09v3rRZfcHhvEwNMuj3mQIB74,4714
|
38
38
|
edu_rdm_integration/collect_and_export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
39
39
|
edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
40
|
-
edu_rdm_integration/collect_data/collect.py,sha256=
|
40
|
+
edu_rdm_integration/collect_data/collect.py,sha256=UR4thiSJ4qFCgMRpJxKtylP5fcrtvxK2Q4FLZSSkN_4,13365
|
41
41
|
edu_rdm_integration/collect_data/generators.py,sha256=f34AAwdEcQNIokX0ypqYgjRD1XolwBVLER_HYv9ibNw,9075
|
42
42
|
edu_rdm_integration/collect_data/helpers.py,sha256=gAFZAm9YhMtQhvlwSF3LMthPPa8LsqG_zbVe7vnW_Ag,2995
|
43
43
|
edu_rdm_integration/collect_data/tests.py,sha256=-UNX3-GL0H9i89GXRfICT0ABltn_9aN_I_cmn1gQcDA,5367
|
@@ -85,7 +85,7 @@ edu_rdm_integration/enum_register/mixins.py,sha256=nmHue9-9bqKZn9_MFiL0E-5wBUBIZ
|
|
85
85
|
edu_rdm_integration/enum_register/register.py,sha256=5OWOjK-M0Erd_5CENpBaXhVtfL0pEaDl3Bev5QKNDJc,2218
|
86
86
|
edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
87
87
|
edu_rdm_integration/export_data/consts.py,sha256=Z6Uho8oViMc0t1tyQFZfrPFBwPx6KnhUf_iZ0qgob8o,16
|
88
|
-
edu_rdm_integration/export_data/export.py,sha256=
|
88
|
+
edu_rdm_integration/export_data/export.py,sha256=3d1POl4xzyh4GVXJYdkXkcyfiUIqsrlPKPO-apw30Uk,16405
|
89
89
|
edu_rdm_integration/export_data/generators.py,sha256=yLDOcHB1PoilJwXtKGxZQhDjpeKBzEWoosahbJJ4Ba4,4020
|
90
90
|
edu_rdm_integration/export_data/helpers.py,sha256=hU346RmQ17Ra2etFvxXI7JQlLyp_0KxH1jm-eeCqejc,2933
|
91
91
|
edu_rdm_integration/export_data/strategies.py,sha256=ocHskG-x54U-ESX56OlZPgOibkGkuv7VKmK7hVmqj_0,6725
|
@@ -162,9 +162,9 @@ edu_rdm_integration/uploader_log/apps.py,sha256=tYJj4-sDlq8fLOSvw18L_yys7SILpTKW
|
|
162
162
|
edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJLdsGmr3SXwH_U,266
|
163
163
|
edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
|
164
164
|
edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
|
165
|
-
edu_rdm_integration-3.2.
|
166
|
-
edu_rdm_integration-3.2.
|
167
|
-
edu_rdm_integration-3.2.
|
168
|
-
edu_rdm_integration-3.2.
|
169
|
-
edu_rdm_integration-3.2.
|
170
|
-
edu_rdm_integration-3.2.
|
165
|
+
edu_rdm_integration-3.2.3.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
166
|
+
edu_rdm_integration-3.2.3.dist-info/METADATA,sha256=gDOXDSYZhHOgHPJgrkZPEzWC4NFV8oZX_hbIaVoORjk,65697
|
167
|
+
edu_rdm_integration-3.2.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
168
|
+
edu_rdm_integration-3.2.3.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
169
|
+
edu_rdm_integration-3.2.3.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
170
|
+
edu_rdm_integration-3.2.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
{edu_rdm_integration-3.2.1.dist-info → edu_rdm_integration-3.2.3.dist-info}/namespace_packages.txt
RENAMED
File without changes
|
File without changes
|