edu-rdm-integration 0.5.8__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/collect_data/helpers.py +96 -0
- edu_rdm_integration/export_data/helpers.py +95 -0
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/METADATA +47 -10
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/RECORD +8 -6
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/WHEEL +1 -1
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/LICENSE +0 -0
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/namespace_packages.txt +0 -0
- {edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from datetime import (
|
|
2
|
+
date,
|
|
3
|
+
datetime,
|
|
4
|
+
time,
|
|
5
|
+
timedelta,
|
|
6
|
+
)
|
|
7
|
+
from typing import (
|
|
8
|
+
Dict,
|
|
9
|
+
Optional,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
from django.conf import (
|
|
13
|
+
settings,
|
|
14
|
+
)
|
|
15
|
+
from django.db.models import (
|
|
16
|
+
OuterRef,
|
|
17
|
+
Subquery,
|
|
18
|
+
Value,
|
|
19
|
+
)
|
|
20
|
+
from django.db.models.functions import (
|
|
21
|
+
Coalesce,
|
|
22
|
+
)
|
|
23
|
+
from django.db.transaction import (
|
|
24
|
+
atomic,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
from educommon import (
|
|
28
|
+
logger,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from edu_rdm_integration.models import (
|
|
32
|
+
CollectingDataStageStatus,
|
|
33
|
+
CollectingExportedDataStage,
|
|
34
|
+
CollectingDataSubStageStatus,
|
|
35
|
+
CollectingExportedDataSubStage,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@atomic
|
|
40
|
+
def set_failed_status_suspended_collecting_data_stages() -> Dict[str, int]:
|
|
41
|
+
"""Установить статус 'Завершено с ошибками' для зависших этапов и подэтапов сбора.
|
|
42
|
+
|
|
43
|
+
Сборка считается зависшей в случае если за определенное в параметре RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT время,
|
|
44
|
+
отсутствуют изменения в связанных подэтапах. Параметр RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT определяется
|
|
45
|
+
в настройках приложения.
|
|
46
|
+
"""
|
|
47
|
+
changed_status_result = {
|
|
48
|
+
'change_stage_count': 0,
|
|
49
|
+
'change_sub_stage_count': 0,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
current_datetime = datetime.now()
|
|
53
|
+
suspended_time_at = current_datetime - timedelta(minutes=settings.RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT)
|
|
54
|
+
|
|
55
|
+
suspended_stage_ids = set(
|
|
56
|
+
CollectingExportedDataStage.objects.annotate(
|
|
57
|
+
last_sub_stage_started_at=Coalesce(
|
|
58
|
+
Subquery(
|
|
59
|
+
CollectingExportedDataSubStage.objects.filter(
|
|
60
|
+
stage_id=OuterRef('pk')
|
|
61
|
+
).values('started_at').order_by('-started_at')[:1]
|
|
62
|
+
),
|
|
63
|
+
Value(datetime.combine(date.min, time.min))
|
|
64
|
+
)
|
|
65
|
+
).filter(
|
|
66
|
+
last_sub_stage_started_at__lt=suspended_time_at,
|
|
67
|
+
status__in=(
|
|
68
|
+
CollectingDataStageStatus.CREATED.key,
|
|
69
|
+
CollectingDataStageStatus.IN_PROGRESS.key,
|
|
70
|
+
),
|
|
71
|
+
).values_list('pk', flat=True)
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
if suspended_stage_ids:
|
|
75
|
+
logger.info(f'find suspended CollectingExportedDataStage: {", ".join(map(str, suspended_stage_ids))}..')
|
|
76
|
+
|
|
77
|
+
change_stage_count = CollectingExportedDataStage.objects.filter(
|
|
78
|
+
pk__in=suspended_stage_ids,
|
|
79
|
+
).update(
|
|
80
|
+
status=CollectingDataStageStatus.FAILED.key,
|
|
81
|
+
ended_at=current_datetime,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
change_sub_stage_count = CollectingExportedDataSubStage.objects.filter(
|
|
85
|
+
stage_id__in=suspended_stage_ids,
|
|
86
|
+
).update(
|
|
87
|
+
status=CollectingDataSubStageStatus.FAILED.key,
|
|
88
|
+
ended_at=current_datetime,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
changed_status_result.update({
|
|
92
|
+
'change_stage_count': change_stage_count,
|
|
93
|
+
'change_sub_stage_count': change_sub_stage_count,
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
return changed_status_result
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
from datetime import (
|
|
2
|
+
date,
|
|
3
|
+
datetime,
|
|
4
|
+
time,
|
|
5
|
+
timedelta,
|
|
6
|
+
)
|
|
7
|
+
from typing import (
|
|
8
|
+
Dict,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
from django.conf import (
|
|
12
|
+
settings,
|
|
13
|
+
)
|
|
14
|
+
from django.db.models import (
|
|
15
|
+
OuterRef,
|
|
16
|
+
Subquery,
|
|
17
|
+
Value,
|
|
18
|
+
)
|
|
19
|
+
from django.db.models.functions import (
|
|
20
|
+
Coalesce,
|
|
21
|
+
)
|
|
22
|
+
from django.db.transaction import (
|
|
23
|
+
atomic,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
from educommon import (
|
|
27
|
+
logger,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
from edu_rdm_integration.models import (
|
|
31
|
+
ExportingDataStage,
|
|
32
|
+
ExportingDataStageStatus,
|
|
33
|
+
ExportingDataSubStage,
|
|
34
|
+
ExportingDataSubStageStatus,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@atomic
|
|
39
|
+
def set_failed_status_suspended_exporting_data_stages() -> Dict[str, int]:
|
|
40
|
+
"""Установить статус 'Завершено с ошибками' для зависших этапов и подэтапов экспорта.
|
|
41
|
+
|
|
42
|
+
Экспорт считается зависшим в случае если за определенное в параметре RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT время,
|
|
43
|
+
отсутствуют изменения в связанных подэтапах. Параметр RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT определяется
|
|
44
|
+
в настройках приложения.
|
|
45
|
+
"""
|
|
46
|
+
changed_status_result = {
|
|
47
|
+
'change_stage_count': 0,
|
|
48
|
+
'change_sub_stage_count': 0,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
current_datetime = datetime.now()
|
|
52
|
+
suspended_time_at = current_datetime - timedelta(minutes=settings.RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT)
|
|
53
|
+
|
|
54
|
+
suspended_stage_ids = set(
|
|
55
|
+
ExportingDataStage.objects.annotate(
|
|
56
|
+
last_sub_stage_started_at=Coalesce(
|
|
57
|
+
Subquery(
|
|
58
|
+
ExportingDataSubStage.objects.filter(
|
|
59
|
+
stage_id=OuterRef('pk')
|
|
60
|
+
).values('started_at').order_by('-started_at')[:1]
|
|
61
|
+
),
|
|
62
|
+
Value(datetime.combine(date.min, time.min))
|
|
63
|
+
)
|
|
64
|
+
).filter(
|
|
65
|
+
last_sub_stage_started_at__lt=suspended_time_at,
|
|
66
|
+
status__in=(
|
|
67
|
+
ExportingDataStageStatus.CREATED.key,
|
|
68
|
+
ExportingDataStageStatus.IN_PROGRESS.key,
|
|
69
|
+
),
|
|
70
|
+
).values_list('pk', flat=True)
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if suspended_stage_ids:
|
|
74
|
+
logger.info(f'find suspended ExportingDataStage: {", ".join(map(str, suspended_stage_ids))}..')
|
|
75
|
+
|
|
76
|
+
change_stage_count = ExportingDataStage.objects.filter(
|
|
77
|
+
pk__in=suspended_stage_ids,
|
|
78
|
+
).update(
|
|
79
|
+
status=ExportingDataStageStatus.FAILED.key,
|
|
80
|
+
ended_at=current_datetime,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
change_sub_stage_count = ExportingDataSubStage.objects.filter(
|
|
84
|
+
stage_id__in=suspended_stage_ids,
|
|
85
|
+
).update(
|
|
86
|
+
status=ExportingDataSubStageStatus.FAILED.key,
|
|
87
|
+
ended_at=current_datetime,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
changed_status_result.update({
|
|
91
|
+
'change_stage_count': change_stage_count,
|
|
92
|
+
'change_sub_stage_count': change_sub_stage_count,
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
return changed_status_result
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: edu-rdm-integration
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
|
5
5
|
Home-page:
|
|
6
6
|
Download-URL:
|
|
@@ -18,15 +18,15 @@ Classifier: Intended Audience :: Developers
|
|
|
18
18
|
Classifier: Environment :: Console
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist: packaging
|
|
22
|
-
Requires-Dist: pip
|
|
23
|
-
Requires-Dist: setuptools
|
|
24
|
-
Requires-Dist: wheel
|
|
25
|
-
Requires-Dist: transliterate
|
|
26
|
-
Requires-Dist: educommon
|
|
27
|
-
Requires-Dist: function-tools
|
|
28
|
-
Requires-Dist: m3-db-utils
|
|
29
|
-
Requires-Dist: uploader-client
|
|
21
|
+
Requires-Dist: packaging <24,>=21.3
|
|
22
|
+
Requires-Dist: pip <24,>=20.1.1
|
|
23
|
+
Requires-Dist: setuptools <69,>=47.3.1
|
|
24
|
+
Requires-Dist: wheel <0.42,>=0.37.1
|
|
25
|
+
Requires-Dist: transliterate <2
|
|
26
|
+
Requires-Dist: educommon <4,>=3.0.0
|
|
27
|
+
Requires-Dist: function-tools <1,>=0.5.0
|
|
28
|
+
Requires-Dist: m3-db-utils <1,>=0.3.4
|
|
29
|
+
Requires-Dist: uploader-client <1,>=0.1.12
|
|
30
30
|
|
|
31
31
|
# Проект "Интеграция с Региональной витриной данных"
|
|
32
32
|
|
|
@@ -96,6 +96,12 @@ Requires-Dist: uploader-client (<1,>=0.1.12)
|
|
|
96
96
|
RDM_UPLOAD_STATUS_TASK_MINUTE = conf.get('rdm_upload_status_task', 'MINUTE')
|
|
97
97
|
RDM_UPLOAD_STATUS_TASK_HOUR = conf.get('rdm_upload_status_task', 'HOUR')
|
|
98
98
|
RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK = conf.get('rdm_upload_status_task', 'DAY_OF_WEEK')
|
|
99
|
+
|
|
100
|
+
# Настройка запуска периодической задачи поиска зависших этапов экспорта:
|
|
101
|
+
RDM_CHECK_SUSPEND_TASK_MINUTE = conf.get('rdm_check_suspend_task', 'MINUTE')
|
|
102
|
+
RDM_CHECK_SUSPEND_TASK_HOUR = conf.get('rdm_check_suspend_task', 'HOUR')
|
|
103
|
+
RDM_CHECK_SUSPEND_TASK_DAY_OF_WEEK = conf.get('rdm_check_suspend_task', 'DAY_OF_WEEK')
|
|
104
|
+
RDM_CHECK_SUSPEND_TASK_TIMEDELTA = conf.get_int('rdm_check_suspend_task', 'TIMEDELTA')
|
|
99
105
|
|
|
100
106
|
# Загрузка данных в Региональную витрину данных (РВД)
|
|
101
107
|
# Адрес витрины (schema://host:port)
|
|
@@ -142,6 +148,14 @@ MINUTE=*/2
|
|
|
142
148
|
HOUR=*
|
|
143
149
|
DAY_OF_WEEK=*
|
|
144
150
|
|
|
151
|
+
# Настройка запуска периодической задачи поиска зависших этапов экспорта
|
|
152
|
+
[rdm_check_suspend_task]
|
|
153
|
+
MINUTE=*/10
|
|
154
|
+
HOUR=*
|
|
155
|
+
DAY_OF_WEEK=*
|
|
156
|
+
# Дельта для определения зависшего подэтапа, мин
|
|
157
|
+
STAGE_TIMEOUT=120
|
|
158
|
+
|
|
145
159
|
[uploader_client]
|
|
146
160
|
# Адрес витрины
|
|
147
161
|
URL = http://localhost:8090
|
|
@@ -176,6 +190,7 @@ ENABLE_REQUEST_EMULATION = True
|
|
|
176
190
|
| RDM_UPLOAD_STATUS_TASK_MINUTE | Настройка запуска периодической задачи статуса загрузки данных в витрину. Минута | '*/30' |
|
|
177
191
|
| RDM_UPLOAD_STATUS_TASK_HOUR | Настройка запуска периодической задачи статуса загрузки данных в витрину. Час | '*' |
|
|
178
192
|
| RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK | Настройка запуска периодической задачи статуса загрузки данных в витрину. День недели | '*' |
|
|
193
|
+
| RDM_CHECK_SUSPEND_TASK_STAGE_TIMEOUT | Дельта для определения зависшего подэтапа. Минута | 120 |
|
|
179
194
|
|
|
180
195
|
|
|
181
196
|
## Сборка и распространение
|
|
@@ -220,6 +235,28 @@ ENABLE_REQUEST_EMULATION = True
|
|
|
220
235
|
### Удалено
|
|
221
236
|
|
|
222
237
|
|
|
238
|
+
## [0.6.0] - 2023-09-29
|
|
239
|
+
|
|
240
|
+
Добавлена функция set_failed_status_suspended_exporting_data_stages для определения и перевода зависших этапов/подэтапов экспорта в статус (FAILED).
|
|
241
|
+
Добавлена функция set_failed_status_suspended_collecting_data_stages для определения и перевода зависших этапов/подэтапов сбора в статус (FAILED).
|
|
242
|
+
|
|
243
|
+
### Добавлено
|
|
244
|
+
|
|
245
|
+
- [EDUSCHL-20487](https://jira.bars.group/browse/EDUSCHL-20487)
|
|
246
|
+
MINOR Добавлена функция ```set_failed_status_suspended_exporting_data_stages```.
|
|
247
|
+
|
|
248
|
+
- [EDUSCHL-20487](https://jira.bars.group/browse/EDUSCHL-20487)
|
|
249
|
+
MINOR Добавлена функция ```set_failed_status_suspended_collecting_data_stages```.
|
|
250
|
+
|
|
251
|
+
## [0.5.9] - 2023-09-25
|
|
252
|
+
|
|
253
|
+
Повышена версия wheel
|
|
254
|
+
|
|
255
|
+
### Изменено
|
|
256
|
+
|
|
257
|
+
- [EDUCLLG-7939](https://jira.bars.group/browse/EDUCLLG-7939)
|
|
258
|
+
PATCH - Повышена версия wheel
|
|
259
|
+
|
|
223
260
|
## [0.5.8] - 2023-09-13
|
|
224
261
|
|
|
225
262
|
Исправлена ошибка добавления в описание асинхронной задачи списка выгруженных сущностей
|
|
@@ -29,6 +29,7 @@ edu_rdm_integration/adapters/validators.py,sha256=LJWnCY8PtXDOj-fm3fBWjQYsHsSLfy
|
|
|
29
29
|
edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
30
30
|
edu_rdm_integration/collect_data/collect.py,sha256=3sicEiX9v_EQtd5vFOyclSV-kYNV7FNxx09DdyIWIAk,12843
|
|
31
31
|
edu_rdm_integration/collect_data/generators.py,sha256=wnwxP4ljVFlLJ-TvRmIlaQuY7tWAsQKV4LX4D0zt3Hw,8685
|
|
32
|
+
edu_rdm_integration/collect_data/helpers.py,sha256=EftJ8R-tn1r7Y_P6tGZmqxJ9-g89m12p9kA8NYmwwbQ,3009
|
|
32
33
|
edu_rdm_integration/collect_data/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
34
|
edu_rdm_integration/collect_data/base/caches.py,sha256=3BaJxYBk9fi0aiAVzym-Jz8aNP1eSOqh4Y8OVw1HnSg,763
|
|
34
35
|
edu_rdm_integration/collect_data/base/functions.py,sha256=HT23EyiD-H50p4NLx2_LtioktTHHFVLRmAgWdbuHErw,2379
|
|
@@ -71,6 +72,7 @@ edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
|
|
|
71
72
|
edu_rdm_integration/export_data/consts.py,sha256=Z6Uho8oViMc0t1tyQFZfrPFBwPx6KnhUf_iZ0qgob8o,16
|
|
72
73
|
edu_rdm_integration/export_data/export.py,sha256=jqJYXg-Vm_IwkOxposZKk5QY8d4aBhNmGJOw4PRFJjk,14935
|
|
73
74
|
edu_rdm_integration/export_data/generators.py,sha256=huonSRK5arfBzU09BnLAHC7AsSFYU0sCz5utCtejdJk,3950
|
|
75
|
+
edu_rdm_integration/export_data/helpers.py,sha256=hU346RmQ17Ra2etFvxXI7JQlLyp_0KxH1jm-eeCqejc,2933
|
|
74
76
|
edu_rdm_integration/export_data/strategies.py,sha256=NcI2VsDfAmm9k-Nul6LZkSxruCmo7B-XH7wzgW6-vfE,6718
|
|
75
77
|
edu_rdm_integration/export_data/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
76
78
|
edu_rdm_integration/export_data/base/caches.py,sha256=dj8LGB7TMtX3cSfp-1UB3dnzcTGMt8aLtOn0YdNPsyQ,1541
|
|
@@ -95,9 +97,9 @@ edu_rdm_integration/management/commands/export_entities_data.py,sha256=Mas1zwsH-
|
|
|
95
97
|
edu_rdm_integration/migrations/0001_initial.py,sha256=toNuYoHZePe5wJ6AKEW9oPOdt2OefmxDEDDJGYQIrFk,18719
|
|
96
98
|
edu_rdm_integration/migrations/0002_init_data_uploadstatus.py,sha256=kht966YNuDbC3qTGrcWswJPsVuAtNO59Ck15G2eS2bU,944
|
|
97
99
|
edu_rdm_integration/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
98
|
-
edu_rdm_integration-0.
|
|
99
|
-
edu_rdm_integration-0.
|
|
100
|
-
edu_rdm_integration-0.
|
|
101
|
-
edu_rdm_integration-0.
|
|
102
|
-
edu_rdm_integration-0.
|
|
103
|
-
edu_rdm_integration-0.
|
|
100
|
+
edu_rdm_integration-0.6.0.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
|
101
|
+
edu_rdm_integration-0.6.0.dist-info/METADATA,sha256=3wZjBeDfR8DboAdXdckQL-l1zhYx5Py_MCohueQfKgM,29208
|
|
102
|
+
edu_rdm_integration-0.6.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
103
|
+
edu_rdm_integration-0.6.0.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
104
|
+
edu_rdm_integration-0.6.0.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
|
105
|
+
edu_rdm_integration-0.6.0.dist-info/RECORD,,
|
|
File without changes
|
{edu_rdm_integration-0.5.8.dist-info → edu_rdm_integration-0.6.0.dist-info}/namespace_packages.txt
RENAMED
|
File without changes
|
|
File without changes
|