edu-rdm-integration 3.2.7__py3-none-any.whl → 3.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/app_settings.py +5 -0
- edu_rdm_integration/apps.py +1 -1
- edu_rdm_integration/collect_data/collect.py +4 -54
- edu_rdm_integration/consts.py +0 -1
- edu_rdm_integration/export_data/base/functions.py +15 -108
- edu_rdm_integration/export_data/consts.py +5 -0
- edu_rdm_integration/export_data/dataclasses.py +11 -0
- edu_rdm_integration/export_data/export_manger.py +246 -0
- edu_rdm_integration/export_data/queue.py +172 -0
- edu_rdm_integration/helpers.py +19 -2
- edu_rdm_integration/management/general.py +0 -12
- edu_rdm_integration/migrations/0009_auto_20240522_1619.py +25 -0
- edu_rdm_integration/migrations/{0009_transferredentity_export_enabled.py → 0010_transferredentity_export_enabled.py} +2 -2
- edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py +30 -0
- edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py +21 -0
- edu_rdm_integration/migrations/0013_set_attachment_size.py +48 -0
- edu_rdm_integration/models.py +35 -4
- edu_rdm_integration/redis_cache.py +52 -0
- edu_rdm_integration/tasks.py +69 -33
- edu_rdm_integration/utils.py +1 -0
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/METADATA +59 -1
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/RECORD +26 -18
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/LICENSE +0 -0
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/namespace_packages.txt +0 -0
- {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,172 @@
|
|
1
|
+
import json
|
2
|
+
from abc import (
|
3
|
+
ABC,
|
4
|
+
abstractmethod,
|
5
|
+
)
|
6
|
+
from typing import (
|
7
|
+
Any,
|
8
|
+
Dict,
|
9
|
+
List,
|
10
|
+
Tuple,
|
11
|
+
Union,
|
12
|
+
)
|
13
|
+
|
14
|
+
from django.conf import (
|
15
|
+
settings,
|
16
|
+
)
|
17
|
+
from redis import (
|
18
|
+
Redis,
|
19
|
+
)
|
20
|
+
|
21
|
+
from edu_rdm_integration.export_data.consts import (
|
22
|
+
REDIS_QUEUE_KEY_DELIMITER,
|
23
|
+
)
|
24
|
+
from edu_rdm_integration.export_data.dataclasses import (
|
25
|
+
UploadFile,
|
26
|
+
)
|
27
|
+
from edu_rdm_integration.redis_cache import (
|
28
|
+
as_text,
|
29
|
+
get_redis_version,
|
30
|
+
)
|
31
|
+
|
32
|
+
|
33
|
+
class Queue(ABC):
|
34
|
+
"""Интерфейс очереди."""
|
35
|
+
queue_key: str = ''
|
36
|
+
|
37
|
+
@property
|
38
|
+
@abstractmethod
|
39
|
+
def count(self) -> int:
|
40
|
+
"""Возвращает кол-во всех элементов в очереди."""
|
41
|
+
|
42
|
+
def is_empty(self) -> bool:
|
43
|
+
"""Возвращает признак пустая ли очередь."""
|
44
|
+
return self.count == 0
|
45
|
+
|
46
|
+
@abstractmethod
|
47
|
+
def enqueue(self, *args, **kwargs) -> None:
|
48
|
+
"""Поместить в очередь."""
|
49
|
+
|
50
|
+
@abstractmethod
|
51
|
+
def dequeue(self) -> Any:
|
52
|
+
"""Вернуть из очереди."""
|
53
|
+
|
54
|
+
@abstractmethod
|
55
|
+
def clear(self) -> None:
|
56
|
+
"""Очистить очередь."""
|
57
|
+
|
58
|
+
@abstractmethod
|
59
|
+
def delete_from_queue(self, *args, **kwargs) -> None:
|
60
|
+
"""Удалить из очереди конкретное значение."""
|
61
|
+
|
62
|
+
|
63
|
+
class RdmRedisSubStageAttachmentQueue(Queue):
|
64
|
+
"""Очередь файлов и подэтапов.
|
65
|
+
|
66
|
+
Данные хранятся следующим образом:
|
67
|
+
- Подэтапы с сущностями (строка вида "407-MARKS" (sub_stage_id-entity)) хранятся в упорядоченном множестве Redis
|
68
|
+
(Sorted Set in Redis)
|
69
|
+
- Информация по файлам стандартно по ключу - ключом выступает sub_stage_id
|
70
|
+
"""
|
71
|
+
queue_key = 'rdm:export_sub_stage_ids_queue'
|
72
|
+
prefix = 'rdm:'
|
73
|
+
|
74
|
+
def __init__(self, *args, **kwargs):
|
75
|
+
"""Инициализация объекта очереди Queue."""
|
76
|
+
super().__init__(*args, **kwargs)
|
77
|
+
|
78
|
+
self.connection = Redis(
|
79
|
+
host=settings.RDM_REDIS_HOST,
|
80
|
+
port=settings.RDM_REDIS_PORT,
|
81
|
+
db=settings.RDM_REDIS_DB,
|
82
|
+
password=settings.RDM_REDIS_PASSWORD
|
83
|
+
)
|
84
|
+
|
85
|
+
def _make_key(self, key: Union[int, str]) -> str:
|
86
|
+
"""Формирование ключа."""
|
87
|
+
return f'{self.prefix}{key}'
|
88
|
+
|
89
|
+
@property
|
90
|
+
def count(self) -> int:
|
91
|
+
"""Возвращает количество подэтапов в очереди."""
|
92
|
+
return self.connection.zcard(self.queue_key)
|
93
|
+
|
94
|
+
def enqueue(self, stage_id, entity_name: str, attachmets: List[UploadFile]) -> None:
|
95
|
+
"""Помещение в очередь.
|
96
|
+
|
97
|
+
Подэтап попадает в упорядоченную очередь.
|
98
|
+
"""
|
99
|
+
stage_info = f'{stage_id}{REDIS_QUEUE_KEY_DELIMITER}{entity_name}'
|
100
|
+
pipe = self.connection.pipeline()
|
101
|
+
# Упрядочиваем подэтапы
|
102
|
+
pipe.zadd(self.queue_key, {stage_info: stage_id})
|
103
|
+
pipe.set(self._make_key(stage_id), json.dumps(attachmets))
|
104
|
+
pipe.execute()
|
105
|
+
|
106
|
+
def dequeue_sub_stage_attachments(self, sub_stage_id: int) -> List[UploadFile]:
|
107
|
+
"""Возвращает файлы подэтапа из кеша."""
|
108
|
+
result = []
|
109
|
+
attachments = self.connection.get(self._make_key(sub_stage_id))
|
110
|
+
attachments = json.loads(attachments) if attachments else ()
|
111
|
+
for attachment in attachments:
|
112
|
+
result.append(UploadFile(*attachment))
|
113
|
+
|
114
|
+
return result
|
115
|
+
|
116
|
+
def dequeue(self) -> Dict[Tuple[Any, Any], List[UploadFile]]:
|
117
|
+
"""Возвращает подэтапы из очереди - берется вся очередь без ограничений."""
|
118
|
+
upload_files = {}
|
119
|
+
exported_sub_stages = self.connection.zrange(self.queue_key, 0, -1)
|
120
|
+
for sub_stage in exported_sub_stages:
|
121
|
+
sub_stage_info = as_text(sub_stage)
|
122
|
+
sub_stage_id, sub_stage_entity = sub_stage_info.split(REDIS_QUEUE_KEY_DELIMITER)
|
123
|
+
upload_files[(sub_stage_id, sub_stage_entity)] = self.dequeue_sub_stage_attachments(sub_stage_id)
|
124
|
+
|
125
|
+
return upload_files
|
126
|
+
|
127
|
+
def delete_sub_stages_attachments(self, sub_stage_id: int) -> None:
|
128
|
+
"""Удаляет информацию о файлах из кеша."""
|
129
|
+
self.connection.delete(self._make_key(sub_stage_id))
|
130
|
+
|
131
|
+
def delete_sub_stages_from_queue(self, sub_stage_id: int, entity_name: str) -> None:
|
132
|
+
"""Удаляет подэтап из очереди."""
|
133
|
+
self.connection.zrem(self.queue_key, f'{sub_stage_id}{REDIS_QUEUE_KEY_DELIMITER}{entity_name}')
|
134
|
+
|
135
|
+
def delete_from_queue(self, sub_stage_id: int, entity_name: str) -> None:
|
136
|
+
"""Удаление элемента из очереди."""
|
137
|
+
self.delete_sub_stages_attachments(sub_stage_id)
|
138
|
+
self.delete_sub_stages_from_queue(sub_stage_id, entity_name)
|
139
|
+
|
140
|
+
def clear(self) -> int:
|
141
|
+
"""Удаление из очереди всех подэтапов."""
|
142
|
+
script = """
|
143
|
+
local prefix = "{0}"
|
144
|
+
local q = KEYS[1]
|
145
|
+
local count = 0
|
146
|
+
while true do
|
147
|
+
local stage = redis.call("zpopmin", q)
|
148
|
+
local stage_id = stage[2]
|
149
|
+
|
150
|
+
if stage_id == nil then
|
151
|
+
break
|
152
|
+
end
|
153
|
+
redis.call("del", prefix..stage_id)
|
154
|
+
count = count + 1
|
155
|
+
end
|
156
|
+
return count
|
157
|
+
""".format(self.prefix).encode('utf-8')
|
158
|
+
|
159
|
+
script = self.connection.register_script(script)
|
160
|
+
|
161
|
+
return script(keys=[self.queue_key])
|
162
|
+
|
163
|
+
@property
|
164
|
+
def connection_info(self) -> str:
|
165
|
+
"""Информация об используемом соединении."""
|
166
|
+
version = '.'.join(map(str, get_redis_version(self.connection)))
|
167
|
+
kwargs = self.connection.connection_pool.connection_kwargs
|
168
|
+
host = kwargs['host']
|
169
|
+
port = kwargs['port']
|
170
|
+
db = kwargs['db']
|
171
|
+
|
172
|
+
return f'Redis {version} on {host}:{port}/{db}'
|
edu_rdm_integration/helpers.py
CHANGED
@@ -40,7 +40,6 @@ from educommon import (
|
|
40
40
|
from uploader_client.adapters import (
|
41
41
|
adapter,
|
42
42
|
)
|
43
|
-
|
44
43
|
from edu_rdm_integration.collect_and_export_data.models import (
|
45
44
|
EduRdmCollectDataCommandProgress,
|
46
45
|
EduRdmExportDataCommandProgress,
|
@@ -51,6 +50,9 @@ from edu_rdm_integration.enums import (
|
|
51
50
|
from edu_rdm_integration.export_data.base.requests import (
|
52
51
|
RegionalDataMartStatusRequest,
|
53
52
|
)
|
53
|
+
from edu_rdm_integration.export_data.consts import (
|
54
|
+
TOTAL_ATTACHMENTS_SIZE_KEY,
|
55
|
+
)
|
54
56
|
from edu_rdm_integration.models import (
|
55
57
|
DataMartRequestStatus,
|
56
58
|
ExportingDataStage,
|
@@ -60,6 +62,9 @@ from edu_rdm_integration.models import (
|
|
60
62
|
UploadStatusRequestLog,
|
61
63
|
ExportingDataStageStatus,
|
62
64
|
)
|
65
|
+
from edu_rdm_integration.redis_cache import (
|
66
|
+
AbstractCache,
|
67
|
+
)
|
63
68
|
|
64
69
|
if TYPE_CHECKING:
|
65
70
|
from datetime import (
|
@@ -80,8 +85,9 @@ if TYPE_CHECKING:
|
|
80
85
|
class UploadStatusHelper:
|
81
86
|
"""Хелпер проверки статуса загрузки данных в витрину."""
|
82
87
|
|
83
|
-
def __init__(self, in_progress_uploads: QuerySet) -> None:
|
88
|
+
def __init__(self, in_progress_uploads: QuerySet, cache: AbstractCache) -> None:
|
84
89
|
self._in_progress_uploads = in_progress_uploads
|
90
|
+
self.cache = cache
|
85
91
|
|
86
92
|
def run(self, thread_count: int = 1) -> None:
|
87
93
|
"""Запускает проверки статусов."""
|
@@ -167,6 +173,17 @@ class UploadStatusHelper:
|
|
167
173
|
"""Обрабатывает запись загрузки данных в витрину."""
|
168
174
|
response, log_entry = self.send_upload_status_request(upload.request_id)
|
169
175
|
self.update_upload_status(upload, response, log_entry)
|
176
|
+
# Обновим размер файлов в кеш (с блокировкой на время обновления)
|
177
|
+
with self.cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
|
178
|
+
queue_total_file_size = self.cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
|
179
|
+
if queue_total_file_size:
|
180
|
+
queue_total_file_size -= upload.attachment.attachment_size
|
181
|
+
if queue_total_file_size > 0:
|
182
|
+
self.cache.set(
|
183
|
+
TOTAL_ATTACHMENTS_SIZE_KEY,
|
184
|
+
queue_total_file_size,
|
185
|
+
timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
|
186
|
+
)
|
170
187
|
|
171
188
|
|
172
189
|
class Graph:
|
@@ -205,18 +205,6 @@ class BaseCollectModelsDataByGeneratingLogsCommand(BaseCollectModelDataCommand):
|
|
205
205
|
"""
|
206
206
|
super().add_arguments(parser=parser)
|
207
207
|
|
208
|
-
parser.add_argument(
|
209
|
-
'--logs_sub_period_days',
|
210
|
-
action='store',
|
211
|
-
dest='logs_sub_period_days',
|
212
|
-
type=int,
|
213
|
-
default=0,
|
214
|
-
help=(
|
215
|
-
'Размер подпериодов, на которые будет разбит основной период, в днях. По умолчанию, '
|
216
|
-
'0 - разбиение на подпериоды отключено.'
|
217
|
-
),
|
218
|
-
)
|
219
|
-
|
220
208
|
parser.add_argument(
|
221
209
|
'--institute_ids',
|
222
210
|
action='store',
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# Generated by Django 3.1.14 on 2024-05-22 16:19
|
2
|
+
|
3
|
+
from django.db import (
|
4
|
+
migrations,
|
5
|
+
)
|
6
|
+
|
7
|
+
|
8
|
+
def add_exporting_data_sub_stage_status(apps, schema_editor):
|
9
|
+
"""Добавление нового статуса этапу выгрузки данных."""
|
10
|
+
ExportingDataSubStageStatus = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageStatus') # noqa: N806
|
11
|
+
ExportingDataSubStageStatus.objects.get_or_create(
|
12
|
+
key='READY_FOR_EXPORT',
|
13
|
+
title='Готов к выгрузке'
|
14
|
+
)
|
15
|
+
|
16
|
+
|
17
|
+
class Migration(migrations.Migration):
|
18
|
+
|
19
|
+
dependencies = [
|
20
|
+
('edu_rdm_integration', '0008_transferredentity'),
|
21
|
+
]
|
22
|
+
|
23
|
+
operations = [
|
24
|
+
migrations.RunPython(add_exporting_data_sub_stage_status, reverse_code=migrations.RunPython.noop),
|
25
|
+
]
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Generated by Django 3.1.14 on 2024-
|
1
|
+
# Generated by Django 3.1.14 on 2024-08-19 14:23
|
2
2
|
|
3
3
|
from django.db import (
|
4
4
|
migrations,
|
@@ -9,7 +9,7 @@ from django.db import (
|
|
9
9
|
class Migration(migrations.Migration):
|
10
10
|
|
11
11
|
dependencies = [
|
12
|
-
('edu_rdm_integration', '
|
12
|
+
('edu_rdm_integration', '0009_auto_20240522_1619'),
|
13
13
|
]
|
14
14
|
|
15
15
|
operations = [
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# Generated by Django 3.1.14 on 2024-08-19 14:24
|
2
|
+
|
3
|
+
import django.db.models.deletion
|
4
|
+
from django.db import (
|
5
|
+
migrations,
|
6
|
+
models,
|
7
|
+
)
|
8
|
+
|
9
|
+
|
10
|
+
class Migration(migrations.Migration):
|
11
|
+
|
12
|
+
dependencies = [
|
13
|
+
('edu_rdm_integration', '0010_transferredentity_export_enabled'),
|
14
|
+
]
|
15
|
+
|
16
|
+
operations = [
|
17
|
+
migrations.CreateModel(
|
18
|
+
name='ExportingDataSubStageEntity',
|
19
|
+
fields=[
|
20
|
+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
21
|
+
('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.regionaldatamartentityenum', verbose_name='Сущность РВД')),
|
22
|
+
('exporting_data_sub_stage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='edu_rdm_integration.exportingdatasubstage', verbose_name='Подэтап выгрузки данных')),
|
23
|
+
],
|
24
|
+
options={
|
25
|
+
'verbose_name': 'Связь сущности и подэтапа выгрузки',
|
26
|
+
'verbose_name_plural': 'Связи сущности и подэтапа выгрузки',
|
27
|
+
'db_table': 'rdm_exporting_data_sub_stage_entity',
|
28
|
+
},
|
29
|
+
),
|
30
|
+
]
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# Generated by Django 3.1.14 on 2024-08-19 14:24
|
2
|
+
|
3
|
+
from django.db import (
|
4
|
+
migrations,
|
5
|
+
models,
|
6
|
+
)
|
7
|
+
|
8
|
+
|
9
|
+
class Migration(migrations.Migration):
|
10
|
+
|
11
|
+
dependencies = [
|
12
|
+
('edu_rdm_integration', '0011_exportingdatasubstageentity'),
|
13
|
+
]
|
14
|
+
|
15
|
+
operations = [
|
16
|
+
migrations.AddField(
|
17
|
+
model_name='exportingdatasubstageattachment',
|
18
|
+
name='attachment_size',
|
19
|
+
field=models.PositiveIntegerField(null=True, verbose_name='Размер файла (байт)'),
|
20
|
+
),
|
21
|
+
]
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# Generated by Django 3.1.14 on 2024-08-19 14:56
|
2
|
+
|
3
|
+
from django.db import (
|
4
|
+
migrations,
|
5
|
+
)
|
6
|
+
|
7
|
+
from edu_rdm_integration.enums import (
|
8
|
+
FileUploadStatusEnum,
|
9
|
+
)
|
10
|
+
|
11
|
+
|
12
|
+
# Значение размера файла по умолчанию (если файл не найден)
|
13
|
+
ATTACHMENT_SIZES = 10_485_760
|
14
|
+
|
15
|
+
|
16
|
+
def set_attachment_size(apps, schema_editor):
|
17
|
+
"""Установка размера файла по умолчанию."""
|
18
|
+
ExportingDataSubStageAttachment = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageAttachment') # noqa: N806
|
19
|
+
ExportingDataSubStageUploaderClientLog = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageUploaderClientLog') # noqa: N806
|
20
|
+
|
21
|
+
attachment_ids = ExportingDataSubStageUploaderClientLog.objects.filter(
|
22
|
+
file_upload_status=FileUploadStatusEnum.IN_PROGRESS
|
23
|
+
).values_list('attachment_id', flat=True)
|
24
|
+
|
25
|
+
sub_stage_with_deleted_attachments = []
|
26
|
+
for sub_stage_attachment in ExportingDataSubStageAttachment.objects.filter(
|
27
|
+
attachment_size__isnull=True,
|
28
|
+
id__in=attachment_ids
|
29
|
+
):
|
30
|
+
if sub_stage_attachment.attachment.field.storage.exists(sub_stage_attachment.attachment.name):
|
31
|
+
sub_stage_attachment.attachment_size = sub_stage_attachment.attachment.size
|
32
|
+
else:
|
33
|
+
sub_stage_attachment.attachment_size = ATTACHMENT_SIZES
|
34
|
+
sub_stage_with_deleted_attachments.append(sub_stage_attachment)
|
35
|
+
|
36
|
+
ExportingDataSubStageAttachment.objects.bulk_update(sub_stage_with_deleted_attachments, ['attachment_size'])
|
37
|
+
|
38
|
+
|
39
|
+
class Migration(migrations.Migration):
|
40
|
+
|
41
|
+
dependencies = [
|
42
|
+
('edu_rdm_integration', '0012_exportingdatasubstageattachment_attachment_size'),
|
43
|
+
]
|
44
|
+
|
45
|
+
operations = [
|
46
|
+
migrations.RunPython(set_attachment_size, reverse_code=migrations.RunPython.noop)
|
47
|
+
]
|
48
|
+
|
edu_rdm_integration/models.py
CHANGED
@@ -25,6 +25,7 @@ from django.db.models import (
|
|
25
25
|
ForeignKey,
|
26
26
|
Manager,
|
27
27
|
OneToOneField,
|
28
|
+
PositiveIntegerField,
|
28
29
|
SmallIntegerField,
|
29
30
|
UUIDField,
|
30
31
|
)
|
@@ -347,9 +348,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
347
348
|
|
348
349
|
|
349
350
|
class ExportingDataSubStageStatus(TitledModelEnum):
|
350
|
-
"""
|
351
|
-
Модель-перечисление статусов этапа выгрузки данных
|
352
|
-
"""
|
351
|
+
"""Модель-перечисление статусов этапа выгрузки данных."""
|
353
352
|
|
354
353
|
CREATED = ModelEnumValue(
|
355
354
|
title='Создан',
|
@@ -366,6 +365,9 @@ class ExportingDataSubStageStatus(TitledModelEnum):
|
|
366
365
|
FINISHED = ModelEnumValue(
|
367
366
|
title='Завершен',
|
368
367
|
)
|
368
|
+
READY_FOR_EXPORT = ModelEnumValue(
|
369
|
+
title='Готов к выгрузке',
|
370
|
+
)
|
369
371
|
|
370
372
|
class Meta:
|
371
373
|
db_table = 'rdm_exporting_data_sub_stage_status'
|
@@ -423,7 +425,11 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
423
425
|
|
424
426
|
def save(self, *args, **kwargs):
|
425
427
|
if (
|
426
|
-
self.status_id in
|
428
|
+
self.status_id in {
|
429
|
+
ExportingDataSubStageStatus.FAILED.key,
|
430
|
+
ExportingDataSubStageStatus.FINISHED.key,
|
431
|
+
ExportingDataSubStageStatus.READY_FOR_EXPORT.key,
|
432
|
+
}
|
427
433
|
and not self.ended_at
|
428
434
|
):
|
429
435
|
self.ended_at = datetime.now()
|
@@ -469,6 +475,10 @@ class ExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
|
|
469
475
|
null=True,
|
470
476
|
blank=True,
|
471
477
|
)
|
478
|
+
attachment_size = PositiveIntegerField(
|
479
|
+
null=True,
|
480
|
+
verbose_name='Размер файла (байт)'
|
481
|
+
)
|
472
482
|
|
473
483
|
class Meta:
|
474
484
|
db_table = 'rdm_exporting_data_sub_stage_attachment'
|
@@ -922,3 +932,24 @@ class TransferredEntity(BaseObjectModel):
|
|
922
932
|
@json_encode
|
923
933
|
def no_export(self):
|
924
934
|
return 'Нет' if self.export_enabled else 'Да'
|
935
|
+
|
936
|
+
|
937
|
+
class ExportingDataSubStageEntity(BaseObjectModel):
|
938
|
+
"""Модель связи сущности и подэтапа выгрузки."""
|
939
|
+
|
940
|
+
entity = ForeignKey(
|
941
|
+
to=RegionalDataMartEntityEnum,
|
942
|
+
verbose_name='Сущность РВД',
|
943
|
+
on_delete=PROTECT,
|
944
|
+
)
|
945
|
+
|
946
|
+
exporting_data_sub_stage = ForeignKey(
|
947
|
+
to=ExportingDataSubStage,
|
948
|
+
verbose_name='Подэтап выгрузки данных',
|
949
|
+
on_delete=CASCADE,
|
950
|
+
)
|
951
|
+
|
952
|
+
class Meta:
|
953
|
+
db_table = 'rdm_exporting_data_sub_stage_entity'
|
954
|
+
verbose_name = 'Связь сущности и подэтапа выгрузки'
|
955
|
+
verbose_name_plural = 'Связи сущности и подэтапа выгрузки'
|
@@ -0,0 +1,52 @@
|
|
1
|
+
from abc import (
|
2
|
+
ABCMeta,
|
3
|
+
abstractmethod,
|
4
|
+
)
|
5
|
+
from typing import (
|
6
|
+
Tuple,
|
7
|
+
Union,
|
8
|
+
)
|
9
|
+
|
10
|
+
from redis import (
|
11
|
+
Redis,
|
12
|
+
ResponseError,
|
13
|
+
)
|
14
|
+
|
15
|
+
|
16
|
+
def get_redis_version(connection: 'Redis') -> Tuple[int, int, int]:
|
17
|
+
"""Возвращает кортеж с версией сервера Redis."""
|
18
|
+
try:
|
19
|
+
version = getattr(connection, '__redis_server_version', None)
|
20
|
+
if not version:
|
21
|
+
version = tuple([int(n) for n in connection.info('server')['redis_version'].split('.')[:3]])
|
22
|
+
setattr(connection, '__redis_server_version', version)
|
23
|
+
except ResponseError:
|
24
|
+
version = (0, 0, 0)
|
25
|
+
|
26
|
+
return version
|
27
|
+
|
28
|
+
|
29
|
+
def as_text(v: Union[bytes, str]) -> str:
|
30
|
+
"""Конвертирует последовательность байт в строку."""
|
31
|
+
if isinstance(v, bytes):
|
32
|
+
return v.decode('utf-8')
|
33
|
+
elif isinstance(v, str):
|
34
|
+
return v
|
35
|
+
else:
|
36
|
+
raise ValueError('Неизвестный тип %r' % type(v))
|
37
|
+
|
38
|
+
|
39
|
+
class AbstractCache(metaclass=ABCMeta):
|
40
|
+
"""Абстрактный интерфейс для кеша отправки."""
|
41
|
+
|
42
|
+
@abstractmethod
|
43
|
+
def get(self, key, default=None, **kwargs):
|
44
|
+
"""Возвращает значение из кеша по ключу."""
|
45
|
+
|
46
|
+
@abstractmethod
|
47
|
+
def set(self, key, value, timeout=None, **kwargs):
|
48
|
+
"""Сохраняет значение в кеш по ключу."""
|
49
|
+
|
50
|
+
@abstractmethod
|
51
|
+
def lock(self, name, timeout=None, **kwargs):
|
52
|
+
"""Захватывает блокировку."""
|