edu-rdm-integration 3.2.7__py3-none-any.whl → 3.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. edu_rdm_integration/app_settings.py +5 -0
  2. edu_rdm_integration/apps.py +1 -1
  3. edu_rdm_integration/collect_data/collect.py +4 -54
  4. edu_rdm_integration/consts.py +0 -1
  5. edu_rdm_integration/export_data/base/functions.py +15 -108
  6. edu_rdm_integration/export_data/consts.py +5 -0
  7. edu_rdm_integration/export_data/dataclasses.py +11 -0
  8. edu_rdm_integration/export_data/export_manger.py +246 -0
  9. edu_rdm_integration/export_data/queue.py +172 -0
  10. edu_rdm_integration/helpers.py +19 -2
  11. edu_rdm_integration/management/general.py +0 -12
  12. edu_rdm_integration/migrations/0009_auto_20240522_1619.py +25 -0
  13. edu_rdm_integration/migrations/{0009_transferredentity_export_enabled.py → 0010_transferredentity_export_enabled.py} +2 -2
  14. edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py +30 -0
  15. edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py +21 -0
  16. edu_rdm_integration/migrations/0013_set_attachment_size.py +48 -0
  17. edu_rdm_integration/models.py +35 -4
  18. edu_rdm_integration/redis_cache.py +52 -0
  19. edu_rdm_integration/tasks.py +69 -33
  20. edu_rdm_integration/utils.py +1 -0
  21. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/METADATA +59 -1
  22. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/RECORD +26 -18
  23. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/LICENSE +0 -0
  24. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/WHEEL +0 -0
  25. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/namespace_packages.txt +0 -0
  26. {edu_rdm_integration-3.2.7.dist-info → edu_rdm_integration-3.3.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,172 @@
1
+ import json
2
+ from abc import (
3
+ ABC,
4
+ abstractmethod,
5
+ )
6
+ from typing import (
7
+ Any,
8
+ Dict,
9
+ List,
10
+ Tuple,
11
+ Union,
12
+ )
13
+
14
+ from django.conf import (
15
+ settings,
16
+ )
17
+ from redis import (
18
+ Redis,
19
+ )
20
+
21
+ from edu_rdm_integration.export_data.consts import (
22
+ REDIS_QUEUE_KEY_DELIMITER,
23
+ )
24
+ from edu_rdm_integration.export_data.dataclasses import (
25
+ UploadFile,
26
+ )
27
+ from edu_rdm_integration.redis_cache import (
28
+ as_text,
29
+ get_redis_version,
30
+ )
31
+
32
+
33
+ class Queue(ABC):
34
+ """Интерфейс очереди."""
35
+ queue_key: str = ''
36
+
37
+ @property
38
+ @abstractmethod
39
+ def count(self) -> int:
40
+ """Возвращает кол-во всех элементов в очереди."""
41
+
42
+ def is_empty(self) -> bool:
43
+ """Возвращает признак пустая ли очередь."""
44
+ return self.count == 0
45
+
46
+ @abstractmethod
47
+ def enqueue(self, *args, **kwargs) -> None:
48
+ """Поместить в очередь."""
49
+
50
+ @abstractmethod
51
+ def dequeue(self) -> Any:
52
+ """Вернуть из очереди."""
53
+
54
+ @abstractmethod
55
+ def clear(self) -> None:
56
+ """Очистить очередь."""
57
+
58
+ @abstractmethod
59
+ def delete_from_queue(self, *args, **kwargs) -> None:
60
+ """Удалить из очереди конкретное значение."""
61
+
62
+
63
+ class RdmRedisSubStageAttachmentQueue(Queue):
64
+ """Очередь файлов и подэтапов.
65
+
66
+ Данные хранятся следующим образом:
67
+ - Подэтапы с сущностями (строка вида "407-MARKS" (sub_stage_id-entity)) хранятся в упорядоченном множестве Redis
68
+ (Sorted Set in Redis)
69
+ - Информация по файлам стандартно по ключу - ключом выступает sub_stage_id
70
+ """
71
+ queue_key = 'rdm:export_sub_stage_ids_queue'
72
+ prefix = 'rdm:'
73
+
74
+ def __init__(self, *args, **kwargs):
75
+ """Инициализация объекта очереди Queue."""
76
+ super().__init__(*args, **kwargs)
77
+
78
+ self.connection = Redis(
79
+ host=settings.RDM_REDIS_HOST,
80
+ port=settings.RDM_REDIS_PORT,
81
+ db=settings.RDM_REDIS_DB,
82
+ password=settings.RDM_REDIS_PASSWORD
83
+ )
84
+
85
+ def _make_key(self, key: Union[int, str]) -> str:
86
+ """Формирование ключа."""
87
+ return f'{self.prefix}{key}'
88
+
89
+ @property
90
+ def count(self) -> int:
91
+ """Возвращает количество подэтапов в очереди."""
92
+ return self.connection.zcard(self.queue_key)
93
+
94
+ def enqueue(self, stage_id, entity_name: str, attachmets: List[UploadFile]) -> None:
95
+ """Помещение в очередь.
96
+
97
+ Подэтап попадает в упорядоченную очередь.
98
+ """
99
+ stage_info = f'{stage_id}{REDIS_QUEUE_KEY_DELIMITER}{entity_name}'
100
+ pipe = self.connection.pipeline()
101
+ # Упрядочиваем подэтапы
102
+ pipe.zadd(self.queue_key, {stage_info: stage_id})
103
+ pipe.set(self._make_key(stage_id), json.dumps(attachmets))
104
+ pipe.execute()
105
+
106
+ def dequeue_sub_stage_attachments(self, sub_stage_id: int) -> List[UploadFile]:
107
+ """Возвращает файлы подэтапа из кеша."""
108
+ result = []
109
+ attachments = self.connection.get(self._make_key(sub_stage_id))
110
+ attachments = json.loads(attachments) if attachments else ()
111
+ for attachment in attachments:
112
+ result.append(UploadFile(*attachment))
113
+
114
+ return result
115
+
116
+ def dequeue(self) -> Dict[Tuple[Any, Any], List[UploadFile]]:
117
+ """Возвращает подэтапы из очереди - берется вся очередь без ограничений."""
118
+ upload_files = {}
119
+ exported_sub_stages = self.connection.zrange(self.queue_key, 0, -1)
120
+ for sub_stage in exported_sub_stages:
121
+ sub_stage_info = as_text(sub_stage)
122
+ sub_stage_id, sub_stage_entity = sub_stage_info.split(REDIS_QUEUE_KEY_DELIMITER)
123
+ upload_files[(sub_stage_id, sub_stage_entity)] = self.dequeue_sub_stage_attachments(sub_stage_id)
124
+
125
+ return upload_files
126
+
127
+ def delete_sub_stages_attachments(self, sub_stage_id: int) -> None:
128
+ """Удаляет информацию о файлах из кеша."""
129
+ self.connection.delete(self._make_key(sub_stage_id))
130
+
131
+ def delete_sub_stages_from_queue(self, sub_stage_id: int, entity_name: str) -> None:
132
+ """Удаляет подэтап из очереди."""
133
+ self.connection.zrem(self.queue_key, f'{sub_stage_id}{REDIS_QUEUE_KEY_DELIMITER}{entity_name}')
134
+
135
+ def delete_from_queue(self, sub_stage_id: int, entity_name: str) -> None:
136
+ """Удаление элемента из очереди."""
137
+ self.delete_sub_stages_attachments(sub_stage_id)
138
+ self.delete_sub_stages_from_queue(sub_stage_id, entity_name)
139
+
140
+ def clear(self) -> int:
141
+ """Удаление из очереди всех подэтапов."""
142
+ script = """
143
+ local prefix = "{0}"
144
+ local q = KEYS[1]
145
+ local count = 0
146
+ while true do
147
+ local stage = redis.call("zpopmin", q)
148
+ local stage_id = stage[2]
149
+
150
+ if stage_id == nil then
151
+ break
152
+ end
153
+ redis.call("del", prefix..stage_id)
154
+ count = count + 1
155
+ end
156
+ return count
157
+ """.format(self.prefix).encode('utf-8')
158
+
159
+ script = self.connection.register_script(script)
160
+
161
+ return script(keys=[self.queue_key])
162
+
163
+ @property
164
+ def connection_info(self) -> str:
165
+ """Информация об используемом соединении."""
166
+ version = '.'.join(map(str, get_redis_version(self.connection)))
167
+ kwargs = self.connection.connection_pool.connection_kwargs
168
+ host = kwargs['host']
169
+ port = kwargs['port']
170
+ db = kwargs['db']
171
+
172
+ return f'Redis {version} on {host}:{port}/{db}'
@@ -40,7 +40,6 @@ from educommon import (
40
40
  from uploader_client.adapters import (
41
41
  adapter,
42
42
  )
43
-
44
43
  from edu_rdm_integration.collect_and_export_data.models import (
45
44
  EduRdmCollectDataCommandProgress,
46
45
  EduRdmExportDataCommandProgress,
@@ -51,6 +50,9 @@ from edu_rdm_integration.enums import (
51
50
  from edu_rdm_integration.export_data.base.requests import (
52
51
  RegionalDataMartStatusRequest,
53
52
  )
53
+ from edu_rdm_integration.export_data.consts import (
54
+ TOTAL_ATTACHMENTS_SIZE_KEY,
55
+ )
54
56
  from edu_rdm_integration.models import (
55
57
  DataMartRequestStatus,
56
58
  ExportingDataStage,
@@ -60,6 +62,9 @@ from edu_rdm_integration.models import (
60
62
  UploadStatusRequestLog,
61
63
  ExportingDataStageStatus,
62
64
  )
65
+ from edu_rdm_integration.redis_cache import (
66
+ AbstractCache,
67
+ )
63
68
 
64
69
  if TYPE_CHECKING:
65
70
  from datetime import (
@@ -80,8 +85,9 @@ if TYPE_CHECKING:
80
85
  class UploadStatusHelper:
81
86
  """Хелпер проверки статуса загрузки данных в витрину."""
82
87
 
83
- def __init__(self, in_progress_uploads: QuerySet) -> None:
88
+ def __init__(self, in_progress_uploads: QuerySet, cache: AbstractCache) -> None:
84
89
  self._in_progress_uploads = in_progress_uploads
90
+ self.cache = cache
85
91
 
86
92
  def run(self, thread_count: int = 1) -> None:
87
93
  """Запускает проверки статусов."""
@@ -167,6 +173,17 @@ class UploadStatusHelper:
167
173
  """Обрабатывает запись загрузки данных в витрину."""
168
174
  response, log_entry = self.send_upload_status_request(upload.request_id)
169
175
  self.update_upload_status(upload, response, log_entry)
176
+ # Обновим размер файлов в кеш (с блокировкой на время обновления)
177
+ with self.cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
178
+ queue_total_file_size = self.cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
179
+ if queue_total_file_size:
180
+ queue_total_file_size -= upload.attachment.attachment_size
181
+ if queue_total_file_size > 0:
182
+ self.cache.set(
183
+ TOTAL_ATTACHMENTS_SIZE_KEY,
184
+ queue_total_file_size,
185
+ timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
186
+ )
170
187
 
171
188
 
172
189
  class Graph:
@@ -205,18 +205,6 @@ class BaseCollectModelsDataByGeneratingLogsCommand(BaseCollectModelDataCommand):
205
205
  """
206
206
  super().add_arguments(parser=parser)
207
207
 
208
- parser.add_argument(
209
- '--logs_sub_period_days',
210
- action='store',
211
- dest='logs_sub_period_days',
212
- type=int,
213
- default=0,
214
- help=(
215
- 'Размер подпериодов, на которые будет разбит основной период, в днях. По умолчанию, '
216
- '0 - разбиение на подпериоды отключено.'
217
- ),
218
- )
219
-
220
208
  parser.add_argument(
221
209
  '--institute_ids',
222
210
  action='store',
@@ -0,0 +1,25 @@
1
+ # Generated by Django 3.1.14 on 2024-05-22 16:19
2
+
3
+ from django.db import (
4
+ migrations,
5
+ )
6
+
7
+
8
+ def add_exporting_data_sub_stage_status(apps, schema_editor):
9
+ """Добавление нового статуса этапу выгрузки данных."""
10
+ ExportingDataSubStageStatus = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageStatus') # noqa: N806
11
+ ExportingDataSubStageStatus.objects.get_or_create(
12
+ key='READY_FOR_EXPORT',
13
+ title='Готов к выгрузке'
14
+ )
15
+
16
+
17
+ class Migration(migrations.Migration):
18
+
19
+ dependencies = [
20
+ ('edu_rdm_integration', '0008_transferredentity'),
21
+ ]
22
+
23
+ operations = [
24
+ migrations.RunPython(add_exporting_data_sub_stage_status, reverse_code=migrations.RunPython.noop),
25
+ ]
@@ -1,4 +1,4 @@
1
- # Generated by Django 3.1.14 on 2024-07-24 09:14
1
+ # Generated by Django 3.1.14 on 2024-08-19 14:23
2
2
 
3
3
  from django.db import (
4
4
  migrations,
@@ -9,7 +9,7 @@ from django.db import (
9
9
  class Migration(migrations.Migration):
10
10
 
11
11
  dependencies = [
12
- ('edu_rdm_integration', '0008_transferredentity'),
12
+ ('edu_rdm_integration', '0009_auto_20240522_1619'),
13
13
  ]
14
14
 
15
15
  operations = [
@@ -0,0 +1,30 @@
1
+ # Generated by Django 3.1.14 on 2024-08-19 14:24
2
+
3
+ import django.db.models.deletion
4
+ from django.db import (
5
+ migrations,
6
+ models,
7
+ )
8
+
9
+
10
+ class Migration(migrations.Migration):
11
+
12
+ dependencies = [
13
+ ('edu_rdm_integration', '0010_transferredentity_export_enabled'),
14
+ ]
15
+
16
+ operations = [
17
+ migrations.CreateModel(
18
+ name='ExportingDataSubStageEntity',
19
+ fields=[
20
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
21
+ ('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.regionaldatamartentityenum', verbose_name='Сущность РВД')),
22
+ ('exporting_data_sub_stage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='edu_rdm_integration.exportingdatasubstage', verbose_name='Подэтап выгрузки данных')),
23
+ ],
24
+ options={
25
+ 'verbose_name': 'Связь сущности и подэтапа выгрузки',
26
+ 'verbose_name_plural': 'Связи сущности и подэтапа выгрузки',
27
+ 'db_table': 'rdm_exporting_data_sub_stage_entity',
28
+ },
29
+ ),
30
+ ]
@@ -0,0 +1,21 @@
1
+ # Generated by Django 3.1.14 on 2024-08-19 14:24
2
+
3
+ from django.db import (
4
+ migrations,
5
+ models,
6
+ )
7
+
8
+
9
+ class Migration(migrations.Migration):
10
+
11
+ dependencies = [
12
+ ('edu_rdm_integration', '0011_exportingdatasubstageentity'),
13
+ ]
14
+
15
+ operations = [
16
+ migrations.AddField(
17
+ model_name='exportingdatasubstageattachment',
18
+ name='attachment_size',
19
+ field=models.PositiveIntegerField(null=True, verbose_name='Размер файла (байт)'),
20
+ ),
21
+ ]
@@ -0,0 +1,48 @@
1
+ # Generated by Django 3.1.14 on 2024-08-19 14:56
2
+
3
+ from django.db import (
4
+ migrations,
5
+ )
6
+
7
+ from edu_rdm_integration.enums import (
8
+ FileUploadStatusEnum,
9
+ )
10
+
11
+
12
+ # Значение размера файла по умолчанию (если файл не найден)
13
+ ATTACHMENT_SIZES = 10_485_760
14
+
15
+
16
+ def set_attachment_size(apps, schema_editor):
17
+ """Установка размера файла по умолчанию."""
18
+ ExportingDataSubStageAttachment = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageAttachment') # noqa: N806
19
+ ExportingDataSubStageUploaderClientLog = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageUploaderClientLog') # noqa: N806
20
+
21
+ attachment_ids = ExportingDataSubStageUploaderClientLog.objects.filter(
22
+ file_upload_status=FileUploadStatusEnum.IN_PROGRESS
23
+ ).values_list('attachment_id', flat=True)
24
+
25
+ sub_stage_with_deleted_attachments = []
26
+ for sub_stage_attachment in ExportingDataSubStageAttachment.objects.filter(
27
+ attachment_size__isnull=True,
28
+ id__in=attachment_ids
29
+ ):
30
+ if sub_stage_attachment.attachment.field.storage.exists(sub_stage_attachment.attachment.name):
31
+ sub_stage_attachment.attachment_size = sub_stage_attachment.attachment.size
32
+ else:
33
+ sub_stage_attachment.attachment_size = ATTACHMENT_SIZES
34
+ sub_stage_with_deleted_attachments.append(sub_stage_attachment)
35
+
36
+ ExportingDataSubStageAttachment.objects.bulk_update(sub_stage_with_deleted_attachments, ['attachment_size'])
37
+
38
+
39
+ class Migration(migrations.Migration):
40
+
41
+ dependencies = [
42
+ ('edu_rdm_integration', '0012_exportingdatasubstageattachment_attachment_size'),
43
+ ]
44
+
45
+ operations = [
46
+ migrations.RunPython(set_attachment_size, reverse_code=migrations.RunPython.noop)
47
+ ]
48
+
@@ -25,6 +25,7 @@ from django.db.models import (
25
25
  ForeignKey,
26
26
  Manager,
27
27
  OneToOneField,
28
+ PositiveIntegerField,
28
29
  SmallIntegerField,
29
30
  UUIDField,
30
31
  )
@@ -347,9 +348,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
347
348
 
348
349
 
349
350
  class ExportingDataSubStageStatus(TitledModelEnum):
350
- """
351
- Модель-перечисление статусов этапа выгрузки данных
352
- """
351
+ """Модель-перечисление статусов этапа выгрузки данных."""
353
352
 
354
353
  CREATED = ModelEnumValue(
355
354
  title='Создан',
@@ -366,6 +365,9 @@ class ExportingDataSubStageStatus(TitledModelEnum):
366
365
  FINISHED = ModelEnumValue(
367
366
  title='Завершен',
368
367
  )
368
+ READY_FOR_EXPORT = ModelEnumValue(
369
+ title='Готов к выгрузке',
370
+ )
369
371
 
370
372
  class Meta:
371
373
  db_table = 'rdm_exporting_data_sub_stage_status'
@@ -423,7 +425,11 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
423
425
 
424
426
  def save(self, *args, **kwargs):
425
427
  if (
426
- self.status_id in (ExportingDataSubStageStatus.FAILED.key, ExportingDataSubStageStatus.FINISHED.key)
428
+ self.status_id in {
429
+ ExportingDataSubStageStatus.FAILED.key,
430
+ ExportingDataSubStageStatus.FINISHED.key,
431
+ ExportingDataSubStageStatus.READY_FOR_EXPORT.key,
432
+ }
427
433
  and not self.ended_at
428
434
  ):
429
435
  self.ended_at = datetime.now()
@@ -469,6 +475,10 @@ class ExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
469
475
  null=True,
470
476
  blank=True,
471
477
  )
478
+ attachment_size = PositiveIntegerField(
479
+ null=True,
480
+ verbose_name='Размер файла (байт)'
481
+ )
472
482
 
473
483
  class Meta:
474
484
  db_table = 'rdm_exporting_data_sub_stage_attachment'
@@ -922,3 +932,24 @@ class TransferredEntity(BaseObjectModel):
922
932
  @json_encode
923
933
  def no_export(self):
924
934
  return 'Нет' if self.export_enabled else 'Да'
935
+
936
+
937
+ class ExportingDataSubStageEntity(BaseObjectModel):
938
+ """Модель связи сущности и подэтапа выгрузки."""
939
+
940
+ entity = ForeignKey(
941
+ to=RegionalDataMartEntityEnum,
942
+ verbose_name='Сущность РВД',
943
+ on_delete=PROTECT,
944
+ )
945
+
946
+ exporting_data_sub_stage = ForeignKey(
947
+ to=ExportingDataSubStage,
948
+ verbose_name='Подэтап выгрузки данных',
949
+ on_delete=CASCADE,
950
+ )
951
+
952
+ class Meta:
953
+ db_table = 'rdm_exporting_data_sub_stage_entity'
954
+ verbose_name = 'Связь сущности и подэтапа выгрузки'
955
+ verbose_name_plural = 'Связи сущности и подэтапа выгрузки'
@@ -0,0 +1,52 @@
1
+ from abc import (
2
+ ABCMeta,
3
+ abstractmethod,
4
+ )
5
+ from typing import (
6
+ Tuple,
7
+ Union,
8
+ )
9
+
10
+ from redis import (
11
+ Redis,
12
+ ResponseError,
13
+ )
14
+
15
+
16
+ def get_redis_version(connection: 'Redis') -> Tuple[int, int, int]:
17
+ """Возвращает кортеж с версией сервера Redis."""
18
+ try:
19
+ version = getattr(connection, '__redis_server_version', None)
20
+ if not version:
21
+ version = tuple([int(n) for n in connection.info('server')['redis_version'].split('.')[:3]])
22
+ setattr(connection, '__redis_server_version', version)
23
+ except ResponseError:
24
+ version = (0, 0, 0)
25
+
26
+ return version
27
+
28
+
29
+ def as_text(v: Union[bytes, str]) -> str:
30
+ """Конвертирует последовательность байт в строку."""
31
+ if isinstance(v, bytes):
32
+ return v.decode('utf-8')
33
+ elif isinstance(v, str):
34
+ return v
35
+ else:
36
+ raise ValueError('Неизвестный тип %r' % type(v))
37
+
38
+
39
+ class AbstractCache(metaclass=ABCMeta):
40
+ """Абстрактный интерфейс для кеша отправки."""
41
+
42
+ @abstractmethod
43
+ def get(self, key, default=None, **kwargs):
44
+ """Возвращает значение из кеша по ключу."""
45
+
46
+ @abstractmethod
47
+ def set(self, key, value, timeout=None, **kwargs):
48
+ """Сохраняет значение в кеш по ключу."""
49
+
50
+ @abstractmethod
51
+ def lock(self, name, timeout=None, **kwargs):
52
+ """Захватывает блокировку."""