edu-rdm-integration 3.2.8__py3-none-any.whl → 3.3.3__py3-none-any.whl

edu_rdm_integration/app_settings.py

@@ -43,3 +43,8 @@ RDM_UPLOAD_STATUS_TASK_MINUTE = '*/30'
 RDM_UPLOAD_STATUS_TASK_HOUR = '*'
 RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK = '*'
 RDM_UPLOAD_STATUS_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
+
+# Настройка очереди Redis для формирования файлов РВД
+RDM_UPLOAD_QUEUE_MAX_SIZE = 500_000_000
+# Таймаут для сохранения параметров в общем кеш
+RDM_REDIS_CACHE_TIMEOUT_SECONDS = 60 * 60 * 2

edu_rdm_integration/apps.py

@@ -30,11 +30,11 @@ class EduRDMIntegrationConfig(AppConfig):
         """
         Инициализация клиента для взаимодействия с Региональной витриной данных.
         """
+
         from django.core.cache import (
             DEFAULT_CACHE_ALIAS,
             caches,
         )
-
         import uploader_client
         from uploader_client.contrib.rdm.interfaces.configurations import (
             RegionalDataMartUploaderConfig,

edu_rdm_integration/collect_data/collect.py

@@ -3,7 +3,6 @@ from datetime import (
     date,
     datetime,
     time,
-    timedelta,
 )
 from typing import (
     TYPE_CHECKING,
@@ -42,7 +41,6 @@ from edu_rdm_integration.base import (
     BaseOperationData,
 )
 from edu_rdm_integration.consts import (
-    LOGS_SUB_PERIOD_DAYS,
     REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
 )
 from edu_rdm_integration.helpers import (
@@ -195,13 +193,11 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
         models: Iterable[str],
         logs_period_started_at=datetime.combine(date.today(), time.min),
         logs_period_ended_at=datetime.combine(date.today(), time.min),
-        logs_sub_period_days=LOGS_SUB_PERIOD_DAYS,
         institute_ids=(),
         **kwargs,
     ):
         super().__init__(models, logs_period_started_at, logs_period_ended_at, **kwargs)
 
-        self.logs_sub_period_days = logs_sub_period_days
         # Учебные заведения, для которых производится выгрузка
         self.institute_ids = institute_ids
         # Генератор логов
@@ -211,42 +207,11 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
         """Возвращает генератор логов."""
         raise NotImplementedError
 
-    def _generate_logs_by_subperiod(self):
-        """
-        Генерация логов с учетом подпериодов.
-        """
-        temp_logs_period_started_at = self.logs_period_started_at
-        temp_logs_period_ended_at = self.logs_period_started_at + timedelta(days=self.logs_sub_period_days)
-
-        if temp_logs_period_ended_at > self.logs_period_ended_at:
-            temp_logs_period_ended_at = self.logs_period_ended_at
-
-        temp_logs: Dict[str, List['AuditLog']] = {}
-
-        while temp_logs_period_started_at < temp_logs_period_ended_at <= self.logs_period_ended_at:
-            for model in self.models:
-                logs = self.log_generator.generate(
-                    model=model,
-                    logs_period_started_at=temp_logs_period_started_at,
-                    logs_period_ended_at=temp_logs_period_ended_at,
-                    institute_ids=self.institute_ids,
-                )
-
-                temp_logs[model.key] = logs
-
-            yield temp_logs, temp_logs_period_started_at, temp_logs_period_ended_at
-
-            temp_logs_period_started_at = temp_logs_period_ended_at
-            temp_logs_period_ended_at += timedelta(days=self.logs_sub_period_days)
-
-            if temp_logs_period_ended_at > self.logs_period_ended_at:
-                temp_logs_period_ended_at = self.logs_period_ended_at
-
-            temp_logs.clear()
+    def _generate_logs(self) -> List[Tuple[Dict[str, List['AuditLog']], datetime, datetime]]:
+        """Генерация логов.
 
-    def _generate_logs_for_all_period(self):
-        """
-        Генерация логов за весь период.
+        Осуществляет генерацию логов по уже существующим записям в базе данных. В качестве параметров указываются
+        начало и конец периода сбора логов. Генерация логов производится только для указанных моделей.
         """
         temp_logs: Dict[str, List['AuditLog']] = {}
 
@@ -262,21 +227,6 @@ class BaseCollectModelsDataByGeneratingLogs(BaseCollectModelsData):
 
         return [(temp_logs, self.logs_period_started_at, self.logs_period_ended_at)]
 
-    def _generate_logs(self) -> List[Tuple[Dict[str, List['AuditLog']], datetime, datetime]]:
-        """
-        Генерация логов.
-
-        Осуществляет генерацию логов по уже существующим записям в базе данных. В качестве параметров указываются
-        начало и конец периода сбора логов, размер подпериодов, на которые должен быть разбит основной период.
-        Генерация логов производится только для указанных моделей.
-        """
-        if self.logs_sub_period_days:
-            logs = self._generate_logs_by_subperiod()
-        else:
-            logs = self._generate_logs_for_all_period()
-
-        return logs
-
     def _collect(self):
         """
         Собирает данные моделей РВД.

edu_rdm_integration/consts.py

@@ -19,7 +19,6 @@ LOGS_DELIMITER = '    '
 
 HASH_ALGORITHM = 'md_gost12_512'
 
-LOGS_SUB_PERIOD_DAYS = 0
 BATCH_SIZE = 5000
 
 ACADEMIC_YEAR = {

edu_rdm_integration/export_data/base/functions.py

@@ -1,22 +1,12 @@
-import json
 from abc import (
     ABCMeta,
 )
-from pathlib import (
-    Path,
-)
 from typing import (
-    Dict,
     List,
-    Optional,
-    Tuple,
     Type,
     Union,
 )
 
-from django.conf import (
-    settings,
-)
 from django.core.files.base import (
     ContentFile,
 )
@@ -26,9 +16,6 @@ from django.core.files.storage import (
 from transliterate import (
     slugify,
 )
-from uploader_client.adapters import (
-    adapter,
-)
 
 from educommon import (
     logger,
@@ -46,22 +33,9 @@ from edu_rdm_integration.adapters.functions import (
 from edu_rdm_integration.consts import (
     LOGS_DELIMITER,
 )
-from edu_rdm_integration.enums import (
-    FileUploadStatusEnum,
-)
-from edu_rdm_integration.export_data.base.consts import (
-    OPERATIONS_METHODS_MAP,
-    OPERATIONS_URLS_MAP,
-)
-from edu_rdm_integration.export_data.base.errors import (
-    BaseExportDataError,
-)
 from edu_rdm_integration.export_data.base.helpers import (
     BaseExportDataFunctionHelper,
 )
-from edu_rdm_integration.export_data.base.requests import (
-    RegionalDataMartEntityRequest,
-)
 from edu_rdm_integration.export_data.base.results import (
     BaseExportDataFunctionResult,
 )
@@ -75,8 +49,8 @@ from edu_rdm_integration.models import (
     ExportingDataStage,
     ExportingDataSubStage,
     ExportingDataSubStageAttachment,
+    ExportingDataSubStageEntity,
     ExportingDataSubStageStatus,
-    ExportingDataSubStageUploaderClientLog,
 )
 from edu_rdm_integration.utils import (
     get_exporting_data_stage_attachment_path,
@@ -99,6 +73,10 @@ class BaseExportDataFunction(
             stage=stage,
             function_id=self.uuid,
         )
+        ExportingDataSubStageEntity.objects.create(
+            exporting_data_sub_stage=self._sub_stage,
+            entity_id=self.first_entity.key
+        )
         # Проставление подэтапа выгрузки
         self.entities[0].main_model_enum.model.objects.filter(pk__in=model_ids).update(
             exporting_sub_stage=self._sub_stage,
@@ -116,11 +94,7 @@ class BaseExportDataFunction(
             EntityLogOperation.DELETE: [],
         }
 
-        self._file_contents: Dict[int, Optional[Tuple[Optional[str], Optional[ExportingDataSubStageAttachment]]]] = {
-            EntityLogOperation.CREATE: (None, None),
-            EntityLogOperation.UPDATE: (None, None),
-            EntityLogOperation.DELETE: (None, None),
-        }
+        self.has_data = False
 
     def _prepare_helper_class(self) -> Type[BaseExportDataFunctionHelper]:
         """
@@ -201,8 +175,10 @@ class BaseExportDataFunction(
         Формирование файлов для дальнейшей выгрузки.
         """
         logger.info(f'{LOGS_DELIMITER * 3}{self.__class__.__name__} prepare files..')
+        has_record = False
 
         for operation in EntityLogOperation.values.keys():
+
             records = self._data[operation]
             if records:
                 title_record = (
@@ -230,83 +206,12 @@ class BaseExportDataFunction(
                 )
 
                 sub_stage_attachment.attachment = default_storage.save(file_path, ContentFile(joined_records))
+                sub_stage_attachment.attachment_size = sub_stage_attachment.attachment.size
 
                 self.do_on_save(sub_stage_attachment)
+                has_record |= True
 
-                self._file_contents[operation] = (file_path, sub_stage_attachment)
-
-    def _send_files(self):
-        """
-        Отправка файлов в "Региональная витрина данных".
-        """
-        # TODO При старте проекта и инициализации приложения function_tools происходит поиск стратегий. При поиске
-        #  возникает ошибка "Не произведена настройка клиента". Настройка производится в конфиге
-        #  приложения regional_data_mart_integration. Нужно доработать механизм так, чтобы сборка стратегий запускалась
-        #  после готовности всех приложений.
-
-        logger.info(f'{LOGS_DELIMITER * 3}{self.__class__.__name__} send files..')
-
-        requests_result = []
-
-        for operation, (relative_file_path, sub_stage_attachment) in self._file_contents.items():
-            if relative_file_path:
-                if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
-                    logger.warning(
-                        f'{LOGS_DELIMITER * 3}ATTENTION!!! REGIONAL DATA MART INTEGRATION REQUEST EMULATION ENABLED!'
-                    )
-
-                method = OPERATIONS_METHODS_MAP.get(operation)
-                file_path = Path.joinpath(Path(settings.MEDIA_ROOT), relative_file_path)
-
-                request = RegionalDataMartEntityRequest(
-                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
-                    table_name=self.first_entity.key.lower(),
-                    method=method,
-                    operation=OPERATIONS_URLS_MAP.get(operation),
-                    parameters={},
-                    headers={
-                        'Content-Type': 'text/csv',
-                    },
-                    files=[],
-                    data=file_path.open('rb').read(),
-                )
-
-                result = adapter.send(request)
-
-                request_id = ''
-                if not result.error:
-                    request_id = result.response.text
-
-                file_upload_status = FileUploadStatusEnum.IN_PROGRESS if request_id else FileUploadStatusEnum.ERROR
-
-                sub_stage_uploader_client_log = ExportingDataSubStageUploaderClientLog(
-                    entry=result.log,
-                    sub_stage=self._sub_stage,
-                    attachment=sub_stage_attachment,
-                    request_id=request_id,
-                    file_upload_status=file_upload_status,
-                    is_emulation=settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION,
-                )
-
-                self.do_on_save(sub_stage_uploader_client_log)
-
-                if result.error:
-                    logger.warning(
-                        f'{result.error}\nrequest - "{result.log.request}"\nresponse - "{result.log.response}"'
-                    )
-
-                    error = BaseExportDataError(
-                        message=result.error,
-                    )
-                    self.result.append_entity(entity=error)
-
-                    break
-                else:
-                    logger.info(
-                        f'Response with {result.response.status_code} code and content {result.response.text}'
-                    )
-
-                requests_result.append(result)
+        self.has_data = has_record
 
     def _prepare(self, *args, **kwargs):
         """
@@ -316,7 +221,6 @@ class BaseExportDataFunction(
             if self.helper.cache.entity_instances:
                 self._prepare_data()
                 self._prepare_files()
-                self._send_files()
             else:
                 logger.info(f'{LOGS_DELIMITER * 3} no data for preparing.')
 
@@ -342,7 +246,10 @@ class BaseExportDataFunction(
         super().run(*args, **kwargs)
 
         if self.result.has_not_errors:
-            self._sub_stage.status_id = ExportingDataSubStageStatus.FINISHED.key
+            self._sub_stage.status_id = (
+                ExportingDataSubStageStatus.FINISHED.key if not self.has_data
+                else ExportingDataSubStageStatus.READY_FOR_EXPORT.key
+            )
         else:
             self._sub_stage.status_id = ExportingDataSubStageStatus.FAILED.key
 

edu_rdm_integration/export_data/consts.py

@@ -1 +1,6 @@
 DELIMITER = ';'
+
+# Разделитель между id подэтапа и сущностью в составном ключе очереди
+REDIS_QUEUE_KEY_DELIMITER = ':'
+# Ключ параметра Общий объем отправленных в РВД файлов
+TOTAL_ATTACHMENTS_SIZE_KEY = 'rdm:total_attachments_size'

edu_rdm_integration/export_data/dataclasses.py

@@ -0,0 +1,11 @@
+from typing import (
+    NamedTuple,
+)
+
+
+class UploadFile(NamedTuple):
+    """Структура файла подэтапа для очереди отправки."""
+    attachment_id: int
+    attachment_name: str
+    attachment_size: int
+    operation: str

edu_rdm_integration/export_data/export_manger.py

@@ -0,0 +1,246 @@
+from datetime import (
+    datetime,
+)
+from pathlib import (
+    Path,
+)
+from typing import (
+    List,
+)
+
+from django.conf import (
+    settings,
+)
+from django.db.models import (
+    F,
+    Sum,
+)
+from django.db.transaction import (
+    atomic,
+)
+from uploader_client.adapters import (
+    adapter,
+)
+
+from educommon import (
+    logger,
+)
+
+from edu_rdm_integration.consts import (
+    LOGS_DELIMITER,
+)
+from edu_rdm_integration.enums import (
+    FileUploadStatusEnum,
+)
+from edu_rdm_integration.export_data.base.consts import (
+    OPERATIONS_METHODS_MAP,
+    OPERATIONS_URLS_MAP,
+)
+from edu_rdm_integration.export_data.base.requests import (
+    RegionalDataMartEntityRequest,
+)
+from edu_rdm_integration.export_data.consts import (
+    TOTAL_ATTACHMENTS_SIZE_KEY,
+)
+from edu_rdm_integration.export_data.dataclasses import (
+    UploadFile,
+)
+from edu_rdm_integration.export_data.queue import (
+    Queue,
+)
+from edu_rdm_integration.models import (
+    ExportingDataSubStage,
+    ExportingDataSubStageAttachment,
+    ExportingDataSubStageStatus,
+    ExportingDataSubStageUploaderClientLog,
+)
+from edu_rdm_integration.redis_cache import (
+    AbstractCache,
+)
+
+
+class ExportQueueSender:
+    """Класс отправки данных в очередь РВД."""
+
+    def __init__(self, data_cache: AbstractCache, queue: Queue):
+        self.queue = queue
+        self.cache = data_cache
+        self.queue_total_file_size = self.cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+
+    @staticmethod
+    def get_exported_file_size() -> int:
+        """Возвращает размер отправленных в РВД файлов.
+
+        При расчете используются только файлы, отмеченные как отправленные, но не обработанные витриной.
+        """
+        sended_attachment_ids = ExportingDataSubStageUploaderClientLog.objects.filter(
+            file_upload_status=FileUploadStatusEnum.IN_PROGRESS
+        ).values_list('attachment_id', flat=True)
+        file_size = ExportingDataSubStageAttachment.objects.filter(
+            id__in=sended_attachment_ids
+        ).aggregate(Sum('attachment_size'))['attachment_size__sum']
+
+        return file_size or 0
+
+    @staticmethod
+    def get_sub_stages_attachments_to_export():
+        """Выборка готовых к экспорту подэтапов."""
+        return ExportingDataSubStage.objects.filter(
+            status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key
+        ).annotate(
+            attachment_id=F('exportingdatasubstageattachment__id'),
+            attachment_name=F('exportingdatasubstageattachment__attachment'),
+            attachment_size=F('exportingdatasubstageattachment__attachment_size'),
+            operation=F('exportingdatasubstageattachment__operation'),
+            entity=F('exportingdatasubstageentity__entity_id')
+        ).order_by('started_at', 'operation').values(
+            'id',
+            'attachment_id',
+            'attachment_name',
+            'attachment_size',
+            'operation',
+            'entity',
+        )
+
+    def set_sub_stage_to_cache(self, sub_stage_id: int, entity_name: str, attachments: List[UploadFile]) -> bool:
+        """Помещение подэтапа в очередь вместе с информацией по файлам и обновление счетчика объема файлов."""
+        sub_stage_total_size = sum((attachment.attachment_size for attachment in attachments))
+        if self.queue_total_file_size + sub_stage_total_size > settings.RDM_UPLOAD_QUEUE_MAX_SIZE:
+            return False
+
+        self.queue.enqueue(sub_stage_id, entity_name, attachments)
+        self.queue_total_file_size += sub_stage_total_size
+        # Обновим размер файлов в кеш
+        self.cache.set(
+            TOTAL_ATTACHMENTS_SIZE_KEY,
+            self.queue_total_file_size,
+            timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+        )
+
+        return True
+
+    def run(self):
+        """Запуск работы менджера."""
+        if not self.queue_total_file_size:
+            self.queue_total_file_size = self.get_exported_file_size()
+
+        if self.queue_total_file_size < settings.RDM_UPLOAD_QUEUE_MAX_SIZE:
+            stage_files = []
+            prev_sub_stage = None
+            entity = ''
+            # Если размер очереди позволяет - то отправляем все файлы подэтапа в очередь - иначе прерываем процесс
+            for stage_attachment in self.get_sub_stages_attachments_to_export():
+                if prev_sub_stage != stage_attachment['id']:
+                    if stage_files:
+                        to_cache = self.set_sub_stage_to_cache(prev_sub_stage, entity, stage_files)
+                        logger.info(
+                            f'ExportedDataSubStage {prev_sub_stage} {entity} added to the queue'
+                        )
+                        stage_files = []
+
+                        if not to_cache:
+                            break
+
+                    prev_sub_stage = stage_attachment['id']
+
+                if stage_attachment['attachment_size']:
+                    stage_files.append(UploadFile(
+                        stage_attachment['attachment_id'],
+                        stage_attachment['attachment_name'],
+                        stage_attachment['attachment_size'],
+                        stage_attachment['operation']
+                    ))
+                    entity = stage_attachment['entity']
+
+            # Обновляем общий объем очереди и закидываем последний элемент
+            if stage_files:
+                self.set_sub_stage_to_cache(prev_sub_stage, entity, stage_files)
+        else:
+            # Сохраняем объем отправленных файлов в кеш
+            self.cache.set(
+                TOTAL_ATTACHMENTS_SIZE_KEY,
+                self.queue_total_file_size,
+                timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+            )
+
+
+class WorkerSender:
+    """Непосредственная отправка файлов."""
+
+    def __init__(self, queue: Queue):
+        self.queue = queue
+
+    def send_files(self):
+        """Отправка файлов."""
+        sub_stages = self.queue.dequeue()
+        for (sub_stage_id, entity_key), upload_files in sub_stages.items():
+            sub_stage = ExportingDataSubStage.objects.filter(id=sub_stage_id).first()
+            if not sub_stage:
+                return
+
+            result_to_save = []
+            errors = []
+            status = ExportingDataSubStageStatus.FINISHED.key
+            for file in upload_files:
+                if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
+                    logger.warning(
+                        f'{LOGS_DELIMITER * 3}ATTENTION!!! REGIONAL DATA MART INTEGRATION REQUEST EMULATION ENABLED!'
+                    )
+
+                method = OPERATIONS_METHODS_MAP.get(file.operation)
+                file_path = Path.joinpath(Path(settings.MEDIA_ROOT), file.attachment_name)
+
+                request = RegionalDataMartEntityRequest(
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    table_name=entity_key.lower(),
+                    method=method,
+                    operation=OPERATIONS_URLS_MAP.get(file.operation),
+                    parameters={},
+                    headers={
+                        'Content-Type': 'text/csv',
+                    },
+                    files=[],
+                    data=file_path.open('rb').read(),
+                )
+
+                result = adapter.send(request)
+
+                request_id = ''
+                if not result.error:
+                    request_id = result.response.text
+
+                file_upload_status = FileUploadStatusEnum.IN_PROGRESS if request_id else FileUploadStatusEnum.ERROR
+
+                sub_stage_uploader_client_log = ExportingDataSubStageUploaderClientLog(
+                    entry=result.log,
+                    sub_stage_id=sub_stage_id,
+                    attachment_id=file.attachment_id,
+                    request_id=request_id,
+                    file_upload_status=file_upload_status,
+                    is_emulation=settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION,
+                )
+                result_to_save.append(sub_stage_uploader_client_log)
+                if result.error:
+                    logger.warning(
+                        f'{result.error}\nrequest - "{result.log.request}"\nresponse - "{result.log.response}"'
+                    )
+
+                    errors.append(result.error)
+                    status = ExportingDataSubStageStatus.FAILED.key
+                else:
+                    logger.info(
+                        f'Response with {result.response.status_code} code and content {result.response.text}'
+                    )
+
+            # Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
+            if result_to_save:
+                with atomic():
+                    ExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
+                    sub_stage.status_id = status
+                    sub_stage.save()
+
+            self.queue.delete_from_queue(sub_stage_id=sub_stage_id, entity_name=entity_key)
+
+    def run(self):
+        """Запуск воркера отправки."""
+        self.send_files()