edu-rdm-integration 3.9.1__py3-none-any.whl → 3.10.0__py3-none-any.whl

edu_rdm_integration/{export_data/export_manager.py → stages/upload_data/export_managers.py}

@@ -3,8 +3,6 @@ from pathlib import (
 )
 from typing import (
     TYPE_CHECKING,
-    Any,
-    Iterable,
     Optional,
 )
 
@@ -22,51 +20,63 @@ from django.db.transaction import (
 from uploader_client.adapters import (
     adapter,
 )
+from web_edu.plugins.gis_ruo.enums import (
+    OPERATIONS_METHODS_MAP,
+)
 
 from educommon import (
     logger,
 )
 
-from edu_rdm_integration.consts import (
+from edu_rdm_integration.core.consts import (
     LOGS_DELIMITER,
 )
-from edu_rdm_integration.enums import (
-    FileUploadStatusEnum,
+from edu_rdm_integration.core.redis_cache import (
+    AbstractCache,
 )
-from edu_rdm_integration.export_data.base.consts import (
-    OPERATIONS_METHODS_MAP,
+from edu_rdm_integration.stages.export_data.consts import (
+    ATTACHMENTS_LIMIT,
+    TOTAL_ATTACHMENTS_SIZE_KEY,
+)
+from edu_rdm_integration.stages.export_data.functions.base.consts import (
     OPERATIONS_URLS_MAP,
 )
-from edu_rdm_integration.export_data.base.requests import (
+from edu_rdm_integration.stages.export_data.functions.base.requests import (
     RegionalDataMartEntityRequest,
 )
-from edu_rdm_integration.export_data.consts import (
-    ATTACHMENTS_LIMIT,
-    TOTAL_ATTACHMENTS_SIZE_KEY,
+from edu_rdm_integration.stages.export_data.models import (
+    ExportingDataSubStage,
+    ExportingDataSubStageAttachment,
+    ExportingDataSubStageStatus,
 )
-from edu_rdm_integration.export_data.dataclasses import (
+from edu_rdm_integration.stages.upload_data.consts import (
+    FAILED_STATUSES,
+)
+from edu_rdm_integration.stages.upload_data.dataclasses import (
     UploadFile,
 )
-from edu_rdm_integration.export_data.queue import (
-    Queue,
+from edu_rdm_integration.stages.upload_data.enums import (
+    FileUploadStatusEnum,
 )
-from edu_rdm_integration.helpers import (
-    FAILED_STATUSES,
+from edu_rdm_integration.stages.upload_data.helpers import (
     UploadStatusHelper,
 )
-from edu_rdm_integration.models import (
+from edu_rdm_integration.stages.upload_data.models import (
     DataMartRequestStatus,
-    ExportingDataSubStage,
-    ExportingDataSubStageAttachment,
-    ExportingDataSubStageStatus,
     ExportingDataSubStageUploaderClientLog,
     UploadStatusRequestLog,
 )
-from edu_rdm_integration.redis_cache import (
-    AbstractCache,
+from edu_rdm_integration.stages.upload_data.queues import (
+    Queue,
 )
 
 
+if TYPE_CHECKING:
+    from uploader_client.models import (
+        Entry,
+    )
+
+
 class ExportQueueSender:
     """Класс отправки данных в очередь РВД."""
 
@@ -86,9 +96,9 @@ class ExportQueueSender:
         sended_attachment_ids = ExportingDataSubStageUploaderClientLog.objects.filter(
             file_upload_status=FileUploadStatusEnum.IN_PROGRESS
         ).values_list('attachment_id', flat=True)
-        file_size = ExportingDataSubStageAttachment.objects.filter(
-            id__in=sended_attachment_ids
-        ).aggregate(Sum('attachment_size'))['attachment_size__sum']
+        file_size = ExportingDataSubStageAttachment.objects.filter(id__in=sended_attachment_ids).aggregate(
+            Sum('attachment_size')
+        )['attachment_size__sum']
 
         return file_size or 0
 
@@ -98,25 +108,30 @@ class ExportQueueSender:
 
     def get_sub_stages_attachments_to_export(self):
         """Выборка готовых к экспорту подэтапов."""
-        sub_stage_ids = ExportingDataSubStage.objects.filter(
-            self._make_stage_filter()
-        ).order_by('started_at').values_list('id', flat=True)[:self.limit]
-
-        return ExportingDataSubStage.objects.filter(
-            id__in=sub_stage_ids
-        ).annotate(
-            attachment_id=F('exportingdatasubstageattachment__id'),
-            attachment_name=F('exportingdatasubstageattachment__attachment'),
-            attachment_size=F('exportingdatasubstageattachment__attachment_size'),
-            operation=F('exportingdatasubstageattachment__operation'),
-            entity=F('exportingdatasubstageentity__entity_id')
-        ).order_by('id', 'operation').values(
-            'id',
-            'attachment_id',
-            'attachment_name',
-            'attachment_size',
-            'operation',
-            'entity',
+        sub_stage_ids = (
+            ExportingDataSubStage.objects.filter(self._make_stage_filter())
+            .order_by('started_at')
+            .values_list('id', flat=True)[: self.limit]
+        )
+
+        return (
+            ExportingDataSubStage.objects.filter(id__in=sub_stage_ids)
+            .annotate(
+                attachment_id=F('exportingdatasubstageattachment__id'),
+                attachment_name=F('exportingdatasubstageattachment__attachment'),
+                attachment_size=F('exportingdatasubstageattachment__attachment_size'),
+                operation=F('exportingdatasubstageattachment__operation'),
+                entity=F('exportingdatasubstageentity__entity_id'),
+            )
+            .order_by('id', 'operation')
+            .values(
+                'id',
+                'attachment_id',
+                'attachment_name',
+                'attachment_size',
+                'operation',
+                'entity',
+            )
         )
 
     def set_sub_stage_to_cache(self, sub_stage_id: int, entity_name: str, attachments: list[UploadFile]) -> bool:
@@ -129,14 +144,10 @@ class ExportQueueSender:
         self.queue_total_file_size += sub_stage_total_size
         # Обновим размер файлов в кеш
         self.cache.set(
-            TOTAL_ATTACHMENTS_SIZE_KEY,
-            self.queue_total_file_size,
-            timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+            TOTAL_ATTACHMENTS_SIZE_KEY, self.queue_total_file_size, timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
         )
 
-        logger.info(
-             f'{LOGS_DELIMITER * 2}ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue'
-        )
+        logger.info(f'{LOGS_DELIMITER * 2}ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue')
 
         return True
 
@@ -162,12 +173,14 @@ class ExportQueueSender:
                     prev_sub_stage = stage_attachment['id']
 
                 if stage_attachment['attachment_size']:
-                    stage_files.append(UploadFile(
-                        stage_attachment['attachment_id'],
-                        stage_attachment['attachment_name'],
-                        stage_attachment['attachment_size'],
-                        stage_attachment['operation']
-                    ))
+                    stage_files.append(
+                        UploadFile(
+                            stage_attachment['attachment_id'],
+                            stage_attachment['attachment_name'],
+                            stage_attachment['attachment_size'],
+                            stage_attachment['operation'],
+                        )
+                    )
                     entity = stage_attachment['entity']
 
             # Обновляем общий объем очереди и закидываем последний элемент
@@ -176,14 +189,10 @@ class ExportQueueSender:
         else:
             # Сохраняем объем отправленных файлов в кеш
             self.cache.set(
-                TOTAL_ATTACHMENTS_SIZE_KEY,
-                self.queue_total_file_size,
-                timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+                TOTAL_ATTACHMENTS_SIZE_KEY, self.queue_total_file_size, timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
             )
             self.queue_is_full = True
-            logger.warning(
-                f'Total exported file size:  {self.queue_total_file_size} - queue is full!!!'
-            )
+            logger.warning(f'Total exported file size:  {self.queue_total_file_size} - queue is full!!!')
 
 
 class WorkerSender:
@@ -303,9 +312,7 @@ class WorkerSender:
                     errors.append(result.error)
                     status = ExportingDataSubStageStatus.FAILED.key
                 else:
-                    logger.info(
-                        f'Response with {result.response.status_code} code and content {result.response.text}'
-                    )
+                    logger.info(f'Response with {result.response.status_code} code and content {result.response.text}')
                     self.entities.add(entity_key)
 
             # Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
@@ -316,9 +323,7 @@ class WorkerSender:
                 sub_stage.save()
 
             self.queue.delete_from_queue(sub_stage_id=sub_stage_id, entity_name=entity_key)
-            logger.info(
-                f'{LOGS_DELIMITER * 3}ExportedDataSubStage {sub_stage_id} {entity_key} sended from the queue'
-            )
+            logger.info(f'{LOGS_DELIMITER * 3}ExportedDataSubStage {sub_stage_id} {entity_key} sent from the queue')
 
     def run(self):
         """Запуск воркера отправки."""

edu_rdm_integration/stages/upload_data/helpers.py

@@ -0,0 +1,164 @@
+from concurrent.futures import (
+    ThreadPoolExecutor,
+)
+from json import (
+    JSONDecodeError,
+)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Optional,
+)
+
+from django.conf import (
+    settings,
+)
+from django.db import (
+    transaction,
+)
+from django.db.models import (
+    QuerySet,
+)
+from uploader_client.adapters import (
+    adapter,
+)
+
+from educommon import (
+    logger,
+)
+
+from edu_rdm_integration.core.redis_cache import (
+    AbstractCache,
+)
+from edu_rdm_integration.stages.export_data.consts import (
+    TOTAL_ATTACHMENTS_SIZE_KEY,
+)
+from edu_rdm_integration.stages.export_data.functions.base.requests import (
+    RegionalDataMartStatusRequest,
+)
+from edu_rdm_integration.stages.export_data.models import (
+    ExportingDataSubStageStatus,
+)
+from edu_rdm_integration.stages.upload_data.consts import (
+    FAILED_STATUSES,
+)
+from edu_rdm_integration.stages.upload_data.enums import (
+    FileUploadStatusEnum,
+)
+from edu_rdm_integration.stages.upload_data.models import (
+    DataMartRequestStatus,
+    ExportingDataSubStageUploaderClientLog,
+    UploadStatusRequestLog,
+)
+
+
+if TYPE_CHECKING:
+    from uploader_client.logging.base import (
+        Entry,
+    )
+
+
+class UploadStatusHelper:
+    """Хелпер проверки статуса загрузки данных в витрину."""
+
+    def __init__(self, in_progress_uploads: QuerySet, cache: AbstractCache) -> None:
+        self._in_progress_uploads = in_progress_uploads
+        self.cache = cache
+
+    def run(self, thread_count: int = 1) -> None:
+        """Запускает проверки статусов."""
+        if thread_count > 1:
+            with ThreadPoolExecutor(max_workers=thread_count) as pool:
+                pool.map(self._process_upload, self._in_progress_uploads)
+        else:
+            for upload in self._in_progress_uploads:
+                self._process_upload(upload)
+
+    @classmethod
+    def send_upload_status_request(cls, request_id: str) -> tuple[Optional[dict[str, Any]], 'Entry']:
+        """Формирует и отправляет запрос для получения статуса загрузки данных в витрину."""
+        request = RegionalDataMartStatusRequest(
+            request_id=request_id,
+            method='GET',
+            parameters={},
+            headers={
+                'Content-Type': 'application/json',
+            },
+        )
+
+        result = adapter.send(request)
+
+        response = None
+
+        if result.error:
+            logger.warning(
+                f'Ошибка при получении статуса загрузки данных в витрину. Идентификатор загрузки: {request_id}. '
+                f'Ошибка: {result.error}, запрос: {result.log.request}, ответ: {result.log.response}',
+            )
+        else:
+            logger.info(
+                f'Получен ответ со статусом {result.response.status_code} и содержимым {result.response.text}. '
+                f'Идентификатор загрузки: {request_id}',
+            )
+            try:
+                response = result.response.json()
+            except JSONDecodeError:
+                logger.error(
+                    f'Не удалось получить данные из ответа запроса статуса загрузки данных в витрину. '
+                    f'Идентификатор загрузки: {request_id}, ответ: {result.response.text}',
+                )
+
+        return response, result.log
+
+    @classmethod
+    def update_upload_status(
+        cls,
+        upload: ExportingDataSubStageUploaderClientLog,
+        response: Optional[dict[str, Any]],
+        log_entry: 'Entry',
+    ) -> None:
+        """Обновляет статус загрузки данных в витрину."""
+        request_status = None
+
+        if isinstance(response, dict):
+            request_status = DataMartRequestStatus.get_values_to_enum_data().get(response.get('code'))
+
+            if not request_status:
+                logger.error(
+                    'Не удалось определить статус загрузки данных в витрину. Идентификатор загрузки: '
+                    f'{upload.request_id}, данные ответа: {response}',
+                )
+
+        with transaction.atomic():
+            UploadStatusRequestLog.objects.create(
+                upload=upload,
+                entry=log_entry,
+                request_status_id=getattr(request_status, 'key', None),
+            )
+
+            if request_status in FAILED_STATUSES:
+                upload.file_upload_status = FileUploadStatusEnum.ERROR
+                upload.sub_stage.status_id = ExportingDataSubStageStatus.PROCESS_ERROR.key
+                upload.sub_stage.save()
+
+            elif request_status == DataMartRequestStatus.SUCCESSFULLY_PROCESSED:
+                upload.file_upload_status = FileUploadStatusEnum.FINISHED
+
+            if upload.file_upload_status != FileUploadStatusEnum.IN_PROGRESS:
+                upload.save()
+
+    def _process_upload(self, upload: ExportingDataSubStageUploaderClientLog) -> None:
+        """Обрабатывает запись загрузки данных в витрину."""
+        response, log_entry = self.send_upload_status_request(upload.request_id)
+        self.update_upload_status(upload, response, log_entry)
+        # Обновим размер файлов в кеш (с блокировкой на время обновления)
+        with self.cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
+            queue_total_file_size = self.cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+            if queue_total_file_size:
+                queue_total_file_size -= upload.attachment.attachment_size
+                if queue_total_file_size > 0:
+                    self.cache.set(
+                        TOTAL_ATTACHMENTS_SIZE_KEY,
+                        queue_total_file_size,
+                        timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS,
+                    )

edu_rdm_integration/stages/upload_data/management/base.py

@@ -0,0 +1,134 @@
+from typing import (
+    TYPE_CHECKING,
+)
+
+import uploader_client
+from django.core.cache import (
+    DEFAULT_CACHE_ALIAS,
+    caches,
+)
+from django.core.management import (
+    BaseCommand,
+)
+from uploader_client.adapters import (
+    UploaderAdapter,
+)
+from uploader_client.contrib.rdm.interfaces.configurations import (
+    RegionalDataMartUploaderConfig,
+)
+
+
+if TYPE_CHECKING:
+    from uploader_client.interfaces import (
+        OpenAPIRequest,
+    )
+
+
+class BaseDatamartClientCommand(BaseCommand):
+    """Базовая команда для загрузки данных/получение статуса в РВД с использованием uploader_client."""
+
+    TIMEOUT = 300
+    REQUEST_RETRIES = 1
+
+    def add_arguments(self, parser):
+        """Добавление параметров."""
+        parser.add_argument(
+            '--url',
+            type=str,
+            required=True,
+            help='url хоста Datamart Studio',
+        )
+        parser.add_argument(
+            '--datamart_mnemonic',
+            type=str,
+            required=True,
+            help='мнемоника Витрины',
+        )
+        parser.add_argument(
+            '--organization_ogrn',
+            type=str,
+            required=True,
+            help='ОГРН организации, в рамках которой развёрнута Витрина',
+        )
+        parser.add_argument(
+            '--installation_name',
+            type=str,
+            required=True,
+            help='имя инсталляции в целевой Витрине',
+        )
+        parser.add_argument(
+            '--installation_id',
+            type=int,
+            required=True,
+            help='идентификатор инсталляции (присутствует в её названии)',
+        )
+        parser.add_argument(
+            '--username',
+            type=str,
+            required=True,
+            help='имя пользователя IAM',
+        )
+        parser.add_argument(
+            '--password',
+            type=str,
+            required=True,
+            help='пароль пользователя IAM',
+        )
+
+    def _configure_agent_client(
+        self,
+        url,
+        datamart_mnemonic,
+        organization_ogrn,
+        installation_name,
+        installation_id,
+        username,
+        password,
+    ):
+        """Конфигурирование клиента загрузчика данных в Витрину."""
+        uploader_client.set_config(
+            RegionalDataMartUploaderConfig(
+                interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
+                cache=caches[DEFAULT_CACHE_ALIAS],
+                url=url,
+                datamart_name=datamart_mnemonic,
+                organization_ogrn=organization_ogrn,
+                installation_name=installation_name,
+                installation_id=installation_id,
+                username=username,
+                password=password,
+                timeout=self.TIMEOUT,
+                request_retries=self.REQUEST_RETRIES,
+            )
+        )
+
+    def _get_request(self, **options) -> 'OpenAPIRequest':
+        """Возвращает запрос для отправки в РВД."""
+        raise NotImplementedError
+
+    def handle(self, *args, **options):
+        """Выполнение действий команды."""
+        self._configure_agent_client(
+            url=options['url'],
+            datamart_mnemonic=options['datamart_mnemonic'],
+            organization_ogrn=options['organization_ogrn'],
+            installation_name=options['installation_name'],
+            installation_id=options['installation_id'],
+            username=options['username'],
+            password=options['password'],
+        )
+
+        request = self._get_request(**options)
+
+        result = UploaderAdapter().send(request)
+
+        if result.error:
+            self.stdout.write(self.style.ERROR('ERROR:\n'))
+            self.stdout.write(
+                f'{result.error}\nREQUEST:\n"{result.log.request}"\n\nRESPONSE:\n"{result.log.response}"\n'
+            )
+        else:
+            self.stdout.write(self.style.SUCCESS('SUCCESS:\n'))
+            self.stdout.write(
+                f'Response with {result.response.status_code} code and content:\n{result.response.text}\n'
+            )