edu-rdm-integration 3.9.1__py3-none-any.whl → 3.10.0__py3-none-any.whl

edu_rdm_integration/stages/upload_data/operations.py

@@ -0,0 +1,156 @@
+import logging
+import os
+from typing import (
+    Union,
+)
+
+from django.conf import (
+    settings,
+)
+
+from educommon import (
+    logger,
+)
+
+from edu_rdm_integration.core.operations import (
+    BaseOperationData,
+)
+from edu_rdm_integration.core.redis_cache import (
+    AbstractCache,
+)
+from edu_rdm_integration.stages.export_data.consts import (
+    TOTAL_ATTACHMENTS_SIZE_KEY,
+)
+from edu_rdm_integration.stages.upload_data.export_managers import (
+    ExportQueueSender,
+    WorkerSender,
+)
+from edu_rdm_integration.stages.upload_data.queues import (
+    Queue,
+)
+
+
+class UploadData(BaseOperationData):
+    """Класс отправки файлов в витрину."""
+
+    def __init__(
+        self,
+        data_cache: AbstractCache,
+        queue: Queue,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.data_cache = data_cache
+        self.queue = queue
+
+        self._configure_agent_client()
+        self.result = {
+            'total_file_size': 0,  # Общий размер отправленных файлов
+            'queue_is_full': False,  # Признак переполнения очереди
+            'uploaded_entities': '',  # Список сущностей, попавших в выгрузку
+        }
+
+    @property
+    def _log_file_path(self) -> Union[str, bytes]:
+        """Путь до лог файла."""
+        return os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, 'upload_entity.log')
+
+    def _add_file_handler(self) -> None:
+        """Добавляет обработчик логов."""
+        self._file_handler = logging.FileHandler(self._log_file_path)
+
+        logging.getLogger('info_logger').addHandler(self._file_handler)
+        logging.getLogger('exception_logger').addHandler(self._file_handler)
+
+    # TODO https://jira.bars.group/browse/EDUSCHL-22492. Вынужденная мера, т.к. при запуске команды не производится
+    #  проверка готовности конфигов приложений. Нужно переработать механизм конфигурирования клиента загрузчика.
+    def _configure_agent_client(self):
+        """Конфигурирование клиента загрузчика данных в Витрину."""
+        import uploader_client
+        from django.core.cache import (
+            DEFAULT_CACHE_ALIAS,
+            caches,
+        )
+        from uploader_client.contrib.rdm.interfaces.configurations import (
+            RegionalDataMartEmulationUploaderConfig,
+            RegionalDataMartUploaderConfig,
+        )
+
+        if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
+            uploader_client.set_config(
+                RegionalDataMartEmulationUploaderConfig(
+                    interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=1,
+                    request_retries=1,
+                    file_status=settings.RDM_RESPONSE_FILE_STATUS,
+                )
+            )
+        elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
+            uploader_client.set_config(
+                RegionalDataMartUploaderConfig(
+                    interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
+                    cache=caches[DEFAULT_CACHE_ALIAS],
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
+                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
+                    organization_ogrn=settings.RDM_UPLOADER_CLIENT_ORGANIZATION_OGRN,
+                    installation_name=settings.RDM_UPLOADER_CLIENT_INSTALLATION_NAME,
+                    installation_id=settings.RDM_UPLOADER_CLIENT_INSTALLATION_ID,
+                    username=settings.RDM_UPLOADER_CLIENT_USERNAME,
+                    password=settings.RDM_UPLOADER_CLIENT_PASSWORD,
+                )
+            )
+        else:
+            uploader_client.set_config(
+                RegionalDataMartUploaderConfig(
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
+                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
+                )
+            )
+
+    def update_total_queue_size_in_cache(self, received_files_size: int):
+        """Обновление размера файлов в кеш."""
+        with self.data_cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
+            queue_total_file_size = self.data_cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+            if queue_total_file_size:
+                queue_total_file_size -= received_files_size
+                if queue_total_file_size > 0:
+                    self.data_cache.set(
+                        TOTAL_ATTACHMENTS_SIZE_KEY,
+                        queue_total_file_size,
+                        timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS,
+                    )
+
+    def upload_data(self, *args, **kwargs):
+        """Запускает отправку данных в витрину."""
+        try:
+            exporter = ExportQueueSender(self.data_cache, self.queue, settings.RDM_UPLOAD_DATA_TASK_EXPORT_STAGES)
+            exporter.run()
+
+            self.result['queue_is_full'] = exporter.queue_is_full
+            self.result['total_file_size'] = exporter.queue_total_file_size
+
+            # Если очередь не переполнена - то отправляем данные в витрину
+            if not exporter.queue_is_full:
+                sender = WorkerSender(self.queue)
+                sender.run()
+
+                if sender.entities:
+                    self.result['uploaded_entities'] = ','.join(sender.entities)
+
+                if sender.received_file_size:
+                    self.update_total_queue_size_in_cache(sender.received_file_size)
+
+        except Exception as err:
+            logger.exception(err)
+            raise err
+        finally:
+            self._remove_file_handler()
+
+        return self.result

edu_rdm_integration/{export_data/queue.py → stages/upload_data/queues.py}

@@ -15,20 +15,21 @@ from redis import (
     Redis,
 )
 
-from edu_rdm_integration.export_data.consts import (
+from edu_rdm_integration.core.helpers import (
+    as_text,
+    get_redis_version,
+)
+from edu_rdm_integration.stages.export_data.consts import (
     REDIS_QUEUE_KEY_DELIMITER,
 )
-from edu_rdm_integration.export_data.dataclasses import (
+from edu_rdm_integration.stages.upload_data.dataclasses import (
     UploadFile,
 )
-from edu_rdm_integration.redis_cache import (
-    as_text,
-    get_redis_version,
-)
 
 
 class Queue(ABC):
     """Интерфейс очереди."""
+
     queue_key: str = ''
 
     @property
@@ -65,6 +66,7 @@ class RdmRedisSubStageAttachmentQueue(Queue):
     (Sorted Set in Redis)
     - Информация по файлам стандартно по ключу - ключом выступает sub_stage_id
     """
+
     queue_key = 'rdm:export_sub_stage_ids_queue'
     prefix = 'rdm:'
 
@@ -76,7 +78,7 @@ class RdmRedisSubStageAttachmentQueue(Queue):
             host=settings.RDM_REDIS_HOST,
             port=settings.RDM_REDIS_PORT,
             db=settings.RDM_REDIS_DB,
-            password=settings.RDM_REDIS_PASSWORD
+            password=settings.RDM_REDIS_PASSWORD,
         )
 
     def _make_key(self, key: Union[int, str]) -> str:
@@ -143,7 +145,7 @@ class RdmRedisSubStageAttachmentQueue(Queue):
             while true do
                 local stage = redis.call("zpopmin", q)
                 local stage_id = stage[2]
-                
+
                 if stage_id == nil then
                     break
                 end

edu_rdm_integration/stages/upload_data/tasks.py

@@ -0,0 +1,103 @@
+import celery
+from celery.schedules import (
+    crontab,
+)
+from django.conf import (
+    settings,
+)
+from django.core.cache import (
+    cache,
+)
+
+from educommon.async_task.models import (
+    AsyncTaskType,
+)
+from educommon.async_task.tasks import (
+    UniquePeriodicAsyncTask,
+)
+
+from edu_rdm_integration.core.consts import (
+    TASK_QUEUE_NAME,
+)
+from edu_rdm_integration.stages.upload_data.enums import (
+    FileUploadStatusEnum,
+)
+from edu_rdm_integration.stages.upload_data.helpers import (
+    UploadStatusHelper,
+)
+from edu_rdm_integration.stages.upload_data.models import (
+    ExportingDataSubStageUploaderClientLog,
+)
+from edu_rdm_integration.stages.upload_data.operations import (
+    UploadData,
+)
+from edu_rdm_integration.stages.upload_data.queues import (
+    RdmRedisSubStageAttachmentQueue,
+)
+
+
+class RDMCheckUploadStatus(UniquePeriodicAsyncTask):
+    """Периодическая задача для сбора статусов по загрузке файла в витрину."""
+
+    queue = TASK_QUEUE_NAME
+    routing_key = TASK_QUEUE_NAME
+    description = 'Сбор статусов загрузки данных в витрину "Региональная витрина данных"'
+    lock_expire_seconds = settings.RDM_UPLOAD_STATUS_TASK_LOCK_EXPIRE_SECONDS
+    task_type = AsyncTaskType.UNKNOWN
+    run_every = crontab(
+        minute=settings.RDM_UPLOAD_STATUS_TASK_MINUTE,
+        hour=settings.RDM_UPLOAD_STATUS_TASK_HOUR,
+        day_of_week=settings.RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK,
+    )
+
+    def process(self, *args, **kwargs):
+        """Выполнение."""
+        super().process(*args, **kwargs)
+
+        # Получаем незавершенные загрузки данных в витрину
+        in_progress_uploads = ExportingDataSubStageUploaderClientLog.objects.filter(
+            file_upload_status=FileUploadStatusEnum.IN_PROGRESS,
+            is_emulation=False,
+        ).select_related('attachment')
+
+        UploadStatusHelper(in_progress_uploads, cache).run()
+
+
+class UploadDataAsyncTask(UniquePeriodicAsyncTask):
+    """Формирование очереди файлов и их отправка."""
+
+    queue = TASK_QUEUE_NAME
+    routing_key = TASK_QUEUE_NAME
+    description = 'Отправка данных в витрину "Региональная витрина данных"'
+    lock_expire_seconds = settings.RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS
+    task_type = AsyncTaskType.SYSTEM
+    run_every = crontab(
+        minute=settings.RDM_UPLOAD_DATA_TASK_MINUTE,
+        hour=settings.RDM_UPLOAD_DATA_TASK_HOUR,
+        day_of_week=settings.RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK,
+    )
+
+    def process(self, *args, **kwargs):
+        """Выполнение."""
+        super().process(*args, **kwargs)
+
+        queue = RdmRedisSubStageAttachmentQueue()
+        upload_data = UploadData(
+            data_cache=cache,
+            queue=queue,
+        )
+
+        upload_result = upload_data.upload_data()
+
+        task_result = {
+            'Общий объем отправленных файлов': f'{upload_result["total_file_size"]}',
+            'Очередь отправки переполнена': 'Да' if upload_result['queue_is_full'] else 'Нет',
+            'Сущности, отправленные в витрину': upload_result['uploaded_entities'],
+        }
+
+        self.set_progress(values=task_result)
+
+
+celery_app = celery.app.app_or_default()
+celery_app.register_task(RDMCheckUploadStatus)
+celery_app.register_task(UploadDataAsyncTask)

edu_rdm_integration/{uploader_log → stages/upload_data/uploader_log}/actions.py

@@ -14,10 +14,6 @@ from typing import (
 from django.conf import (
     settings,
 )
-from edu_rdm_integration.uploader_log.enums import (
-    RequestResultStatus,
-)
-
 from django.db.models import (
     Q,
 )
@@ -27,6 +23,9 @@ from m3_ext.ui.icons import (
 from objectpack.actions import (
     ObjectPack,
 )
+from objectpack.filters import (
+    CustomFilter,
+)
 
 from educommon.objectpack.actions import (
     ExtObjectRowsAction,
@@ -44,21 +43,19 @@ from educommon.utils.ui import (
     append_template_globals,
 )
 
-from edu_rdm_integration.models import (
+from edu_rdm_integration.stages.upload_data.uploader_log.enums import (
+    RequestResultStatus,
+)
+from edu_rdm_integration.stages.upload_data.uploader_log.models import (
     UploaderClientLog,
 )
-from edu_rdm_integration.uploader_log.ui import (
+from edu_rdm_integration.stages.upload_data.uploader_log.ui import (
     UploaderLogInfoWindow,
 )
-from objectpack.filters import (
-    CustomFilter,
-)
 
 
 class UploaderLogPack(ObjectPack):
-    """
-    Пак журнала Загрузчика данных в витрину.
-    """
+    """Пак журнала Загрузчика данных в витрину."""
 
     title = 'Журнал логов РВД'
     model = UploaderClientLog
@@ -104,13 +101,14 @@ class UploaderLogPack(ObjectPack):
                 tooltip='C',
                 default_value=date.today() - timedelta(days=1),
                 editable=False,
-                lookup=lambda v: Q(request_datetime__gte=datetime.combine(v, time.min))
-            ) & _fd(
+                lookup=lambda v: Q(request_datetime__gte=datetime.combine(v, time.min)),
+            )
+            & _fd(
                 'request_datetime',
                 tooltip='По',
                 default_value=date.today,
                 editable=False,
-                lookup=lambda v: Q(request_datetime__lte=datetime.combine(v, time.max))
+                lookup=lambda v: Q(request_datetime__lte=datetime.combine(v, time.max)),
             ),
             'width': 11,
         },
@@ -251,10 +249,10 @@ class UploaderLogPack(ObjectPack):
 
         Args:
             menu: Объект меню.
+
         Returns:
             Подменю "Администрирование -> Региональная витрина данных -> Журнал логов РВД".
         """
-
         if settings.RDM_MENU_ITEM:
             return menu.SubMenu(
                 'Администрирование',

edu_rdm_integration/stages/upload_data/uploader_log/apps.py

@@ -0,0 +1,11 @@
+from django.apps import (
+    AppConfig,
+)
+
+
+class UploaderLoggerConfig(AppConfig):  # noqa D101
+    """Приложение для работы с логами выгрузки данных в РВД."""
+
+    name = 'edu_rdm_integration.stages.upload_data.uploader_log'
+    label = 'edu_rdm_integration_uploader_log'
+    verbose_name = 'Журнал логов РВД'

edu_rdm_integration/{uploader_log → stages/upload_data/uploader_log}/managers.py

@@ -12,7 +12,7 @@ from django.db.models.functions import (
     Cast,
 )
 
-from edu_rdm_integration.uploader_log.enums import (
+from edu_rdm_integration.stages.upload_data.uploader_log.enums import (
     RequestResultStatus,
 )
 
@@ -26,9 +26,8 @@ class UploaderClientLogManager(Manager):
 
         result_status = Case(
             When(
-                Q(Q(error__isnull=True) | Q(error__exact=''))
-                & Q(Q(response__isnull=False) & ~Q(response__exact='')),
-                then=Value(RequestResultStatus.SUCCESS)
+                Q(Q(error__isnull=True) | Q(error__exact='')) & Q(Q(response__isnull=False) & ~Q(response__exact='')),
+                then=Value(RequestResultStatus.SUCCESS),
             ),
             default=Value(RequestResultStatus.ERROR),
             output_field=PositiveSmallIntegerField(),
@@ -37,30 +36,21 @@ class UploaderClientLogManager(Manager):
         query = query.annotate(
             request_datetime=F('date_time'),
             attachment_file=Case(
-                When(
-                    request__icontains='POST',
-                    then=F('uploader_client_log__attachment__attachment')
-                ),
+                When(request__icontains='POST', then=F('uploader_client_log__attachment__attachment')),
                 When(
                     request__icontains='GET',
                     then=F('upload_status_request_log__upload__attachment__attachment'),
                 ),
                 default=Value(''),
-                output_field = CharField(),
+                output_field=CharField(),
             ),
             status_code=Case(
-                When(
-                    request__icontains='POST',
-                    then=Value('', output_field=CharField())
-                ),
+                When(request__icontains='POST', then=Value('', output_field=CharField())),
                 When(
                     request__icontains='GET',
-                    then=Cast(
-                        'upload_status_request_log__request_status__value',
-                        output_field=CharField()
-                    ),
+                    then=Cast('upload_status_request_log__request_status__value', output_field=CharField()),
                 ),
-                default=Value('')
+                default=Value(''),
             ),
             status_description=Case(
                 When(
@@ -71,7 +61,8 @@ class UploaderClientLogManager(Manager):
                     request__icontains='GET',
                     then=F('upload_status_request_log__request_status__title'),
                 ),
-                default=Value(''), output_field=CharField()
+                default=Value(''),
+                output_field=CharField(),
             ),
             is_emulation=Case(
                 When(
@@ -82,20 +73,14 @@ class UploaderClientLogManager(Manager):
                     request__icontains='GET',
                     then=F('upload_status_request_log__upload__is_emulation'),
                 ),
-                default=Value(False)
+                default=Value(False),
             ),
             request_id=Case(
-                When(
-                    request__icontains='POST',
-                    then=F('uploader_client_log__request_id')
-                ),
-                When(
-                    request__icontains='GET',
-                    then=F('upload_status_request_log__upload__request_id')
-                ),
-                default=Value('')
+                When(request__icontains='POST', then=F('uploader_client_log__request_id')),
+                When(request__icontains='GET', then=F('upload_status_request_log__upload__request_id')),
+                default=Value(''),
             ),
-            result_status=result_status
+            result_status=result_status,
         )
 
         return query

edu_rdm_integration/stages/upload_data/uploader_log/migrations/0001_initial.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.24 on 2025-06-28 14:22
+
+from django.db import (
+    migrations,
+)
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ('uploader_client', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.CreateModel(
+                    name='UploaderClientLog',
+                    fields=[],
+                    options={
+                        'proxy': True,
+                        'indexes': [],
+                        'constraints': [],
+                    },
+                    bases=('uploader_client.entry',),
+                ),
+            ],
+            database_operations=[],
+        ),
+    ]

edu_rdm_integration/stages/upload_data/uploader_log/models.py

@@ -0,0 +1,86 @@
+from functools import (
+    cached_property,
+)
+from io import (
+    StringIO,
+)
+from typing import (
+    Optional,
+)
+
+from django.db.models import (
+    Manager,
+)
+from uploader_client.models import (
+    Entry,
+)
+
+from edu_rdm_integration.stages.upload_data.uploader_log.enums import (
+    RequestResultStatus,
+)
+from edu_rdm_integration.stages.upload_data.uploader_log.managers import (
+    UploaderClientLogManager,
+)
+
+
+class UploaderClientLog(Entry):
+    """Прокси модель Загрузчика данных в витрину."""
+
+    objects = UploaderClientLogManager()
+    base_objects = Manager()
+
+    @cached_property
+    def http_method_and_url(self) -> tuple[str, str]:
+        """Возвращает http-метод и url из поля запроса Entry.request."""
+        request = StringIO(self.request)
+        request_first_line = request.readline()
+        request.close()
+
+        method, url = request_first_line.split(' ')[:2]
+        if not (method and url.startswith('http')):
+            method = url = ''
+
+        return method.strip('[]'), url
+
+    @cached_property
+    def http_response_status(self) -> Optional[str]:
+        """Статус-код запроса к витрине."""
+        try:
+            http_status = self.response.split(' ')[0].strip('[]')
+
+            if not http_status:
+                return None
+
+            if 200 <= int(http_status) < 300:
+                http_status = 'Успех'
+            elif int(http_status) >= 400:
+                http_status = 'Ошибка'
+        except (IndexError, AttributeError):
+            http_status = None
+
+        return http_status
+
+    @property
+    def http_method(self) -> str:
+        """Значение http-метода."""
+        return self.http_method_and_url[0]
+
+    @property
+    def request_url(self) -> str:
+        """URL запроса."""
+        return self.http_method_and_url[1]
+
+    @property
+    def request_error(self) -> Optional[str]:
+        """Ошибка запроса."""
+        return self.error
+
+    @property
+    def result_status_display(self) -> str:
+        """Результат запроса."""
+        result_status = getattr(self, 'result_status', RequestResultStatus.ERROR)
+
+        return RequestResultStatus.values.get(result_status) or RequestResultStatus.values[RequestResultStatus.ERROR]
+
+    class Meta:
+        proxy = True

edu_rdm_integration/{uploader_log → stages/upload_data/uploader_log}/ui.py

@@ -45,12 +45,14 @@ class UploaderLogInfoWindow(BaseEditWindow):
         """Размещение компонентов окна на форме."""
         super(UploaderLogInfoWindow, self)._do_layout()
 
-        self.form.items.extend((
-            self.field__request,
-            self.field__response,
-            self.field__error,
-            self.field__attachment,
-        ))
+        self.form.items.extend(
+            (
+                self.field__request,
+                self.field__response,
+                self.field__error,
+                self.field__attachment,
+            )
+        )
 
     def set_params(self, params):
         """Настройка окна."""